diff --git a/2.0/src/client/CMakeLists.txt b/2.0/src/client/CMakeLists.txt deleted file mode 100644 index df0cf15fe00eda5bc8e6004e2794733537b8aee0..0000000000000000000000000000000000000000 --- a/2.0/src/client/CMakeLists.txt +++ /dev/null @@ -1,95 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(inc) -INCLUDE_DIRECTORIES(jni) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/plugins/http/inc) -AUX_SOURCE_DIRECTORY(src SRC) - -IF (TD_LINUX) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) - - # set the static lib name - ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m rt cJson ${VAR_TSZ}) - SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") - SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) - - # generate dynamic library (*.so) - ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m rt cJson) - IF (TD_LINUX_64) - TARGET_LINK_LIBRARIES(taos lua) - ENDIF () - - SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) - - #set version of .so - #VERSION so version - #SOVERSION api version - #MESSAGE(STATUS "build version ${TD_VER_NUMBER}") - SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) - - ADD_SUBDIRECTORY(tests) - -ELSEIF (TD_DARWIN) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) - - # set the static lib name - ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m lua cJson) - SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") - SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) - - # generate dynamic library (*.dylib) - ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m lua cJson) - SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) - - #set version of .dylib - #VERSION dylib version - #SOVERSION dylib version - #MESSAGE(STATUS "build version ${TD_VER_NUMBER}") - SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) - - ADD_SUBDIRECTORY(tests) - -ELSEIF (TD_WINDOWS) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32) - - CONFIGURE_FILE("${TD_COMMUNITY_DIR}/src/client/src/taos.rc.in" "${TD_COMMUNITY_DIR}/src/client/src/taos.rc") - - ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static trpc tutil query cJson) - - # generate dynamic library (*.dll) - ADD_LIBRARY(taos SHARED ${SRC} ${TD_COMMUNITY_DIR}/src/client/src/taos.rc) - IF (NOT TD_GODLL) - SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_COMMUNITY_DIR}/src/client/src/taos.def) - ENDIF () - TARGET_LINK_LIBRARIES(taos trpc tutil query lua cJson) - -ELSEIF (TD_DARWIN) - SET(CMAKE_MACOSX_RPATH 1) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) - - ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m lua cJson) - SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") - - # generate dynamic library (*.dylib) - ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m lua cJson) - - SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) - - #set version of .so - #VERSION so version - #SOVERSION api version - #MESSAGE(STATUS "build version ${TD_VER_NUMBER}") - SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) -ENDIF () - diff --git a/2.0/src/client/inc/tscGlobalmerge.h b/2.0/src/client/inc/tscGlobalmerge.h deleted file mode 100644 index 7aa16f843f3d40f8a183a076bc781270d206c35a..0000000000000000000000000000000000000000 --- a/2.0/src/client/inc/tscGlobalmerge.h +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSCGLOBALMERGE_H -#define TDENGINE_TSCGLOBALMERGE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "qExtbuffer.h" -#include "qFill.h" -#include "tmsg.h" -#include "tlosertree.h" -#include "qExecutor.h" - -#define MAX_NUM_OF_SUBQUERY_RETRY 3 - -struct SQLFunctionCtx; - -typedef struct SLocalDataSource { - tExtMemBuffer *pMemBuffer; - int32_t flushoutIdx; - int32_t pageId; - int32_t rowIdx; - tFilePage filePage; -} SLocalDataSource; - -typedef struct SGlobalMerger { - SLocalDataSource **pLocalDataSrc; - int32_t numOfBuffer; - int32_t numOfCompleted; - int32_t numOfVnode; - SLoserTreeInfo *pLoserTree; - int32_t rowSize; // size of each intermediate result. - tOrderDescriptor *pDesc; - tExtMemBuffer **pExtMemBuffer; // disk-based buffer - char *buf; // temp buffer -} SGlobalMerger; - -struct SSqlObj; - -typedef struct SRetrieveSupport { - tExtMemBuffer ** pExtMemBuffer; // for build loser tree - tOrderDescriptor *pOrderDescriptor; - int32_t subqueryIndex; // index of current vnode in vnode list - struct SSqlObj *pParentSql; - tFilePage * localBuffer; // temp buffer, there is a buffer for each vnode to - uint32_t numOfRetry; // record the number of retry times -} SRetrieveSupport; - -int32_t tscCreateGlobalMergerEnv(SQueryInfo* pQueryInfo, tExtMemBuffer ***pMemBuffer, int32_t numOfSub, tOrderDescriptor **pDesc, uint32_t nBufferSize, int64_t id); - -void tscDestroyGlobalMergerEnv(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDesc, int32_t numOfVnodes); - -int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, void *data, - int32_t numOfRows, int32_t orderType); - -int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, int32_t orderType); - -/* - * create local reducer to launch the second-stage reduce process at client site - */ -int32_t tscCreateGlobalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, - SQueryInfo *pQueryInfo, SGlobalMerger **pMerger, int64_t id); - -void tscDestroyGlobalMerger(SGlobalMerger* pMerger); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TSCGLOBALMERGE_H diff --git a/2.0/src/client/inc/tscLog.h b/2.0/src/client/inc/tscLog.h deleted file mode 100644 index f25ec02bd8204d124477e6eba1070520b4d00c4e..0000000000000000000000000000000000000000 --- a/2.0/src/client/inc/tscLog.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSCLOG_H -#define TDENGINE_TSCLOG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "tlog.h" - -extern uint32_t cDebugFlag; -extern int8_t tscEmbedded; - -#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) -#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) -#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) -#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) -#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/2.0/src/client/inc/tscParseLine.h b/2.0/src/client/inc/tscParseLine.h deleted file mode 100644 index e7c5e548414d54e8ebcaac25aa2677e4819d3195..0000000000000000000000000000000000000000 --- a/2.0/src/client/inc/tscParseLine.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2021 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSCPARSELINE_H -#define TDENGINE_TSCPARSELINE_H - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - char* key; - uint8_t type; - int16_t length; - char* value; -} TAOS_SML_KV; - -typedef struct { - char* stbName; - - char* childTableName; - TAOS_SML_KV* tags; - int32_t tagNum; - - // first kv must be timestamp - TAOS_SML_KV* fields; - int32_t fieldNum; -} TAOS_SML_DATA_POINT; - -typedef enum { - SML_TIME_STAMP_NOW, - SML_TIME_STAMP_SECONDS, - SML_TIME_STAMP_MILLI_SECONDS, - SML_TIME_STAMP_MICRO_SECONDS, - SML_TIME_STAMP_NANO_SECONDS -} SMLTimeStampType; - -typedef struct { - uint64_t id; - SHashObj* smlDataToSchema; -} SSmlLinesInfo; - -int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info); -bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info); -int32_t isValidChildTableName(const char *pTbName, int16_t len); - -bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, - uint16_t len, SSmlLinesInfo* info); -int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, - uint16_t len, SSmlLinesInfo* info); - -void destroySmlDataPoint(TAOS_SML_DATA_POINT* point); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TSCPARSELINE_H diff --git a/2.0/src/client/inc/tscSubquery.h b/2.0/src/client/inc/tscSubquery.h deleted file mode 100644 index a012ca5a7fe741b8859465504cbc971a7e46952c..0000000000000000000000000000000000000000 --- a/2.0/src/client/inc/tscSubquery.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSCSUBQUERY_H -#define TDENGINE_TSCSUBQUERY_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "tscUtil.h" -#include "tsclient.h" - -void tscFetchDatablockForSubquery(SSqlObj* pSql); - -void tscSetupOutputColumnIndex(SSqlObj* pSql); -void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code); - -SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index); - -void tscHandleMasterJoinQuery(SSqlObj* pSql); - -int32_t tscHandleMasterSTableQuery(SSqlObj *pSql); -int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql); - -int32_t tscHandleMultivnodeInsert(SSqlObj *pSql); - -int32_t tscHandleInsertRetry(SSqlObj* parent, SSqlObj* child); - -void tscBuildResFromSubqueries(SSqlObj *pSql); -TAOS_ROW doSetResultRowData(SSqlObj *pSql); - -char *getArithmeticInputSrc(void *param, const char *name, int32_t colId); - -void tscLockByThread(int64_t *lockedBy); - -void tscUnlockByThread(int64_t *lockedBy); - -int tsInsertInitialCheck(SSqlObj *pSql); - -void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs); - -void tscFreeRetrieveSup(SSqlObj *pSql); - - - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TSCSUBQUERY_H diff --git a/2.0/src/client/inc/tscUtil.h b/2.0/src/client/inc/tscUtil.h deleted file mode 100644 index c858bd5867c64da4c7397aed2035119ff414d112..0000000000000000000000000000000000000000 --- a/2.0/src/client/inc/tscUtil.h +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSCUTIL_H -#define TDENGINE_TSCUTIL_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "exception.h" -#include "os.h" -#include "qExtbuffer.h" -#include "taosdef.h" -#include "tbuffer.h" -#include "tscGlobalmerge.h" -#include "tsched.h" -#include "tsclient.h" - -#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo) \ - (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE)) - -#define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \ - (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE)) - -#define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo) \ - (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo))) - -#define UTIL_TABLE_IS_TMP_TABLE(metaInfo) \ - (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_TEMP_TABLE)) - -#pragma pack(push,1) -// this struct is transfered as binary, padding two bytes to avoid -// an 'uid' whose low bytes is 0xff being recoginized as NULL, -// and set 'pack' to 1 to avoid break existing code. -typedef struct STidTags { - int16_t padding; - int64_t uid; - int32_t tid; - int32_t vgId; - char tag[]; -} STidTags; -#pragma pack(pop) - -typedef struct SJoinSupporter { - SSqlObj* pObj; // parent SqlObj - int32_t subqueryIndex; // index of sub query - SInterval interval; - SLimitVal limit; // limit info - uint64_t uid; // query table uid - SArray* colList; // previous query information, no need to use this attribute, and the corresponding attribution - SArray* exprList; - SArray* colCond; - SFieldInfo fieldsInfo; - STagCond tagCond; - SGroupbyExpr groupInfo; // group by info - struct STSBuf* pTSBuf; // the TSBuf struct that holds the compressed timestamp array - FILE* f; // temporary file in order to create TSBuf - char path[PATH_MAX]; // temporary file path, todo dynamic allocate memory - int32_t tagSize; // the length of each in the first filter stage - char* pIdTagList; // result of first stage tags - int32_t totalLen; - int32_t num; - SArray* pVgroupTables; -} SJoinSupporter; - - -typedef struct SMergeCtx { - SJoinSupporter* p; - int32_t idx; - SArray* res; - int8_t compared; -}SMergeCtx; - -typedef struct SMergeTsCtx { - SJoinSupporter* p; - STSBuf* res; - int64_t numOfInput; - int8_t compared; -}SMergeTsCtx; - -typedef struct SVgroupTableInfo { - SVgroupMsg vgInfo; - SArray *itemList; // SArray -} SVgroupTableInfo; - -typedef struct SBlockKeyTuple { - TSKEY skey; - void* payloadAddr; -} SBlockKeyTuple; - -typedef struct SBlockKeyInfo { - int32_t maxBytesAlloc; - SBlockKeyTuple* pKeyTuple; -} SBlockKeyInfo; - -int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len); - -int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); -void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta); -void tscSortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf); -int tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo); -int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows); - -void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo); -void doRetrieveSubqueryData(SSchedMsg *pMsg); - -SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, - uint32_t offset); - -void* tscDestroyBlockArrayList(SArray* pDataBlockList); -void* tscDestroyUdfArrayList(SArray* pUdfList); -void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta); - -int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock); -int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBlockMap); -int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, SName* pName, STableMeta* pTableMeta, - STableDataBlocks** dataBlocks, SArray* pBlockList); - -/** - * for the projection query on metric or point interpolation query on metric, - * we iterate all the meters, instead of invoke query on all qualified meters simultaneously. - * - * @param pSql sql object - * @return - */ -bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo); -bool tscIsTWAQuery(SQueryInfo* pQueryInfo); -bool tscIsIrateQuery(SQueryInfo* pQueryInfo); - -bool tscIsSessionWindowQuery(SQueryInfo* pQueryInfo); -bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo); -bool tsIsArithmeticQueryOnAggResult(SQueryInfo* pQueryInfo); -bool tscGroupbyColumn(SQueryInfo* pQueryInfo); -int32_t tscGetTopBotQueryExprIndex(SQueryInfo* pQueryInfo); -bool tscIsTopBotQuery(SQueryInfo* pQueryInfo); -bool hasTagValOutput(SQueryInfo* pQueryInfo); -bool timeWindowInterpoRequired(SQueryInfo *pQueryInfo); -bool isStabledev(SQueryInfo* pQueryInfo); -bool isTsCompQuery(SQueryInfo* pQueryInfo); -bool isBlockDistQuery(SQueryInfo* pQueryInfo); -bool isSimpleAggregateRv(SQueryInfo* pQueryInfo); - -bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo *pQueryInfo, int32_t tableIndex); -bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex); -bool tscIsDiffDerivQuery(SQueryInfo* pQueryInfo); -bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex); - -bool tscIsProjectionQuery(SQueryInfo* pQueryInfo); -bool tscHasColumnFilter(SQueryInfo* pQueryInfo); - -bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); -bool tscQueryTags(SQueryInfo* pQueryInfo); -bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); -bool tscQueryBlockInfo(SQueryInfo* pQueryInfo); - -SExprInfo* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, - SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType, int16_t colId); - -int32_t tscSetTableFullName(SName* pName, SStrToken* pzTableName, SSqlObj* pSql); -void tscClearInterpInfo(SQueryInfo* pQueryInfo); - -bool tscIsInsertData(char* sqlstr); - -// the memory is not reset in case of fast allocate payload function -int32_t tscAllocPayloadFast(SSqlCmd *pCmd, size_t size); -int32_t tscAllocPayload(SSqlCmd* pCmd, int size); - -TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes); - -SInternalField* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField); -SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field); - -SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index); -TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index); - -void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo); - -int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); -void tscFieldInfoClear(SFieldInfo* pFieldInfo); -void tscFieldInfoCopy(SFieldInfo* pFieldInfo, const SFieldInfo* pSrc, const SArray* pExprList); - -static FORCE_INLINE int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutput; } -int32_t tscGetFirstInvisibleFieldPos(SQueryInfo* pQueryInfo); - -int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2, int32_t *diffSize); -void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, uint64_t uid); - -int32_t tscFieldInfoSetSize(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2); -void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes); - -int32_t tscGetResRowLength(SArray* pExprList); - -SExprInfo* tscExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t resColId, int16_t interSize, bool isTagCol); - -SExprInfo* tscExprCreate(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t resColId, int16_t interSize, int32_t colType); - -void tscExprAddParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes); - -SExprInfo* tscExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t resColId, int16_t interSize, bool isTagCol); - -SExprInfo* tscExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, - int16_t size); - -size_t tscNumOfExprs(SQueryInfo* pQueryInfo); -int32_t tscExprTopBottomIndex(SQueryInfo* pQueryInfo); -SExprInfo *tscExprGet(SQueryInfo* pQueryInfo, int32_t index); -int32_t tscExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); -int32_t tscExprCopyAll(SArray* dst, const SArray* src, bool deepcopy); -void tscExprAssign(SExprInfo* dst, const SExprInfo* src); -void tscExprDestroy(SArray* pExprInfo); - -int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num); - -void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta, uint64_t id); - -SColumn* tscColumnClone(const SColumn* src); -void tscColumnCopy(SColumn* pDest, const SColumn* pSrc); -int32_t tscColumnExists(SArray* pColumnList, int32_t columnId, uint64_t uid); -SColumn* tscColumnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema); -void tscColumnListDestroy(SArray* pColList); -void tscColumnListCopy(SArray* dst, const SArray* src, uint64_t tableUid); -void tscColumnListCopyAll(SArray* dst, const SArray* src); - -void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, bool convertNchar); - -void tscDequoteAndTrimToken(SStrToken* pToken); -int32_t tscValidateName(SStrToken* pToken); - -void tscIncStreamExecutionCount(void* pStream); - -bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams); - -// get starter position of metric query condition (query on tags) in SSqlCmd.payload -SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid); -void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw); - -int32_t tscTagCondCopy(STagCond* dest, const STagCond* src); -int32_t tscColCondCopy(SArray** dest, const SArray* src, uint64_t uid, int16_t tidx); -void tscTagCondRelease(STagCond* pCond); -void tscColCondRelease(SArray** pCond); -void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo); - -bool tscShouldBeFreed(SSqlObj* pSql); - -STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t tableIndex); -STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); - -void tscInitQueryInfo(SQueryInfo* pQueryInfo); -void tscClearSubqueryInfo(SSqlCmd* pCmd); -int32_t tscAddQueryInfo(SSqlCmd *pCmd); -SQueryInfo *tscGetQueryInfo(SSqlCmd* pCmd); -SQueryInfo *tscGetQueryInfoS(SSqlCmd *pCmd); - -void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo); - -STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableMeta* pTableMeta, - SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables); - -STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo); - -void tscFreeVgroupTableInfo(SArray* pVgroupTables); -SArray* tscVgroupTableInfoDup(SArray* pVgroupTables); -void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index); -void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo); - -int tscGetSTableVgroupInfo(SSqlObj* pSql, SQueryInfo* pQueryInfo); -int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); -int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists, bool onlyLocal); -int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo); - -void tscResetForNextRetrieve(SSqlRes* pRes); -void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo); -void doExecuteQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo); - -SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *pInfo); -void* tscVgroupInfoClear(SVgroupsInfo *pInfo); - -#if 0 -void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src); -#endif - -/** - * The create object function must be successful expect for the out of memory issue. - * - * Therefore, the metermeta/metricmeta object is directly passed to the newly created subquery object from the - * previous sql object, instead of retrieving the metermeta/metricmeta from cache. - * - * Because the metermeta/metricmeta may have been released by other threads, resulting in the retrieving failed as - * well as the create function. - * - * @param pSql - * @param vnodeIndex - * @param tableIndex - * @param fp - * @param param - * @param pPrevSql - * @return - */ -SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, int32_t cmd); - -void registerSqlObj(SSqlObj* pSql); -void tscInitResForMerge(SSqlRes* pRes); - -SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t fp, void* param, int32_t cmd, SSqlObj* pPrevSql); -void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex); - -void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex, SSqlCmd* pCmd); - -int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid); -int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId); -int32_t doInitSubState(SSqlObj* pSql, int32_t numOfSubqueries); - -void tscPrintSelNodeList(SSqlObj* pSql, int32_t subClauseIndex); - -bool hasMoreVnodesToTry(SSqlObj *pSql); -bool hasMoreClauseToTry(SSqlObj* pSql); - -void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeCachedMeta, uint64_t id); - -void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); -void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp); -int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corEpSet); -int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone); - -int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray); - -bool subAndCheckDone(SSqlObj *pSql, SSqlObj *pParentSql, int idx); - -bool tscSetSqlOwner(SSqlObj* pSql); -void tscClearSqlOwner(SSqlObj* pSql); -int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize); - -char* serializeTagData(STagData* pTagData, char* pMsg); -int32_t copyTagData(STagData* dst, const STagData* src); - -STableMeta* createSuperTableMeta(STableMetaMsg* pChild); -uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); -CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); -uint32_t tscGetTableMetaMaxSize(); -int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity, STableMeta **ppStable); -STableMeta* tscTableMetaDup(STableMeta* pTableMeta); -SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo); - -int32_t tscGetTagFilterSerializeLen(SQueryInfo* pQueryInfo); -int32_t tscGetColFilterSerializeLen(SQueryInfo* pQueryInfo); -int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr); -void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, STableGroupInfo* pTableGroupInfo, SOperatorInfo* pOperator, char* sql, void* addr, int32_t stage, uint64_t qId); - -void* malloc_throw(size_t size); -void* calloc_throw(size_t nmemb, size_t size); -char* strdup_throw(const char* str); - -bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src); -SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg); -STblCond* tsGetTableFilter(SArray* filters, uint64_t uid, int16_t idx); - -void tscRemoveCachedTableMeta(STableMetaInfo* pTableMetaInfo, uint64_t id); - -char* cloneCurrentDBName(SSqlObj* pSql); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TSCUTIL_H diff --git a/2.0/src/client/inc/tsclient.h b/2.0/src/client/inc/tsclient.h deleted file mode 100644 index d5021f663d8b56be873ce99b36370a9f4b786dca..0000000000000000000000000000000000000000 --- a/2.0/src/client/inc/tsclient.h +++ /dev/null @@ -1,837 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSCLIENT_H -#define TDENGINE_TSCLIENT_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" - -#include "../../../include/client/taos.h" -#include "qAggMain.h" -#include "taosdef.h" -#include "tmsg.h" -#include "tarray.h" -#include "tcache.h" -#include "tglobal.h" -#include "tref.h" -#include "tutil.h" - -#include "qExecutor.h" -#include "qSqlparser.h" -#include "qTsbuf.h" -#include "qUtil.h" -#include "tmsgtype.h" - -typedef enum { - TAOS_REQ_FROM_SHELL, - TAOS_REQ_FROM_HTTP -} SReqOrigin; - -// forward declaration -struct SSqlInfo; - -typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows); - -typedef struct SNewVgroupInfo { - int32_t vgId; - int8_t inUse; - int8_t numOfEps; - SEpAddrMsg ep[TSDB_MAX_REPLICA]; -} SNewVgroupInfo; - -typedef struct CChildTableMeta { - int32_t vgId; - STableId id; - uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; // TODO: refactor super table name, not full name - uint64_t suid; // super table id -} CChildTableMeta; - -typedef struct SColumnIndex { - int16_t tableIndex; - int16_t columnIndex; -} SColumnIndex; - -typedef struct SColumn { - uint64_t tableUid; - int32_t columnIndex; - SColumnInfo info; -} SColumn; - -typedef struct SInternalField { - TAOS_FIELD field; - bool visible; - SExprInfo *pExpr; -} SInternalField; - -typedef struct SParamInfo { - int32_t idx; - uint8_t type; - uint8_t timePrec; - int16_t bytes; - uint32_t offset; -} SParamInfo; - -typedef struct SBoundColumn { - int32_t offset; // all column offset value - int32_t toffset; // first part offset for SDataRow TODO: get offset from STSchema on future - uint8_t valStat; // denote if current column bound or not(0 means has val, 1 means no val) -} SBoundColumn; -typedef enum { - VAL_STAT_HAS = 0x0, // 0 means has val - VAL_STAT_NONE = 0x01, // 1 means no val -} EValStat; - -typedef struct { - uint16_t schemaColIdx; - uint16_t boundIdx; - uint16_t finalIdx; -} SBoundIdxInfo; - -typedef enum _COL_ORDER_STATUS { - ORDER_STATUS_UNKNOWN = 0, - ORDER_STATUS_ORDERED = 1, - ORDER_STATUS_DISORDERED = 2, -} EOrderStatus; -typedef struct SParsedDataColInfo { - int16_t numOfCols; - int16_t numOfBound; - uint16_t flen; // TODO: get from STSchema - uint16_t allNullLen; // TODO: get from STSchema(base on SDataRow) - uint16_t extendedVarLen; - uint16_t boundNullLen; // bound column len with all NULL value(without VarDataOffsetT/SColIdx part) - int32_t *boundColumns; // bound column idx according to schema - SBoundColumn * cols; - SBoundIdxInfo *colIdxInfo; - int8_t orderStatus; // bound columns -} SParsedDataColInfo; - -#define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) - -typedef struct { - uint8_t rowType; // default is 0, that is SDataRow - int32_t rowSize; -} SMemRowBuilder; - -int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec); - -int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, SParsedDataColInfo *pColInfo); -void destroyMemRowBuilder(SMemRowBuilder *pBuilder); - -/** - * @brief - * - * @param rowType - * @param spd - * @param idx the absolute bound index of columns - * @return FORCE_INLINE - */ -static FORCE_INLINE void tscGetSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, int32_t idx, - int32_t *toffset, int16_t *colId) { - int32_t schemaIdx = 0; - if (IS_DATA_COL_ORDERED(spd)) { - schemaIdx = spd->boundColumns[idx]; - if (isDataRowT(rowType)) { - *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart - } else { - *toffset = idx * sizeof(SColIdx); // the offset of SColIdx - } - } else { - ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx); - schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx; - if (isDataRowT(rowType)) { - *toffset = (spd->cols + schemaIdx)->toffset; - } else { - *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx); - } - } - *colId = pSchema[schemaIdx].colId; -} - -typedef struct STableDataBlocks { - SName tableName; - int8_t tsSource; // where does the UNIX timestamp come from, server or client - bool ordered; // if current rows are ordered or not - int64_t vgId; // virtual group id - int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending - int32_t numOfTables; // number of tables in current submit block - int32_t rowSize; // row size for current table - uint32_t nAllocSize; - uint32_t headerSize; // header for table info (uid, tid, submit metadata) - uint32_t size; - STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache - char *pData; - bool cloned; - - SParsedDataColInfo boundColumnInfo; - - // for parameter ('?') binding - uint32_t numOfAllocedParams; - uint32_t numOfParams; - SParamInfo * params; - SMemRowBuilder rowBuilder; -} STableDataBlocks; - -typedef struct { - STableMeta *pTableMeta; - SArray *vgroupIdList; -} STableMetaVgroupInfo; - -typedef struct SInsertStatementParam { - SName **pTableNameList; // all involved tableMeta list of current insert sql statement. - int32_t numOfTables; // number of tables in table name list - SHashObj *pTableBlockHashList; // data block for each table - SArray *pDataBlocks; // SArray. Merged submit block for each vgroup - int8_t schemaAttached; // denote if submit block is built with table schema or not - uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert - STagData tagData; // NOTE: pTagData->data is used as a variant length array - - int32_t batchSize; // for parameter ('?') binding and batch processing - int32_t numOfParams; - - char msg[512]; // error message - uint32_t insertType; // insert data from [file|sql statement| bound statement] - uint64_t objectId; // sql object id - char *sql; // current sql statement position -} SInsertStatementParam; - -typedef enum { - PAYLOAD_TYPE_KV = 0, - PAYLOAD_TYPE_RAW = 1, -} EPayloadType; - -#define IS_RAW_PAYLOAD(t) \ - (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert - -// TODO extract sql parser supporter -typedef struct { - int command; - uint8_t msgType; - SInsertStatementParam insertParam; - char reserve1[3]; // fix bus error on arm32 - int32_t count; // todo remove it - bool subCmd; - - char reserve2[3]; // fix bus error on arm32 - int16_t numOfCols; - char reserve3[2]; // fix bus error on arm32 - uint32_t allocSize; - char * payload; - int32_t payloadLen; - - SHashObj *pTableMetaMap; // local buffer to keep the queried table meta, before validating the AST - SQueryInfo *pQueryInfo; - SQueryInfo *active; // current active query info - int32_t batchSize; // for parameter ('?') binding and batch processing - int32_t resColumnId; -} SSqlCmd; - -typedef struct { - int32_t numOfRows; // num of results in current retrieval - int64_t numOfTotal; // num of total results - int64_t numOfClauseTotal; // num of total result in current subclause - char * pRsp; - int32_t rspType; - int32_t rspLen; - uint64_t qId; // query id of SQInfo - int64_t useconds; - int64_t offset; // offset value from vnode during projection query of stable - int32_t row; - int16_t numOfCols; - int16_t precision; - bool completed; - int32_t code; - char * data; - TAOS_ROW tsrow; - TAOS_ROW urow; - int32_t* length; // length for each field for current row - char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) - SColumnIndex* pColumnIndex; - - TAOS_FIELD* final; - struct SGlobalMerger *pMerger; -} SSqlRes; - -typedef struct { - char key[512]; - void *pDnodeConn; -} SRpcObj; - -typedef struct STscObj { - void * signature; - void * pTimer; - char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; - char acctId[TSDB_ACCT_ID_LEN]; - char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; - char sversion[TSDB_VERSION_LEN]; - char writeAuth : 1; - char superAuth : 1; - uint32_t connId; - uint64_t rid; // ref ID returned by taosAddRef - int64_t hbrid; - struct SSqlObj * sqlList; - struct SSqlStream *streamList; - SRpcObj *pRpcObj; - SRpcCorEpSet *tscCorMgmtEpSet; - pthread_mutex_t mutex; - int32_t numOfObj; // number of sqlObj from this tscObj - SReqOrigin from; -} STscObj; - -typedef struct SSubqueryState { - pthread_mutex_t mutex; - int8_t *states; - int32_t numOfSub; // the number of total sub-queries - uint64_t numOfRetrievedRows; // total number of points in this query -} SSubqueryState; - -typedef struct SSqlObj { - void *signature; - int64_t owner; // owner of sql object, by which it is executed - STscObj *pTscObj; - int64_t rpcRid; - __async_cb_func_t fp; - __async_cb_func_t fetchFp; - void *param; - int64_t stime; - uint32_t queryId; - void * pStream; - void * pSubscription; - char * sqlstr; - void * pBuf; // table meta buffer - char parseRetry; - char retry; - char maxRetry; - SRpcEpSet epSet; - char listed; - tsem_t rspSem; - SSqlCmd cmd; - SSqlRes res; - - SSubqueryState subState; - struct SSqlObj **pSubs; - - int64_t metaRid; - int64_t svgroupRid; - - int64_t squeryLock; - int32_t retryReason; // previous error code - struct SSqlObj *prev, *next; - int64_t self; -} SSqlObj; - -typedef struct SSqlStream { - SSqlObj *pSql; - void * cqhandle; // stream belong to SCQContext handle - const char* dstTable; - uint32_t streamId; - char listed; - bool isProject; - int16_t precision; - int64_t num; // number of computing count - - /* - * keep the number of current result in computing, - * the value will be set to 0 before set timer for next computing - */ - int64_t numOfRes; - - int64_t useconds; // total elapsed time - int64_t ctime; // stream created time - int64_t stime; // stream next executed time - int64_t etime; // stream end query time, when time is larger then etime, the stream will be closed - int64_t ltime; // stream last row time in stream table - SInterval interval; - void * pTimer; - - void (*fp)(); - void *param; - - void (*callback)(void *); // Callback function when stream is stopped from client level - struct SSqlStream *prev, *next; -} SSqlStream; - -void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable); - -int tscAcquireRpc(const char *key, const char *user, const char *secret,void **pRpcObj); -void tscReleaseRpc(void *param); -void tscInitMsgsFp(); - -int tsParseSql(SSqlObj *pSql, bool initial); - -void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet); -int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo); - -int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex); -void tscAsyncResultOnError(SSqlObj *pSql); - -void tscQueueAsyncError(void(*fp), void *param, int32_t code); - -int tscProcessLocalCmd(SSqlObj *pSql); -int tscCfgDynamicOptions(char *msg); - -int32_t tscTansformFuncForSTableQuery(SQueryInfo *pQueryInfo); -void tscRestoreFuncForSTableQuery(SQueryInfo *pQueryInfo); - -int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); -void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo); -void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock, bool convertNchar); - -void handleDownstreamOperator(SSqlObj** pSqlList, int32_t numOfUpstream, SQueryInfo* px, SSqlObj* pParent); -void destroyTableNameList(SInsertStatementParam* pInsertParam); - -void tscResetSqlCmd(SSqlCmd *pCmd, bool removeMeta, uint64_t id); - -/** - * free query result of the sql object - * @param pObj - */ -void tscFreeSqlResult(SSqlObj *pSql); - -void* tscCleanupTableMetaMap(SHashObj* pTableMetaMap); - -/** - * free sql object, release allocated resource - * @param pObj - */ -void tscFreeSqlObj(SSqlObj *pSql); -void tscFreeSubobj(SSqlObj* pSql); - -void tscFreeRegisteredSqlObj(void *pSql); - -void tscCloseTscObj(void *pObj); - -// todo move to taos? or create a new file: taos_internal.h -TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), - void *param, TAOS **taos); -TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res); -TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param); - -void waitForQueryRsp(void *param, TAOS_RES *tres, int code); - -void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen); - -void tscImportDataFromFile(SSqlObj *pSql); -struct SGlobalMerger* tscInitResObjForLocalQuery(int32_t numOfRes, int32_t rowLen, uint64_t id); -bool tscIsUpdateQuery(SSqlObj* pSql); -char* tscGetSqlStr(SSqlObj* pSql); -bool tscIsQueryWithLimit(SSqlObj* pSql); - -bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); -void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols); - -char *tscGetErrorMsgPayload(SSqlCmd *pCmd); -int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql); - -int32_t tscInvalidOperationMsg(char *msg, const char *additionalInfo, const char *sql); -int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql); - -int32_t tscValidateSqlInfo(SSqlObj *pSql, struct SSqlInfo *pInfo); - -int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows); -extern int32_t sentinel; -extern SHashObj *tscVgroupMap; -extern SHashObj *tscTableMetaMap; -extern SCacheObj *tscVgroupListBuf; - -extern int tscObjRef; -extern void *tscTmr; -extern void *tscQhandle; -extern int tscKeepConn[]; -extern int tscRefId; -extern int tscNumOfObj; // number of existed sqlObj in current process. - -extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); - -void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables); -int16_t getNewResColId(SSqlCmd* pCmd); - -int32_t schemaIdxCompar(const void *lhs, const void *rhs); -int32_t boundIdxCompar(const void *lhs, const void *rhs); -static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) { - ASSERT(pBlock->rowSize == pBlock->pTableMeta->tableInfo.rowSize); - return pBlock->rowSize + TD_MEM_ROW_DATA_HEAD_SIZE + pBlock->boundColumnInfo.extendedVarLen; -} - -static FORCE_INLINE void initSMemRow(SMemRow row, uint8_t memRowType, STableDataBlocks *pBlock, int16_t nBoundCols) { - memRowSetType(row, memRowType); - if (isDataRowT(memRowType)) { - dataRowSetVersion(memRowDataBody(row), pBlock->pTableMeta->sversion); - dataRowSetLen(memRowDataBody(row), (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBlock->boundColumnInfo.flen)); - } else { - ASSERT(nBoundCols > 0); - memRowSetKvVersion(row, pBlock->pTableMeta->sversion); - kvRowSetNCols(memRowKvBody(row), nBoundCols); - kvRowSetLen(memRowKvBody(row), (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols)); - } -} -/** - * TODO: Move to tdataformat.h and refactor when STSchema available. - * - fetch flen and toffset from STSChema and remove param spd - */ -static FORCE_INLINE void convertToSDataRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, - SParsedDataColInfo *spd) { - ASSERT(isKvRow(src)); - SKVRow kvRow = memRowKvBody(src); - SDataRow dataRow = memRowDataBody(dest); - - memRowSetType(dest, SMEM_ROW_DATA); - dataRowSetVersion(dataRow, memRowKvVersion(src)); - dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + spd->flen)); - - int32_t kvIdx = 0; - for (int i = 0; i < nCols; ++i) { - SSchema *schema = pSchema + i; - void * val = tdGetKVRowValOfColEx(kvRow, schema->colId, &kvIdx); - tdAppendDataColVal(dataRow, val != NULL ? val : getNullValue(schema->type), true, schema->type, - (spd->cols + i)->toffset); - } -} - -// TODO: Move to tdataformat.h and refactor when STSchema available. -static FORCE_INLINE void convertToSKVRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, int nBoundCols, - SParsedDataColInfo *spd) { - ASSERT(isDataRow(src)); - - SDataRow dataRow = memRowDataBody(src); - SKVRow kvRow = memRowKvBody(dest); - - memRowSetType(dest, SMEM_ROW_KV); - memRowSetKvVersion(kvRow, dataRowVersion(dataRow)); - kvRowSetNCols(kvRow, nBoundCols); - kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols)); - - int32_t toffset = 0, kvOffset = 0; - for (int i = 0; i < nCols; ++i) { - if ((spd->cols + i)->valStat == VAL_STAT_HAS) { - SSchema *schema = pSchema + i; - toffset = (spd->cols + i)->toffset; - void *val = tdGetRowDataOfCol(dataRow, schema->type, toffset + TD_DATA_ROW_HEAD_SIZE); - tdAppendKvColVal(kvRow, val, true, schema->colId, schema->type, kvOffset); - kvOffset += sizeof(SColIdx); - } - } -} - -// TODO: Move to tdataformat.h and refactor when STSchema available. -static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlocks *pBlock) { - STableMeta * pTableMeta = pBlock->pTableMeta; - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - SSchema * pSchema = tscGetTableSchema(pTableMeta); - SParsedDataColInfo *spd = &pBlock->boundColumnInfo; - - ASSERT(dest != src); - - if (isDataRow(src)) { - // TODO: Can we use pBlock -> numOfParam directly? - ASSERT(spd->numOfBound > 0); - convertToSKVRow(dest, src, pSchema, tinfo.numOfColumns, spd->numOfBound, spd); - } else { - convertToSDataRow(dest, src, pSchema, tinfo.numOfColumns, spd); - } -} - -static bool isNullStr(SStrToken *pToken) { - return (pToken->type == TK_NULL) || ((pToken->type == TK_STRING) && (pToken->n != 0) && - (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)); -} - -static FORCE_INLINE int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) { - errno = 0; - *value = strtold(pToken->z, endPtr); - - // not a valid integer number, return error - if ((*endPtr - pToken->z) != pToken->n) { - return TK_ILLEGAL; - } - - return pToken->type; -} - -static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; -static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; - -static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, SMemRow row, char *msg, char **str, - bool primaryKey, int16_t timePrec, int32_t toffset, int16_t colId) { - int64_t iv; - int32_t ret; - char * endptr = NULL; - - if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { - return tscInvalidOperationMsg(msg, "invalid numeric data", pToken->z); - } - - switch (pSchema->type) { - case TSDB_DATA_TYPE_BOOL: { // bool - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { - if (strncmp(pToken->z, "true", pToken->n) == 0) { - tdAppendMemRowColVal(row, &TRUE_VALUE, true, colId, pSchema->type, toffset); - } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - tdAppendMemRowColVal(row, &FALSE_VALUE, true, colId, pSchema->type, toffset); - } else { - return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z); - } - } else if (pToken->type == TK_INTEGER) { - iv = strtoll(pToken->z, NULL, 10); - tdAppendMemRowColVal(row, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset); - } else if (pToken->type == TK_FLOAT) { - double dv = strtod(pToken->z, NULL); - tdAppendMemRowColVal(row, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset); - } else { - return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z); - } - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid tinyint data", pToken->z); - } else if (!IS_VALID_TINYINT(iv)) { - return tscInvalidOperationMsg(msg, "data overflow", pToken->z); - } - - uint8_t tmpVal = (uint8_t)iv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - - break; - - case TSDB_DATA_TYPE_UTINYINT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned tinyint data", pToken->z); - } else if (!IS_VALID_UTINYINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned tinyint data overflow", pToken->z); - } - - uint8_t tmpVal = (uint8_t)iv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - - break; - - case TSDB_DATA_TYPE_SMALLINT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid smallint data", pToken->z); - } else if (!IS_VALID_SMALLINT(iv)) { - return tscInvalidOperationMsg(msg, "smallint data overflow", pToken->z); - } - - int16_t tmpVal = (int16_t)iv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - - break; - - case TSDB_DATA_TYPE_USMALLINT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned smallint data", pToken->z); - } else if (!IS_VALID_USMALLINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned smallint data overflow", pToken->z); - } - - uint16_t tmpVal = (uint16_t)iv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - - break; - - case TSDB_DATA_TYPE_INT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid int data", pToken->z); - } else if (!IS_VALID_INT(iv)) { - return tscInvalidOperationMsg(msg, "int data overflow", pToken->z); - } - - int32_t tmpVal = (int32_t)iv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - - break; - - case TSDB_DATA_TYPE_UINT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned int data", pToken->z); - } else if (!IS_VALID_UINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned int data overflow", pToken->z); - } - - uint32_t tmpVal = (uint32_t)iv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - - break; - - case TSDB_DATA_TYPE_BIGINT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z); - } - - tdAppendMemRowColVal(row, &iv, true, colId, pSchema->type, toffset); - } - break; - - case TSDB_DATA_TYPE_UBIGINT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned bigint data", pToken->z); - } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { - return tscInvalidOperationMsg(msg, "unsigned bigint data overflow", pToken->z); - } - - uint64_t tmpVal = (uint64_t)iv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - break; - - case TSDB_DATA_TYPE_FLOAT: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - double dv; - if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { - return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); - } - - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || - isnan(dv)) { - return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); - } - - float tmpVal = (float)dv; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - break; - - case TSDB_DATA_TYPE_DOUBLE: - if (isNullStr(pToken)) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - double dv; - if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { - return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); - } - - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { - return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); - } - - tdAppendMemRowColVal(row, &dv, true, colId, pSchema->type, toffset); - } - break; - - case TSDB_DATA_TYPE_BINARY: - // binary data cannot be null-terminated char string, otherwise the last char of the string is lost - if (pToken->type == TK_NULL) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { // too long values will return invalid sql, not be truncated automatically - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor - return tscInvalidOperationMsg(msg, "string data overflow", pToken->z); - } - // STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); - char *rowEnd = memRowEnd(row); - STR_WITH_SIZE_TO_VARSTR(rowEnd, pToken->z, pToken->n); - tdAppendMemRowColVal(row, rowEnd, false, colId, pSchema->type, toffset); - } - break; - - case TSDB_DATA_TYPE_NCHAR: - if (pToken->type == TK_NULL) { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } else { - // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - int32_t output = 0; - char * rowEnd = memRowEnd(row); - if (!taosMbsToUcs4(pToken->z, pToken->n, (char *)varDataVal(rowEnd), pSchema->bytes - VARSTR_HEADER_SIZE, - &output)) { - char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); - return tscInvalidOperationMsg(msg, buf, pToken->z); - } - varDataSetLen(rowEnd, output); - tdAppendMemRowColVal(row, rowEnd, false, colId, pSchema->type, toffset); - } - break; - - case TSDB_DATA_TYPE_TIMESTAMP: { - if (pToken->type == TK_NULL) { - if (primaryKey) { - // When building SKVRow primaryKey, we should not skip even with NULL value. - int64_t tmpVal = 0; - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } else { - tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset); - } - } else { - int64_t tmpVal; - if (tsParseTime(pToken, &tmpVal, str, msg, timePrec) != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z); - } - tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset); - } - - break; - } - } - - return TSDB_CODE_SUCCESS; -} - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/2.0/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/2.0/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h deleted file mode 100644 index 7181c658ddcdfde3efe7df3c0784c20f18bd4c03..0000000000000000000000000000000000000000 --- a/2.0/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ /dev/null @@ -1,237 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -/* Header for class com_taosdata_jdbc_TSDBJNIConnector */ - -#ifndef _Included_com_taosdata_jdbc_TSDBJNIConnector -#define _Included_com_taosdata_jdbc_TSDBJNIConnector -#ifdef __cplusplus -extern "C" { -#endif -#undef com_taosdata_jdbc_TSDBJNIConnector_INVALID_CONNECTION_POINTER_VALUE -#define com_taosdata_jdbc_TSDBJNIConnector_INVALID_CONNECTION_POINTER_VALUE 0LL -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: - * Signature: (Ljava/lang/String;)V - */ -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setAllocModeImp - (JNIEnv *, jclass, jint, jstring, jboolean); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: - * Signature: ()Ljava/lang/String; - */ -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_dumpMemoryLeakImp - (JNIEnv *, jclass); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: initImp - * Signature: (Ljava/lang/String;)V - */ -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_initImp - (JNIEnv *, jclass, jstring); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: setOptions - * Signature: (ILjava/lang/String;)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setOptions - (JNIEnv *, jclass, jint, jstring); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: getTsCharset - * Signature: ()Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getTsCharset - (JNIEnv *, jclass); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: getResultTimePrecisionImp - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TDDBJNIConnector_getResultTimePrecisionImp - (JNIEnv *, jobject, jlong, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: connectImp - * Signature: (Ljava/lang/String;ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;)J - */ -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp - (JNIEnv *, jobject, jstring, jint, jstring, jstring, jstring); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: executeQueryImp - * Signature: ([BJ)I - */ -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp - (JNIEnv *, jobject, jbyteArray, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: getErrCodeImp - * Signature: (J)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrCodeImp - (JNIEnv *, jobject, jlong, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: getErrMsgImp - * Signature: (J)Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp - (JNIEnv *, jobject, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: getResultSetImp - * Signature: (J)J - */ -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp - (JNIEnv *env, jobject jobj, jlong con, jlong tres); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: isUpdateQueryImp - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp - (JNIEnv *env, jobject jobj, jlong con, jlong tres); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: freeResultSetImp - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp - (JNIEnv *, jobject, jlong, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: getAffectedRowsImp - * Signature: (J)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsImp - (JNIEnv *env, jobject jobj, jlong con, jlong res); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: getSchemaMetaDataImp - * Signature: (JJLjava/util/List;)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getSchemaMetaDataImp - (JNIEnv *, jobject, jlong, jlong, jobject); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: fetchRowImp - * Signature: (JJLcom/taosdata/jdbc/TSDBResultSetRowData;)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp - (JNIEnv *, jobject, jlong, jlong, jobject); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: fetchBlockImp - * Signature: (JJLcom/taosdata/jdbc/TSDBResultSetBlockData;)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp - (JNIEnv *, jobject, jlong, jlong, jobject); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: closeConnectionImp - * Signature: (J)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeConnectionImp - (JNIEnv *, jobject, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: subscribeImp - * Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;JI)J - */ -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp - (JNIEnv *, jobject, jlong, jboolean, jstring, jstring, jint); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: consumeImp - * Signature: (J)Lcom/taosdata/jdbc/TSDBResultSetRowData; - */ -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp - (JNIEnv *, jobject, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: unsubscribeImp - * Signature: (J)V - */ -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp - (JNIEnv *, jobject, jlong, jboolean); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: validateCreateTableSqlImp - * Signature: (J[B)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_validateCreateTableSqlImp - (JNIEnv *, jobject, jlong, jbyteArray); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: prepareStmtImp - * Signature: ([BJ)I - */ -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtImp - (JNIEnv *, jobject, jbyteArray, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: setBindTableNameImp - * Signature: (JLjava/lang/String;J)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameImp - (JNIEnv *, jobject, jlong, jstring, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: bindColDataImp - * Signature: (J[B[B[BIIIIJ)J - */ -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp -(JNIEnv *, jobject, jlong, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jlong); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: executeBatchImp - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeBatchImp(JNIEnv *env, jobject jobj, jlong stmt, jlong con); - -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: closeStmt - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeStmt(JNIEnv *env, jobject jobj, jlong stmt, jlong con); - -/** - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: setTableNameTagsImp - * Signature: (JLjava/lang/String;I[B[B[B[BJ)I - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsImp - (JNIEnv *, jobject, jlong, jstring, jint, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jlong); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/2.0/src/client/src/TSDBJNIConnector.c b/2.0/src/client/src/TSDBJNIConnector.c deleted file mode 100644 index 0d22b44f5af7ccf1aa1ba1cc624385d480f5bd2a..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/TSDBJNIConnector.c +++ /dev/null @@ -1,1029 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "../../../include/client/taos.h" -#include "os.h" -#include "tlog.h" -#include "tscUtil.h" - -#include "com_taosdata_jdbc_TSDBJNIConnector.h" - -#define jniFatal(...) \ - { \ - if (jniDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("JNI FATAL ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniError(...) \ - { \ - if (jniDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("JNI ERROR ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniWarn(...) \ - { \ - if (jniDebugFlag & DEBUG_WARN) { \ - taosPrintLog("JNI WARN ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniInfo(...) \ - { \ - if (jniDebugFlag & DEBUG_INFO) { \ - taosPrintLog("JNI ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniDebug(...) \ - { \ - if (jniDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniTrace(...) \ - { \ - if (jniDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); \ - } \ - } - -int __init = 0; - -JavaVM *g_vm = NULL; - -jclass g_arrayListClass; -jmethodID g_arrayListConstructFp; -jmethodID g_arrayListAddFp; - -jclass g_metadataClass; -jmethodID g_metadataConstructFp; -jfieldID g_metadataColtypeField; -jfieldID g_metadataColnameField; -jfieldID g_metadataColsizeField; -jfieldID g_metadataColindexField; - -jclass g_rowdataClass; -jmethodID g_rowdataConstructor; -jmethodID g_rowdataClearFp; -jmethodID g_rowdataSetBooleanFp; -jmethodID g_rowdataSetByteFp; -jmethodID g_rowdataSetShortFp; -jmethodID g_rowdataSetIntFp; -jmethodID g_rowdataSetLongFp; -jmethodID g_rowdataSetFloatFp; -jmethodID g_rowdataSetDoubleFp; -jmethodID g_rowdataSetStringFp; -jmethodID g_rowdataSetTimestampFp; -jmethodID g_rowdataSetByteArrayFp; - -jmethodID g_blockdataSetByteArrayFp; -jmethodID g_blockdataSetNumOfRowsFp; -jmethodID g_blockdataSetNumOfColsFp; - -#define JNI_SUCCESS 0 -#define JNI_TDENGINE_ERROR -1 -#define JNI_CONNECTION_NULL -2 -#define JNI_RESULT_SET_NULL -3 -#define JNI_NUM_OF_FIELDS_0 -4 -#define JNI_SQL_NULL -5 -#define JNI_FETCH_END -6 -#define JNI_OUT_OF_MEMORY -7 - -static void jniGetGlobalMethod(JNIEnv *env) { - // make sure init function executed once - switch (atomic_val_compare_exchange_32(&__init, 0, 1)) { - case 0: - break; - case 1: - do { - taosMsleep(0); - } while (atomic_load_32(&__init) == 1); - case 2: - return; - } - - if (g_vm == NULL) { - (*env)->GetJavaVM(env, &g_vm); - } - - jclass arrayListClass = (*env)->FindClass(env, "java/util/ArrayList"); - g_arrayListClass = (*env)->NewGlobalRef(env, arrayListClass); - g_arrayListConstructFp = (*env)->GetMethodID(env, g_arrayListClass, "", "()V"); - g_arrayListAddFp = (*env)->GetMethodID(env, g_arrayListClass, "add", "(Ljava/lang/Object;)Z"); - (*env)->DeleteLocalRef(env, arrayListClass); - - jclass metadataClass = (*env)->FindClass(env, "com/taosdata/jdbc/ColumnMetaData"); - g_metadataClass = (*env)->NewGlobalRef(env, metadataClass); - g_metadataConstructFp = (*env)->GetMethodID(env, g_metadataClass, "", "()V"); - g_metadataColtypeField = (*env)->GetFieldID(env, g_metadataClass, "colType", "I"); - g_metadataColnameField = (*env)->GetFieldID(env, g_metadataClass, "colName", "Ljava/lang/String;"); - g_metadataColsizeField = (*env)->GetFieldID(env, g_metadataClass, "colSize", "I"); - g_metadataColindexField = (*env)->GetFieldID(env, g_metadataClass, "colIndex", "I"); - (*env)->DeleteLocalRef(env, metadataClass); - - jclass rowdataClass = (*env)->FindClass(env, "com/taosdata/jdbc/TSDBResultSetRowData"); - g_rowdataClass = (*env)->NewGlobalRef(env, rowdataClass); - g_rowdataConstructor = (*env)->GetMethodID(env, g_rowdataClass, "", "(I)V"); - g_rowdataClearFp = (*env)->GetMethodID(env, g_rowdataClass, "clear", "()V"); - g_rowdataSetBooleanFp = (*env)->GetMethodID(env, g_rowdataClass, "setBoolean", "(IZ)V"); - g_rowdataSetByteFp = (*env)->GetMethodID(env, g_rowdataClass, "setByte", "(IB)V"); - g_rowdataSetShortFp = (*env)->GetMethodID(env, g_rowdataClass, "setShort", "(IS)V"); - g_rowdataSetIntFp = (*env)->GetMethodID(env, g_rowdataClass, "setInt", "(II)V"); - g_rowdataSetLongFp = (*env)->GetMethodID(env, g_rowdataClass, "setLong", "(IJ)V"); - g_rowdataSetFloatFp = (*env)->GetMethodID(env, g_rowdataClass, "setFloat", "(IF)V"); - g_rowdataSetDoubleFp = (*env)->GetMethodID(env, g_rowdataClass, "setDouble", "(ID)V"); - g_rowdataSetStringFp = (*env)->GetMethodID(env, g_rowdataClass, "setString", "(ILjava/lang/String;)V"); - g_rowdataSetTimestampFp = (*env)->GetMethodID(env, g_rowdataClass, "setTimestamp", "(IJI)V"); - g_rowdataSetByteArrayFp = (*env)->GetMethodID(env, g_rowdataClass, "setByteArray", "(I[B)V"); - (*env)->DeleteLocalRef(env, rowdataClass); - - jclass blockdataClass = (*env)->FindClass(env, "com/taosdata/jdbc/TSDBResultSetBlockData"); - jclass g_blockdataClass = (*env)->NewGlobalRef(env, blockdataClass); - g_blockdataSetByteArrayFp = (*env)->GetMethodID(env, g_blockdataClass, "setByteArray", "(II[B)V"); - g_blockdataSetNumOfRowsFp = (*env)->GetMethodID(env, g_blockdataClass, "setNumOfRows", "(I)V"); - g_blockdataSetNumOfColsFp = (*env)->GetMethodID(env, g_blockdataClass, "setNumOfCols", "(I)V"); - (*env)->DeleteLocalRef(env, blockdataClass); - - atomic_store_32(&__init, 2); - jniDebug("native method register finished"); -} - -static int32_t check_for_params(jobject jobj, jlong conn, jlong res) { - if ((TAOS *)conn == NULL) { - jniError("jobj:%p, connection is closed", jobj); - return JNI_CONNECTION_NULL; - } - - if ((TAOS_RES *)res == NULL) { - jniError("jobj:%p, conn:%p, res is null", jobj, (TAOS *)conn); - return JNI_RESULT_SET_NULL; - } - - return JNI_SUCCESS; -} - -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setAllocModeImp(JNIEnv *env, jobject jobj, jint jMode, - jstring jPath, jboolean jAutoDump) { - if (jPath != NULL) { - const char *path = (*env)->GetStringUTFChars(env, jPath, NULL); - taosSetAllocMode(jMode, path, !!jAutoDump); - (*env)->ReleaseStringUTFChars(env, jPath, path); - } else { - taosSetAllocMode(jMode, NULL, !!jAutoDump); - } -} - -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_dumpMemoryLeakImp(JNIEnv *env, jobject jobj) { - taosDumpMemoryLeak(); -} - -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_initImp(JNIEnv *env, jobject jobj, jstring jconfigDir) { - if (jconfigDir != NULL) { - const char *confDir = (*env)->GetStringUTFChars(env, jconfigDir, NULL); - if (confDir && strlen(confDir) != 0) { - tstrncpy(configDir, confDir, TSDB_FILENAME_LEN); - } - (*env)->ReleaseStringUTFChars(env, jconfigDir, confDir); - } - - jniGetGlobalMethod(env); - jniDebug("jni initialized successfully, config directory: %s", configDir); -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setOptions(JNIEnv *env, jobject jobj, jint optionIndex, - jstring optionValue) { - if (optionValue == NULL) { - jniDebug("option index:%d value is null", (int32_t)optionIndex); - return 0; - } - - int res = 0; - - if (optionIndex == TSDB_OPTION_LOCALE) { - const char *locale = (*env)->GetStringUTFChars(env, optionValue, NULL); - if (locale && strlen(locale) != 0) { - res = taos_options(TSDB_OPTION_LOCALE, locale); - jniDebug("set locale to %s, result:%d", locale, res); - } else { - jniDebug("input locale is empty"); - } - (*env)->ReleaseStringUTFChars(env, optionValue, locale); - } else if (optionIndex == TSDB_OPTION_CHARSET) { - const char *charset = (*env)->GetStringUTFChars(env, optionValue, NULL); - if (charset && strlen(charset) != 0) { - res = taos_options(TSDB_OPTION_CHARSET, charset); - jniDebug("set character encoding to %s, result:%d", charset, res); - } else { - jniDebug("input character encoding is empty"); - } - (*env)->ReleaseStringUTFChars(env, optionValue, charset); - } else if (optionIndex == TSDB_OPTION_TIMEZONE) { - const char *tz1 = (*env)->GetStringUTFChars(env, optionValue, NULL); - if (tz1 && strlen(tz1) != 0) { - res = taos_options(TSDB_OPTION_TIMEZONE, tz1); - jniDebug("set timezone to %s, result:%d", tz1, res); - } else { - jniDebug("input timezone is empty"); - } - (*env)->ReleaseStringUTFChars(env, optionValue, tz1); - } else { - jniError("option index:%d is not found", (int32_t)optionIndex); - } - - return res; -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp(JNIEnv *env, jobject jobj, jstring jhost, - jint jport, jstring jdbName, jstring juser, - jstring jpass) { - jlong ret = 0; - const char *host = NULL; - const char *user = NULL; - const char *pass = NULL; - const char *dbname = NULL; - - if (jhost != NULL) { - host = (*env)->GetStringUTFChars(env, jhost, NULL); - } - - if (jdbName != NULL) { - dbname = (*env)->GetStringUTFChars(env, jdbName, NULL); - } - - if (juser != NULL) { - user = (*env)->GetStringUTFChars(env, juser, NULL); - } - - if (jpass != NULL) { - pass = (*env)->GetStringUTFChars(env, jpass, NULL); - } - - if (user == NULL) { - jniDebug("jobj:%p, user not specified, use default user %s", jobj, TSDB_DEFAULT_USER); - } - - if (pass == NULL) { - jniDebug("jobj:%p, pass not specified, use default password", jobj); - } - - ret = (jlong)taos_connect((char *)host, (char *)user, (char *)pass, (char *)dbname, (uint16_t)jport); - if (ret == 0) { - jniError("jobj:%p, conn:%p, connect to database failed, host=%s, user=%s, dbname=%s, port=%d", jobj, (void *)ret, - (char *)host, (char *)user, (char *)dbname, (int32_t)jport); - } else { - jniDebug("jobj:%p, conn:%p, connect to database succeed, host=%s, user=%s, dbname=%s, port=%d", jobj, (void *)ret, - (char *)host, (char *)user, (char *)dbname, (int32_t)jport); - } - - if (host != NULL) { - (*env)->ReleaseStringUTFChars(env, jhost, host); - } - - if (dbname != NULL) { - (*env)->ReleaseStringUTFChars(env, jdbName, dbname); - } - - if (user != NULL) { - (*env)->ReleaseStringUTFChars(env, juser, user); - } - - if (pass != NULL) { - (*env)->ReleaseStringUTFChars(env, jpass, pass); - } - - return ret; -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(JNIEnv *env, jobject jobj, - jbyteArray jsql, jlong con) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - if (jsql == NULL) { - jniError("jobj:%p, conn:%p, empty sql string", jobj, tscon); - return JNI_SQL_NULL; - } - - jsize len = (*env)->GetArrayLength(env, jsql); - - char *str = (char *)calloc(1, sizeof(char) * (len + 1)); - if (str == NULL) { - jniError("jobj:%p, conn:%p, alloc memory failed", jobj, tscon); - return JNI_OUT_OF_MEMORY; - } - - (*env)->GetByteArrayRegion(env, jsql, 0, len, (jbyte *)str); - if ((*env)->ExceptionCheck(env)) { - // todo handle error - } - - SSqlObj *pSql = taos_query(tscon, str); - int32_t code = taos_errno(pSql); - - if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s, msg:%s", jobj, tscon, tstrerror(code), taos_errstr(pSql)); - } else { - if (pSql->cmd.command == TSDB_SQL_INSERT) { - int32_t affectRows = taos_affected_rows(pSql); - jniDebug("jobj:%p, conn:%p, code:%s, affect rows:%d", jobj, tscon, tstrerror(code), affectRows); - } else { - jniDebug("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - } - } - - free(str); - return (jlong)pSql; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrCodeImp(JNIEnv *env, jobject jobj, jlong con, - jlong tres) { - int32_t code = check_for_params(jobj, con, tres); - if (code != JNI_SUCCESS) { - return code; - } - - return (jint)taos_errno((TAOS_RES *)tres); -} - -JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp(JNIEnv *env, jobject jobj, jlong tres) { - TAOS_RES *pSql = (TAOS_RES *)tres; - return (*env)->NewStringUTF(env, (const char *)taos_errstr(pSql)); -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp(JNIEnv *env, jobject jobj, jlong con, - jlong tres) { - TAOS * tscon = (TAOS *)con; - int32_t code = check_for_params(jobj, con, tres); - if (code != JNI_SUCCESS) { - return code; - } - - SSqlObj *pSql = (TAOS_RES *)tres; - if (tscIsUpdateQuery(pSql)) { - jniDebug("jobj:%p, conn:%p, update query, no resultset, %p", jobj, tscon, (void *)tres); - } else { - jniDebug("jobj:%p, conn:%p, get resultset, %p", jobj, tscon, (void *)tres); - } - - return tres; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp(JNIEnv *env, jobject jobj, jlong con, - jlong tres) { - int32_t code = check_for_params(jobj, con, tres); - if (code != JNI_SUCCESS) { - return code; - } - - SSqlObj *pSql = (TAOS_RES *)tres; - - return (tscIsUpdateQuery(pSql) ? 1 : 0); -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp(JNIEnv *env, jobject jobj, jlong con, - jlong res) { - int32_t code = check_for_params(jobj, con, res); - if (code != JNI_SUCCESS) { - return code; - } - - taos_free_result((void *)res); - jniDebug("jobj:%p, conn:%p, free resultset:%p", jobj, (TAOS *)con, (void *)res); - - return JNI_SUCCESS; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsImp(JNIEnv *env, jobject jobj, jlong con, - jlong res) { - TAOS * tscon = (TAOS *)con; - int32_t code = check_for_params(jobj, con, res); - if (code != JNI_SUCCESS) { - return code; - } - - jint ret = taos_affected_rows((SSqlObj *)res); - jniDebug("jobj:%p, conn:%p, sql:%p, res: %p, affect rows:%d", jobj, tscon, (TAOS *)con, (TAOS_RES *)res, - (int32_t)ret); - - return ret; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getSchemaMetaDataImp(JNIEnv *env, jobject jobj, - jlong con, jlong res, - jobject arrayListObj) { - TAOS * tscon = (TAOS *)con; - int32_t code = check_for_params(jobj, con, res); - if (code != JNI_SUCCESS) { - return code; - } - - TAOS_RES * tres = (TAOS_RES *)res; - TAOS_FIELD *fields = taos_fetch_fields(tres); - - int32_t num_fields = taos_num_fields(tres); - if (num_fields == 0) { - jniError("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, tres, num_fields); - return JNI_NUM_OF_FIELDS_0; - } else { - jniDebug("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, tres, num_fields); - for (int i = 0; i < num_fields; ++i) { - jobject metadataObj = (*env)->NewObject(env, g_metadataClass, g_metadataConstructFp); - (*env)->SetIntField(env, metadataObj, g_metadataColtypeField, fields[i].type); - (*env)->SetIntField(env, metadataObj, g_metadataColsizeField, fields[i].bytes); - (*env)->SetIntField(env, metadataObj, g_metadataColindexField, i); - jstring metadataObjColname = (*env)->NewStringUTF(env, fields[i].name); - (*env)->SetObjectField(env, metadataObj, g_metadataColnameField, metadataObjColname); - (*env)->CallBooleanMethod(env, arrayListObj, g_arrayListAddFp, metadataObj); - } - } - - return JNI_SUCCESS; -} - -/** - * - * @param env vm - * @param nchar true multibytes data - * @param maxBytes the maximum allowable field length - * @return - */ -jstring jniFromNCharToByteArray(JNIEnv *env, char *nchar, int32_t maxBytes) { - jbyteArray bytes = (*env)->NewByteArray(env, maxBytes); - (*env)->SetByteArrayRegion(env, bytes, 0, maxBytes, (jbyte *)nchar); - return bytes; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEnv *env, jobject jobj, jlong con, - jlong res, jobject rowobj) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection is closed", jobj); - return JNI_CONNECTION_NULL; - } - - TAOS_RES *result = (TAOS_RES *)res; - if (result == NULL) { - jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon); - return JNI_RESULT_SET_NULL; - } - - TAOS_FIELD *fields = taos_fetch_fields(result); - - int32_t numOfFields = taos_num_fields(result); - if (numOfFields == 0) { - jniError("jobj:%p, conn:%p, resultset:%p, fields size %d", jobj, tscon, (void *)res, numOfFields); - return JNI_NUM_OF_FIELDS_0; - } - - TAOS_ROW row = taos_fetch_row(result); - if (row == NULL) { - int code = taos_errno(result); - if (code == TSDB_CODE_SUCCESS) { - jniDebug("jobj:%p, conn:%p, resultset:%p, fields size is %d, fetch row to the end", jobj, tscon, (void *)res, - numOfFields); - return JNI_FETCH_END; - } else { - jniDebug("jobj:%p, conn:%p, interrupted query", jobj, tscon); - return JNI_RESULT_SET_NULL; - } - } - - int32_t *length = taos_fetch_lengths(result); - - char tmp[TSDB_MAX_BYTES_PER_ROW] = {0}; - - for (int i = 0; i < numOfFields; i++) { - if (row[i] == NULL) { - continue; - } - - switch (fields[i].type) { - case TSDB_DATA_TYPE_BOOL: - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetBooleanFp, i, (jboolean)(*((char *)row[i]) == 1)); - break; - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetByteFp, i, (jbyte) * ((int8_t *)row[i])); - break; - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetShortFp, i, (jshort) * ((int16_t *)row[i])); - break; - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetIntFp, i, (jint) * (int32_t *)row[i]); - break; - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetLongFp, i, (jlong) * ((int64_t *)row[i])); - break; - case TSDB_DATA_TYPE_FLOAT: { - float fv = 0; - fv = GET_FLOAT_VAL(row[i]); - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetFloatFp, i, (jfloat)fv); - } break; - case TSDB_DATA_TYPE_DOUBLE: { - double dv = 0; - dv = GET_DOUBLE_VAL(row[i]); - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetDoubleFp, i, (jdouble)dv); - } break; - case TSDB_DATA_TYPE_BINARY: { - memcpy(tmp, row[i], length[i]); // handle the case that terminated does not exist - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetStringFp, i, (*env)->NewStringUTF(env, tmp)); - - memset(tmp, 0, length[i]); - break; - } - case TSDB_DATA_TYPE_NCHAR: { - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetByteArrayFp, i, - jniFromNCharToByteArray(env, (char *)row[i], length[i])); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: { - int precision = taos_result_precision(result); - (*env)->CallVoidMethod(env, rowobj, g_rowdataSetTimestampFp, i, (jlong) * ((int64_t *)row[i]), precision); - break; - } - default: - break; - } - } - - return JNI_SUCCESS; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNIEnv *env, jobject jobj, jlong con, - jlong res, jobject rowobj) { - TAOS * tscon = (TAOS *)con; - int32_t code = check_for_params(jobj, con, res); - if (code != JNI_SUCCESS) { - return code; - } - - TAOS_RES * tres = (TAOS_RES *)res; - TAOS_FIELD *fields = taos_fetch_fields(tres); - - int32_t numOfFields = taos_num_fields(tres); - assert(numOfFields > 0); - - TAOS_ROW row = NULL; - int32_t numOfRows = taos_fetch_block(tres, &row); - if (numOfRows == 0) { - code = taos_errno(tres); - if (code == JNI_SUCCESS) { - jniDebug("jobj:%p, conn:%p, resultset:%p, numOfFields:%d, no data to retrieve", jobj, tscon, (void *)res, - numOfFields); - return JNI_FETCH_END; - } else { - jniDebug("jobj:%p, conn:%p, query interrupted", jobj, tscon); - return JNI_RESULT_SET_NULL; - } - } - - (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfRowsFp, (jint)numOfRows); - (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfColsFp, (jint)numOfFields); - - for (int i = 0; i < numOfFields; i++) { - int bytes = fields[i].bytes; - - if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_NCHAR) { - bytes += 2; - } - (*env)->CallVoidMethod(env, rowobj, g_blockdataSetByteArrayFp, i, bytes * numOfRows, - jniFromNCharToByteArray(env, (char *)row[i], bytes * numOfRows)); - } - - return JNI_SUCCESS; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeConnectionImp(JNIEnv *env, jobject jobj, - jlong con) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection is already closed", jobj); - return JNI_CONNECTION_NULL; - } else { - jniDebug("jobj:%p, conn:%p, close connection success", jobj, tscon); - taos_close(tscon); - return JNI_SUCCESS; - } -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNIEnv *env, jobject jobj, jlong con, - jboolean restart, jstring jtopic, - jstring jsql, jint jinterval) { - jlong sub = 0; - TAOS *taos = (TAOS *)con; - char *topic = NULL; - char *sql = NULL; - - jniGetGlobalMethod(env); - jniDebug("jobj:%p, in TSDBJNIConnector_subscribeImp", jobj); - - if (jtopic != NULL) { - topic = (char *)(*env)->GetStringUTFChars(env, jtopic, NULL); - } - if (jsql != NULL) { - sql = (char *)(*env)->GetStringUTFChars(env, jsql, NULL); - } - - if (topic == NULL || sql == NULL) { - jniDebug("jobj:%p, invalid argument: topic or sql is NULL", jobj); - return sub; - } - - TAOS_SUB *tsub = taos_subscribe(taos, (int)restart, topic, sql, NULL, NULL, jinterval); - sub = (jlong)tsub; - - if (sub == 0) { - jniDebug("jobj:%p, failed to subscribe: topic:%s", jobj, topic); - } else { - jniDebug("jobj:%p, successfully subscribe: topic: %s", jobj, topic); - } - - (*env)->ReleaseStringUTFChars(env, jtopic, topic); - (*env)->ReleaseStringUTFChars(env, jsql, sql); - - return sub; -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub) { - jniDebug("jobj:%p, in TSDBJNIConnector_consumeImp, sub:%lld", jobj, sub); - jniGetGlobalMethod(env); - - TAOS_SUB *tsub = (TAOS_SUB *)sub; - TAOS_RES *res = taos_consume(tsub); - - if (res == NULL) { - jniError("jobj:%p, tsub:%p, taos_consume returns NULL", jobj, tsub); - return 0l; - } - - return (jlong)res; -} - -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp(JNIEnv *env, jobject jobj, jlong sub, - jboolean keepProgress) { - TAOS_SUB *tsub = (TAOS_SUB *)sub; - taos_unsubscribe(tsub, keepProgress); -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_validateCreateTableSqlImp(JNIEnv *env, jobject jobj, - jlong con, jbyteArray jsql) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection is closed", jobj); - return JNI_CONNECTION_NULL; - } - - if (jsql == NULL) { - jniError("jobj:%p, conn:%p, sql is null", jobj, tscon); - return JNI_SQL_NULL; - } - - jsize len = (*env)->GetArrayLength(env, jsql); - - char *str = (char *)calloc(1, sizeof(char) * (len + 1)); - (*env)->GetByteArrayRegion(env, jsql, 0, len, (jbyte *)str); - if ((*env)->ExceptionCheck(env)) { - // todo handle error - } - - int code = taos_validate_sql(tscon, str); - jniDebug("jobj:%p, conn:%p, code is %d", jobj, tscon, code); - - free(str); - return code; -} - -JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getTsCharset(JNIEnv *env, jobject jobj) { - return (*env)->NewStringUTF(env, (const char *)tsCharset); -} - -/** - * Get Result Time Precision - * @param env vm - * @param jobj the TSDBJNIConnector java object - * @param con the c connection pointer - * @param res the TAOS_RES object, i.e. the SSqlObject - * @return precision 0:ms 1:us 2:ns - */ -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultTimePrecisionImp(JNIEnv *env, jobject jobj, - jlong con, jlong res) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection is closed", jobj); - return JNI_CONNECTION_NULL; - } - - TAOS_RES *result = (TAOS_RES *)res; - if (result == NULL) { - jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon); - return JNI_RESULT_SET_NULL; - } - - return taos_result_precision(result); -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtImp(JNIEnv *env, jobject jobj, - jbyteArray jsql, jlong con) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - if (jsql == NULL) { - jniError("jobj:%p, conn:%p, empty sql string", jobj, tscon); - return JNI_SQL_NULL; - } - - jsize len = (*env)->GetArrayLength(env, jsql); - - char *str = (char *)calloc(1, sizeof(char) * (len + 1)); - if (str == NULL) { - jniError("jobj:%p, conn:%p, alloc memory failed", jobj, tscon); - return JNI_OUT_OF_MEMORY; - } - - (*env)->GetByteArrayRegion(env, jsql, 0, len, (jbyte *)str); - if ((*env)->ExceptionCheck(env)) { - // todo handle error - } - - TAOS_STMT *pStmt = taos_stmt_init(tscon); - int32_t code = taos_stmt_prepare(pStmt, str, len); - tfree(str); - if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - return JNI_TDENGINE_ERROR; - } - - return (jlong)pStmt; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameImp(JNIEnv *env, jobject jobj, - jlong stmt, jstring jname, - jlong conn) { - TAOS *tsconn = (TAOS *)conn; - if (tsconn == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - TAOS_STMT *pStmt = (TAOS_STMT *)stmt; - if (pStmt == NULL) { - jniError("jobj:%p, conn:%p, invalid stmt handle", jobj, tsconn); - return JNI_SQL_NULL; - } - - const char *name = (*env)->GetStringUTFChars(env, jname, NULL); - - int32_t code = taos_stmt_set_tbname((void *)stmt, name); - if (code != TSDB_CODE_SUCCESS) { - (*env)->ReleaseStringUTFChars(env, jname, name); - - jniError("jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); - return JNI_TDENGINE_ERROR; - } - - jniDebug("jobj:%p, conn:%p, set stmt bind table name:%s", jobj, tsconn, name); - (*env)->ReleaseStringUTFChars(env, jname, name); - return JNI_SUCCESS; -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp( - JNIEnv *env, jobject jobj, jlong stmt, jbyteArray colDataList, jbyteArray lengthList, jbyteArray nullList, - jint dataType, jint dataBytes, jint numOfRows, jint colIndex, jlong con) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - TAOS_STMT *pStmt = (TAOS_STMT *)stmt; - if (pStmt == NULL) { - jniError("jobj:%p, conn:%p, invalid stmt", jobj, tscon); - return JNI_SQL_NULL; - } - - // todo refactor - jsize len = (*env)->GetArrayLength(env, colDataList); - char *colBuf = (char *)calloc(1, len); - (*env)->GetByteArrayRegion(env, colDataList, 0, len, (jbyte *)colBuf); - if ((*env)->ExceptionCheck(env)) { - // todo handle error - } - - len = (*env)->GetArrayLength(env, lengthList); - char *lengthArray = (char *)calloc(1, len); - (*env)->GetByteArrayRegion(env, lengthList, 0, len, (jbyte *)lengthArray); - if ((*env)->ExceptionCheck(env)) { - } - - len = (*env)->GetArrayLength(env, nullList); - char *nullArray = (char *)calloc(1, len); - (*env)->GetByteArrayRegion(env, nullList, 0, len, (jbyte *)nullArray); - if ((*env)->ExceptionCheck(env)) { - } - - // bind multi-rows with only one invoke. - TAOS_MULTI_BIND *b = calloc(1, sizeof(TAOS_MULTI_BIND)); - - b->num = numOfRows; - b->buffer_type = dataType; // todo check data type - b->buffer_length = IS_VAR_DATA_TYPE(dataType) ? dataBytes : tDataTypes[dataType].bytes; - b->is_null = nullArray; - b->buffer = colBuf; - b->length = (int32_t *)lengthArray; - - // set the length and is_null array - if (!IS_VAR_DATA_TYPE(dataType)) { - int32_t bytes = tDataTypes[dataType].bytes; - for (int32_t i = 0; i < numOfRows; ++i) { - b->length[i] = bytes; - } - } - - int32_t code = taos_stmt_bind_single_param_batch(pStmt, b, colIndex); - tfree(b->length); - tfree(b->buffer); - tfree(b->is_null); - tfree(b); - - if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - return JNI_TDENGINE_ERROR; - } - - return JNI_SUCCESS; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeBatchImp(JNIEnv *env, jobject jobj, jlong stmt, - jlong con) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - TAOS_STMT *pStmt = (TAOS_STMT *)stmt; - if (pStmt == NULL) { - jniError("jobj:%p, conn:%p, invalid stmt", jobj, tscon); - return JNI_SQL_NULL; - } - - taos_stmt_add_batch(pStmt); - int32_t code = taos_stmt_execute(pStmt); - if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - return JNI_TDENGINE_ERROR; - } - - jniDebug("jobj:%p, conn:%p, batch execute", jobj, tscon); - return JNI_SUCCESS; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeStmt(JNIEnv *env, jobject jobj, jlong stmt, - jlong con) { - TAOS *tscon = (TAOS *)con; - if (tscon == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - TAOS_STMT *pStmt = (TAOS_STMT *)stmt; - if (pStmt == NULL) { - jniError("jobj:%p, conn:%p, invalid stmt", jobj, tscon); - return JNI_SQL_NULL; - } - - int32_t code = taos_stmt_close(pStmt); - if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - return JNI_TDENGINE_ERROR; - } - - jniDebug("jobj:%p, conn:%p, stmt closed", jobj, tscon); - return JNI_SUCCESS; -} - -JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsImp( - JNIEnv *env, jobject jobj, jlong stmt, jstring tableName, jint numOfTags, jbyteArray tags, jbyteArray typeList, - jbyteArray lengthList, jbyteArray nullList, jlong conn) { - TAOS *tsconn = (TAOS *)conn; - if (tsconn == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - TAOS_STMT *pStmt = (TAOS_STMT *)stmt; - if (pStmt == NULL) { - jniError("jobj:%p, conn:%p, invalid stmt handle", jobj, tsconn); - return JNI_SQL_NULL; - } - - jsize len = (*env)->GetArrayLength(env, tags); - char *tagsData = (char *)calloc(1, len); - (*env)->GetByteArrayRegion(env, tags, 0, len, (jbyte *)tagsData); - if ((*env)->ExceptionCheck(env)) { - // todo handle error - } - - len = (*env)->GetArrayLength(env, lengthList); - int64_t *lengthArray = (int64_t *)calloc(1, len); - (*env)->GetByteArrayRegion(env, lengthList, 0, len, (jbyte *)lengthArray); - if ((*env)->ExceptionCheck(env)) { - } - - len = (*env)->GetArrayLength(env, typeList); - char *typeArray = (char *)calloc(1, len); - (*env)->GetByteArrayRegion(env, typeList, 0, len, (jbyte *)typeArray); - if ((*env)->ExceptionCheck(env)) { - } - - len = (*env)->GetArrayLength(env, nullList); - int32_t *nullArray = (int32_t *)calloc(1, len); - (*env)->GetByteArrayRegion(env, nullList, 0, len, (jbyte *)nullArray); - if ((*env)->ExceptionCheck(env)) { - } - - const char *name = (*env)->GetStringUTFChars(env, tableName, NULL); - char * curTags = tagsData; - - TAOS_BIND *tagsBind = calloc(numOfTags, sizeof(TAOS_BIND)); - for (int32_t i = 0; i < numOfTags; ++i) { - tagsBind[i].buffer_type = typeArray[i]; - tagsBind[i].buffer = curTags; - tagsBind[i].is_null = &nullArray[i]; - tagsBind[i].length = (uintptr_t *)&lengthArray[i]; - - curTags += lengthArray[i]; - } - - int32_t code = taos_stmt_set_tbname_tags((void *)stmt, name, tagsBind); - - int32_t nTags = (int32_t)numOfTags; - jniDebug("jobj:%p, conn:%p, set table name:%s, numOfTags:%d", jobj, tsconn, name, nTags); - - tfree(tagsData); - tfree(lengthArray); - tfree(typeArray); - tfree(nullArray); - tfree(tagsBind); - (*env)->ReleaseStringUTFChars(env, tableName, name); - - if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); - return JNI_TDENGINE_ERROR; - } - return JNI_SUCCESS; -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_insertLinesImp(JNIEnv *env, jobject jobj, - jobjectArray lines, jlong conn) { - TAOS *taos = (TAOS *)conn; - if (taos == NULL) { - jniError("jobj:%p, connection already closed", jobj); - return JNI_CONNECTION_NULL; - } - - int numLines = (*env)->GetArrayLength(env, lines); - char **c_lines = calloc(numLines, sizeof(char *)); - if (c_lines == NULL) { - jniError("c_lines:%p, alloc memory failed", c_lines); - return JNI_OUT_OF_MEMORY; - } - for (int i = 0; i < numLines; ++i) { - jstring line = (jstring)((*env)->GetObjectArrayElement(env, lines, i)); - c_lines[i] = (char *)(*env)->GetStringUTFChars(env, line, 0); - } - - int code = taos_insert_lines(taos, c_lines, numLines); - - for (int i = 0; i < numLines; ++i) { - jstring line = (jstring)((*env)->GetObjectArrayElement(env, lines, i)); - (*env)->ReleaseStringUTFChars(env, line, c_lines[i]); - } - - tfree(c_lines); - if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, taos, tstrerror(code)); - - return JNI_TDENGINE_ERROR; - } - return code; -} \ No newline at end of file diff --git a/2.0/src/client/src/taos.def b/2.0/src/client/src/taos.def deleted file mode 100644 index 7d3b8e80c20226c4a509c95ab5728f41852110f5..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/taos.def +++ /dev/null @@ -1,45 +0,0 @@ -EXPORTS -taos_init -taos_cleanup -taos_options -taos_connect -taos_connect_auth -taos_close -taos_stmt_init -taos_stmt_prepare -taos_stmt_set_tbname_tags -taos_stmt_set_tbname -taos_stmt_is_insert -taos_stmt_num_params -taos_stmt_bind_param -taos_stmt_add_batch -taos_stmt_execute -taos_stmt_use_result -taos_stmt_close -taos_stmt_errstr -taos_query -taos_fetch_row -taos_result_precision -taos_free_result -taos_field_count -taos_num_fields -taos_affected_rows -taos_fetch_fields -taos_select_db -taos_print_row -taos_stop_query -taos_fetch_block -taos_validate_sql -taos_fetch_lengths -taos_get_server_info -taos_get_client_info -taos_errstr -taos_errno -taos_query_a -taos_fetch_rows_a -taos_subscribe -taos_consume -taos_unsubscribe -taos_open_stream -taos_close_stream -taos_load_table_info diff --git a/2.0/src/client/src/taos.rc.in b/2.0/src/client/src/taos.rc.in deleted file mode 100644 index 751be85fd01ebf36546a05453df4f29f1937135f..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/taos.rc.in +++ /dev/null @@ -1,31 +0,0 @@ -1 VERSIONINFO - FILEVERSION ${TD_VER_NUMBER} - PRODUCTVERSION ${TD_VER_NUMBER} - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x0L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "FileDescription", "Native C Driver for TDengine" - VALUE "FileVersion", "${TD_VER_NUMBER}" - VALUE "InternalName", "taos.dll(${TD_VER_CPUTYPE})" - VALUE "LegalCopyright", "Copyright (C) 2020 TAOS Data" - VALUE "OriginalFilename", "" - VALUE "ProductName", "taos.dll(${TD_VER_CPUTYPE})" - VALUE "ProductVersion", "${TD_VER_NUMBER}" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END \ No newline at end of file diff --git a/2.0/src/client/src/tscAsync.c b/2.0/src/client/src/tscAsync.c deleted file mode 100644 index b2f8eda474b317af0bd89dcb1e901419c08e8dfa..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscAsync.c +++ /dev/null @@ -1,419 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tutil.h" - -#include "tnote.h" -#include "trpc.h" -#include "tscLog.h" -#include "tscSubquery.h" -#include "tscUtil.h" -#include "tsched.h" -#include "qTableMeta.h" -#include "tsclient.h" - -static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows); - -/* - * Proxy function to perform sequentially query&retrieve operation. - * If sql queries upon a super table and two-stage merge procedure is not involved (when employ the projection - * query), it will sequentially query&retrieve data for all vnodes - */ -static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows); - -void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* param, const char* sqlstr, size_t sqlLen) { - SSqlCmd* pCmd = &pSql->cmd; - - pSql->signature = pSql; - pSql->param = param; - pSql->pTscObj = pObj; - pSql->parseRetry= 0; - pSql->maxRetry = TSDB_MAX_REPLICA; - pSql->fp = fp; - pSql->fetchFp = fp; - - registerSqlObj(pSql); - - pSql->sqlstr = calloc(1, sqlLen + 1); - if (pSql->sqlstr == NULL) { - tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self); - pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscAsyncResultOnError(pSql); - return; - } - - strntolower(pSql->sqlstr, sqlstr, (int32_t)sqlLen); - - tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); - pCmd->resColumnId = TSDB_RES_COL_ID; - - taosAcquireRef(tscObjRef, pSql->self); - - int32_t code = tsParseSql(pSql, true); - - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); - return; - } - - if (code != TSDB_CODE_SUCCESS) { - pSql->res.code = code; - tscAsyncResultOnError(pSql); - taosReleaseRef(tscObjRef, pSql->self); - return; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - executeQuery(pSql, pQueryInfo); - taosReleaseRef(tscObjRef, pSql->self); -} - -// TODO return the correct error code to client in tscQueueAsyncError -void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) { - taos_query_ra(taos, sqlstr, fp, param); -} - -TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) { - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - tscError("pObj:%p is NULL or freed", pObj); - terrno = TSDB_CODE_TSC_DISCONNECTED; - tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED); - return NULL; - } - - int32_t sqlLen = (int32_t)strlen(sqlstr); - if (sqlLen > tsMaxSQLStringLen) { - tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); - terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - tscQueueAsyncError(fp, param, terrno); - return NULL; - } - - nPrintTsc("%s", sqlstr); - - SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - tscError("failed to malloc sqlObj"); - tscQueueAsyncError(fp, param, TSDB_CODE_TSC_OUT_OF_MEMORY); - return NULL; - } - - doAsyncQuery(pObj, pSql, fp, param, sqlstr, sqlLen); - - return pSql; -} - - -static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) { - if (tres == NULL) { - return; - } - - SSqlObj *pSql = (SSqlObj *)tres; - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - if (numOfRows == 0) { - if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes. - tscTryQueryNextVnode(pSql, tscAsyncQueryRowsForNextVnode); - } else { - /* - * all available virtual node has been checked already, now we need to check - * for the next subclause queries - */ - if (pCmd->active->sibling != NULL) { - pCmd->active = pCmd->active->sibling; - tscTryQueryNextClause(pSql, tscAsyncQueryRowsForNextVnode); - return; - } - - /* - * 1. has reach the limitation - * 2. no remain virtual nodes to be retrieved anymore - */ - (*pSql->fetchFp)(param, pSql, 0); - } - - return; - } - - // local merge has handle this situation during super table non-projection query. - if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE) { - pRes->numOfClauseTotal += pRes->numOfRows; - } - - (*pSql->fetchFp)(param, tres, numOfRows); -} - -// actual continue retrieve function with user-specified callback function -static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRows, __async_cb_func_t fp) { - SSqlObj *pSql = (SSqlObj *)tres; - if (pSql == NULL) { // error - tscError("sql object is NULL"); - return; - } - - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if ((pRes->qId == 0 || numOfRows != 0) && pCmd->command < TSDB_SQL_LOCAL) { - if (pRes->qId == 0 && numOfRows != 0) { - tscError("qhandle is NULL"); - } else { - pRes->code = numOfRows; - } - - tscAsyncResultOnError(pSql); - return; - } - - pSql->fp = fp; - if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE && pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; - } - - if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscFetchDatablockForSubquery(pSql); - } else { - tscBuildAndSendRequest(pSql, NULL); - } -} - -/* - * retrieve callback for fetch rows proxy. - * The below two functions both serve as the callback function of query virtual node. - * query callback first, and then followed by retrieve callback - */ -static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows) { - // query completed, continue to retrieve - tscProcessAsyncRetrieveImpl(param, tres, numOfRows, tscAsyncFetchRowsProxy); -} - -void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) { - SSqlObj *pSql = (SSqlObj *)tres; - if (pSql == NULL || pSql->signature != pSql) { - tscError("sql object is NULL"); - tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED); - return; - } - - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - // user-defined callback function is stored in fetchFp - pSql->fetchFp = fp; - pSql->fp = tscAsyncFetchRowsProxy; - pSql->param = param; - - tscResetForNextRetrieve(pRes); - - // handle outer query based on the already retrieved nest query results. - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { - SSchedMsg schedMsg = {0}; - schedMsg.fp = doRetrieveSubqueryData; - schedMsg.ahandle = (void *)pSql; - schedMsg.thandle = (void *)1; - schedMsg.msg = 0; - taosScheduleTask(tscQhandle, &schedMsg); - return; - } - - if (pRes->qId == 0) { - tscError("qhandle is invalid"); - pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; - tscAsyncResultOnError(pSql); - return; - } - - if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscFetchDatablockForSubquery(pSql); - } else if (pRes->completed) { - if(pCmd->command == TSDB_SQL_FETCH || (pCmd->command >= TSDB_SQL_SERV_STATUS && pCmd->command <= TSDB_SQL_CURRENT_USER)) { - if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes. - tscTryQueryNextVnode(pSql, tscAsyncQueryRowsForNextVnode); - } else { - /* - * all available virtual nodes in current clause has been checked already, now try the - * next one in the following union subclause - */ - if (pCmd->active->sibling != NULL) { - pCmd->active = pCmd->active->sibling; // todo refactor - tscTryQueryNextClause(pSql, tscAsyncQueryRowsForNextVnode); - return; - } - - /* - * 1. has reach the limitation - * 2. no remain virtual nodes to be retrieved anymore - */ - (*pSql->fetchFp)(param, pSql, 0); - } - - return; - } else if (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE) { - // in case of show command, return no data - (*pSql->fetchFp)(param, pSql, 0); - } else { - assert(0); - } - } else { // current query is not completed, continue retrieve from node - if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE && pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; - } - - SQueryInfo* pQueryInfo1 = tscGetQueryInfo(&pSql->cmd); - tscBuildAndSendRequest(pSql, pQueryInfo1); - } -} - -// this function will be executed by queue task threads, so the terrno is not valid -static void tscProcessAsyncError(SSchedMsg *pMsg) { - void (*fp)() = pMsg->ahandle; - terrno = *(int32_t*) pMsg->msg; - tfree(pMsg->msg); - (*fp)(pMsg->thandle, NULL, terrno); -} - -void tscQueueAsyncError(void(*fp), void *param, int32_t code) { - int32_t* c = malloc(sizeof(int32_t)); - *c = code; - - SSchedMsg schedMsg = {0}; - schedMsg.fp = tscProcessAsyncError; - schedMsg.ahandle = fp; - schedMsg.thandle = param; - schedMsg.msg = c; - taosScheduleTask(tscQhandle, &schedMsg); -} - -static void tscAsyncResultCallback(SSchedMsg *pMsg) { - SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pMsg->ahandle); - if (pSql == NULL || pSql->signature != pSql) { - tscDebug("%p SqlObj is freed, not add into queue async res", pMsg->ahandle); - return; - } - - assert(pSql->res.code != TSDB_CODE_SUCCESS); - tscError("0x%"PRIx64" async result callback, code:%s", pSql->self, tstrerror(pSql->res.code)); - - SSqlRes *pRes = &pSql->res; - if (pSql->fp == NULL || pSql->fetchFp == NULL){ - taosReleaseRef(tscObjRef, pSql->self); - return; - } - - pSql->fp = pSql->fetchFp; - (*pSql->fp)(pSql->param, pSql, pRes->code); - taosReleaseRef(tscObjRef, pSql->self); -} - -void tscAsyncResultOnError(SSqlObj* pSql) { - SSchedMsg schedMsg = {0}; - schedMsg.fp = tscAsyncResultCallback; - schedMsg.ahandle = (void *)pSql->self; - schedMsg.thandle = (void *)1; - schedMsg.msg = 0; - taosScheduleTask(tscQhandle, &schedMsg); -} - -int tscSendMsgToServer(SSqlObj *pSql); - -void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { - SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)param); - if (pSql == NULL) return; - - assert(pSql->signature == pSql && (int64_t)param == pSql->self); - - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - pRes->code = code; - - SSqlObj *sub = (SSqlObj*) res; - const char* msg = (sub->cmd.command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"multi-tableMeta"; - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" get %s failed, code:%s", pSql->self, msg, tstrerror(code)); - if (code == TSDB_CODE_RPC_FQDN_ERROR) { - size_t sz = strlen(tscGetErrorMsgPayload(&sub->cmd)); - tscAllocPayload(&pSql->cmd, (int)sz + 1); - memcpy(tscGetErrorMsgPayload(&pSql->cmd), tscGetErrorMsgPayload(&sub->cmd), sz); - } - goto _error; - } - - tscDebug("0x%"PRIx64" get %s successfully", pSql->self, msg); - if (pSql->pStream == NULL) { - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - if (pQueryInfo != NULL && TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT)) { - tscDebug("0x%" PRIx64 " continue parse sql after get table-meta", pSql->self); - - code = tsParseSql(pSql, false); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); - return; - } else if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { // stmt insert - (*pSql->fp)(pSql->param, pSql, code); - } else if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { // file insert - tscImportDataFromFile(pSql); - } else { // sql string insert - tscHandleMultivnodeInsert(pSql); - } - } else { - if (pSql->retryReason != TSDB_CODE_SUCCESS) { - tscDebug("0x%" PRIx64 " update cached table-meta, re-validate sql statement and send query again", pSql->self); - pSql->retryReason = TSDB_CODE_SUCCESS; - } else { - tscDebug("0x%" PRIx64 " cached table-meta, continue validate sql statement and send query", pSql->self); - } - - code = tsParseSql(pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); - return; - } else if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pCmd); - executeQuery(pSql, pQueryInfo1); - } - - taosReleaseRef(tscObjRef, pSql->self); - return; - } else { // stream computing - tscDebug("0x%"PRIx64" stream:%p meta is updated, start new query, command:%d", pSql->self, pSql->pStream, pCmd->command); - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - if (tscNumOfExprs(pQueryInfo) == 0) { - tsParseSql(pSql, false); - } - - (*pSql->fp)(pSql->param, pSql, code); - taosReleaseRef(tscObjRef, pSql->self); - return; - } - - _error: - pRes->code = code; - tscAsyncResultOnError(pSql); - taosReleaseRef(tscObjRef, pSql->self); -} diff --git a/2.0/src/client/src/tscGlobalmerge.c b/2.0/src/client/src/tscGlobalmerge.c deleted file mode 100644 index c8ff36db51d9e5bd3e64621369780c01cc1d15cd..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscGlobalmerge.c +++ /dev/null @@ -1,1144 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "texpr.h" -#include "tlosertree.h" - -#include "tscGlobalmerge.h" -#include "tscSubquery.h" -#include "tscLog.h" -#include "qUtil.h" - -#define COLMODEL_GET_VAL(data, schema, rowId, colId) \ - (data + (schema)->pFields[colId].offset * ((schema)->capacity) + (rowId) * (schema)->pFields[colId].field.bytes) - - -typedef struct SCompareParam { - SLocalDataSource **pLocalData; - tOrderDescriptor * pDesc; - int32_t num; - int32_t groupOrderType; -} SCompareParam; - -static bool needToMerge(SSDataBlock* pBlock, SArray* columnIndexList, int32_t index, char **buf) { - int32_t ret = 0; - - size_t size = taosArrayGetSize(columnIndexList); - if (size > 0) { - ret = compare_aRv(pBlock, columnIndexList, (int32_t) size, index, buf, TSDB_ORDER_ASC); - } - - // if ret == 0, means the result belongs to the same group - return (ret == 0); -} - -static int32_t treeComparator(const void *pLeft, const void *pRight, void *param) { - int32_t pLeftIdx = *(int32_t *)pLeft; - int32_t pRightIdx = *(int32_t *)pRight; - - SCompareParam * pParam = (SCompareParam *)param; - tOrderDescriptor * pDesc = pParam->pDesc; - SLocalDataSource **pLocalData = pParam->pLocalData; - - /* this input is exhausted, set the special value to denote this */ - if (pLocalData[pLeftIdx]->rowIdx == -1) { - return 1; - } - - if (pLocalData[pRightIdx]->rowIdx == -1) { - return -1; - } - - if (pParam->groupOrderType == TSDB_ORDER_DESC) { // desc - return compare_d(pDesc, pParam->num, pLocalData[pLeftIdx]->rowIdx, pLocalData[pLeftIdx]->filePage.data, - pParam->num, pLocalData[pRightIdx]->rowIdx, pLocalData[pRightIdx]->filePage.data); - } else { - return compare_a(pDesc, pParam->num, pLocalData[pLeftIdx]->rowIdx, pLocalData[pLeftIdx]->filePage.data, - pParam->num, pLocalData[pRightIdx]->rowIdx, pLocalData[pRightIdx]->filePage.data); - } -} - -int32_t tscCreateGlobalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, - SQueryInfo* pQueryInfo, SGlobalMerger **pMerger, int64_t id) { - if (pMemBuffer == NULL) { - tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer); - tscError("0x%"PRIx64" %p pMemBuffer is NULL", id, pMemBuffer); - return TSDB_CODE_TSC_APP_ERROR; - } - - if (pDesc->pColumnModel == NULL) { - tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer); - tscError("0x%"PRIx64" no local buffer or intermediate result format model", id); - return TSDB_CODE_TSC_APP_ERROR; - } - - int32_t numOfFlush = 0; - for (int32_t i = 0; i < numOfBuffer; ++i) { - int32_t len = pMemBuffer[i]->fileMeta.flushoutData.nLength; - if (len == 0) { - tscDebug("0x%"PRIx64" no data retrieved from orderOfVnode:%d", id, i + 1); - continue; - } - - numOfFlush += len; - } - - if (numOfFlush == 0 || numOfBuffer == 0) { - tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer); - tscDebug("0x%"PRIx64" no data to retrieve", id); - return TSDB_CODE_SUCCESS; - } - - if (pDesc->pColumnModel->capacity >= pMemBuffer[0]->pageSize) { - tscError("0x%"PRIx64" Invalid value of buffer capacity %d and page size %d ", id, pDesc->pColumnModel->capacity, - pMemBuffer[0]->pageSize); - - tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer); - return TSDB_CODE_TSC_APP_ERROR; - } - - *pMerger = (SGlobalMerger *) calloc(1, sizeof(SGlobalMerger)); - if ((*pMerger) == NULL) { - tscError("0x%"PRIx64" failed to create local merge structure, out of memory", id); - - tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - (*pMerger)->pExtMemBuffer = pMemBuffer; - (*pMerger)->pLocalDataSrc = calloc(numOfFlush, POINTER_BYTES); - assert((*pMerger)->pLocalDataSrc != NULL); - - (*pMerger)->numOfBuffer = numOfFlush; - (*pMerger)->numOfVnode = numOfBuffer; - - (*pMerger)->pDesc = pDesc; - tscDebug("0x%"PRIx64" the number of merged leaves is: %d", id, (*pMerger)->numOfBuffer); - - int32_t idx = 0; - for (int32_t i = 0; i < numOfBuffer; ++i) { - int32_t numOfFlushoutInFile = pMemBuffer[i]->fileMeta.flushoutData.nLength; - - for (int32_t j = 0; j < numOfFlushoutInFile; ++j) { - SLocalDataSource *ds = (SLocalDataSource *)malloc(sizeof(SLocalDataSource) + pMemBuffer[0]->pageSize); - if (ds == NULL) { - tscError("0x%"PRIx64" failed to create merge structure", id); - tfree(*pMerger); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - (*pMerger)->pLocalDataSrc[idx] = ds; - - ds->pMemBuffer = pMemBuffer[i]; - ds->flushoutIdx = j; - ds->filePage.num = 0; - ds->pageId = 0; - ds->rowIdx = 0; - - tscDebug("0x%"PRIx64" load data from disk into memory, orderOfVnode:%d, total:%d", id, i + 1, idx + 1); - tExtMemBufferLoadData(pMemBuffer[i], &(ds->filePage), j, 0); -#ifdef _DEBUG_VIEW - printf("load data page into mem for build loser tree: %" PRIu64 " rows\n", ds->filePage.num); - SSrcColumnInfo colInfo[256] = {0}; - SQueryInfo * pQueryInfo = tscGetQueryInfo(pCmd); - - tscGetSrcColumnInfo(colInfo, pQueryInfo); - - tColModelDisplayEx(pDesc->pColumnModel, ds->filePage.data, ds->filePage.num, - pMemBuffer[0]->numOfElemsPerPage, colInfo); -#endif - - if (ds->filePage.num == 0) { // no data in this flush, the index does not increase - tscDebug("0x%"PRIx64" flush data is empty, ignore %d flush record", id, idx); - tfree(ds); - continue; - } - - idx += 1; - } - } - - // no data actually, no need to merge result. - if (idx == 0) { - tscDebug("0x%"PRIx64" retrieved no data", id); - tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer); - return TSDB_CODE_SUCCESS; - } - - (*pMerger)->numOfBuffer = idx; - - SCompareParam *param = malloc(sizeof(SCompareParam)); - if (param == NULL) { - tfree((*pMerger)); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - param->pLocalData = (*pMerger)->pLocalDataSrc; - param->pDesc = (*pMerger)->pDesc; - param->num = (*pMerger)->pLocalDataSrc[0]->pMemBuffer->numOfElemsPerPage; - - param->groupOrderType = pQueryInfo->groupbyExpr.orderType; - - int32_t code = tLoserTreeCreate(&(*pMerger)->pLoserTree, (*pMerger)->numOfBuffer, param, treeComparator); - if ((*pMerger)->pLoserTree == NULL || code != TSDB_CODE_SUCCESS) { - tfree(param); - tfree((*pMerger)); - return code; - } - - (*pMerger)->rowSize = pMemBuffer[0]->nElemSize; - - // todo fixed row size is larger than the minimum page size; - assert((*pMerger)->rowSize <= pMemBuffer[0]->pageSize); - - if ((*pMerger)->pLoserTree == NULL) { - tfree((*pMerger)->pLoserTree); - tfree(param); - tfree((*pMerger)); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - // restore the limitation value at the last stage - if (pQueryInfo->orderProjectQuery) { - pQueryInfo->limit.limit = pQueryInfo->clauseLimit; - pQueryInfo->limit.offset = pQueryInfo->prjOffset; - } - - // we change the capacity of schema to denote that there is only one row in temp buffer - (*pMerger)->pDesc->pColumnModel->capacity = 1; - - return TSDB_CODE_SUCCESS; -} - -static int32_t tscFlushTmpBufferImpl(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, - int32_t orderType) { - if (pPage->num == 0) { - return 0; - } - - assert(pPage->num <= pDesc->pColumnModel->capacity); - - // sort before flush to disk, the data must be consecutively put on tFilePage. - if (pDesc->orderInfo.numOfCols > 0) { - tColDataQSort(pDesc, (int32_t)pPage->num, 0, (int32_t)pPage->num - 1, pPage->data, orderType); - } - -#ifdef _DEBUG_VIEW - printf("%" PRIu64 " rows data flushed to disk after been sorted:\n", pPage->num); - tColModelDisplay(pDesc->pColumnModel, pPage->data, pPage->num, pPage->num); -#endif - - // write to cache after being sorted - if (tExtMemBufferPut(pMemoryBuf, pPage->data, (int32_t)pPage->num) < 0) { - tscError("failed to save data in temporary buffer"); - return -1; - } - - pPage->num = 0; - return 0; -} - -int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, int32_t orderType) { - int32_t ret = 0; - if ((ret = tscFlushTmpBufferImpl(pMemoryBuf, pDesc, pPage, orderType)) != 0) { - return ret; - } - - if ((ret = tExtMemBufferFlush(pMemoryBuf)) != 0) { - return ret; - } - - return 0; -} - -int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, void *data, - int32_t numOfRows, int32_t orderType) { - SColumnModel *pModel = pDesc->pColumnModel; - - if (pPage->num + numOfRows <= pModel->capacity) { - tColModelAppend(pModel, pPage, data, 0, numOfRows, numOfRows); - return 0; - } - - // current buffer is overflow, flush data to extensive buffer - int32_t numOfRemainEntries = pModel->capacity - (int32_t)pPage->num; - tColModelAppend(pModel, pPage, data, 0, numOfRemainEntries, numOfRows); - - // current buffer is full, need to flushed to disk - assert(pPage->num == pModel->capacity); - int32_t code = tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType); - if (code != 0) { - return code; - } - - int32_t remain = numOfRows - numOfRemainEntries; - - while (remain > 0) { - int32_t numOfWriteElems = 0; - if (remain > pModel->capacity) { - numOfWriteElems = pModel->capacity; - } else { - numOfWriteElems = remain; - } - - tColModelAppend(pModel, pPage, data, numOfRows - remain, numOfWriteElems, numOfRows); - - if (pPage->num == pModel->capacity) { - if ((code = tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType)) != TSDB_CODE_SUCCESS) { - return code; - } - } else { - pPage->num = numOfWriteElems; - } - - remain -= numOfWriteElems; - numOfRemainEntries += numOfWriteElems; - } - - return 0; -} - -void tscDestroyGlobalMerger(SGlobalMerger* pMerger) { - if (pMerger == NULL) { - return; - } - - for (int32_t i = 0; i < pMerger->numOfBuffer; ++i) { - tfree(pMerger->pLocalDataSrc[i]); - } - - pMerger->numOfBuffer = 0; - tscDestroyGlobalMergerEnv(pMerger->pExtMemBuffer, pMerger->pDesc, pMerger->numOfVnode); - - pMerger->numOfCompleted = 0; - - if (pMerger->pLoserTree) { - tfree(pMerger->pLoserTree->param); - tfree(pMerger->pLoserTree); - } - - tfree(pMerger->buf); - tfree(pMerger->pLocalDataSrc); - free(pMerger); -} - -static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SQueryInfo* pQueryInfo, SColumnModel *pModel) { - int32_t numOfGroupByCols = 0; - - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { - numOfGroupByCols = pQueryInfo->groupbyExpr.numOfGroupCols; - } - - // primary timestamp column is involved in final result - if (pQueryInfo->interval.interval != 0 || pQueryInfo->orderProjectQuery) { - numOfGroupByCols++; - } - - int32_t *orderColIndexList = (int32_t *)calloc(numOfGroupByCols, sizeof(int32_t)); - if (orderColIndexList == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - if (numOfGroupByCols > 0) { - - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { - int32_t numOfInternalOutput = (int32_t) tscNumOfExprs(pQueryInfo); - - // the last "pQueryInfo->groupbyExpr.numOfGroupCols" columns are order-by columns - for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i); - for(int32_t j = 0; j < numOfInternalOutput; ++j) { - SExprInfo* pExprInfo = tscExprGet(pQueryInfo, j); - - int32_t functionId = pExprInfo->base.functionId; - if (pColIndex->colId == pExprInfo->base.colInfo.colId && (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAG)) { - orderColIndexList[i] = j; - break; - } - } - } - - if (pQueryInfo->interval.interval != 0) { - // the first column is the timestamp, handles queries like "interval(10m) group by tags" - orderColIndexList[numOfGroupByCols - 1] = PRIMARYKEY_TIMESTAMP_COL_INDEX; //TODO ??? - } - } else { - /* - * 1. the orderby ts asc/desc projection query for the super table - * 2. interval query without groupby clause - */ - if (pQueryInfo->interval.interval != 0) { - orderColIndexList[0] = PRIMARYKEY_TIMESTAMP_COL_INDEX; - } else { - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SExprInfo *pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - orderColIndexList[0] = i; - } - } - } - - assert(pQueryInfo->order.orderColId == PRIMARYKEY_TIMESTAMP_COL_INDEX); - } - } - - *pOrderDesc = tOrderDesCreate(orderColIndexList, numOfGroupByCols, pModel, pQueryInfo->order.order); - tfree(orderColIndexList); - - if (*pOrderDesc == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } else { - return TSDB_CODE_SUCCESS; - } -} - -int32_t tscCreateGlobalMergerEnv(SQueryInfo *pQueryInfo, tExtMemBuffer ***pMemBuffer, int32_t numOfSub, - tOrderDescriptor **pOrderDesc, uint32_t nBufferSizes, int64_t id) { - SSchema *pSchema = NULL; - SColumnModel *pModel = NULL; - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * numOfSub); - if (*pMemBuffer == NULL) { - tscError("0x%"PRIx64" failed to allocate memory", id); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - size_t size = tscNumOfExprs(pQueryInfo); - - pSchema = (SSchema *)calloc(1, sizeof(SSchema) * size); - if (pSchema == NULL) { - tscError("0x%"PRIx64" failed to allocate memory", id); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int32_t rlen = 0; - for (int32_t i = 0; i < size; ++i) { - SExprInfo *pExpr = tscExprGet(pQueryInfo, i); - - pSchema[i].bytes = pExpr->base.resBytes; - pSchema[i].type = (int8_t)pExpr->base.resType; - tstrncpy(pSchema[i].name, pExpr->base.aliasName, tListLen(pSchema[i].name)); - - rlen += pExpr->base.resBytes; - } - - int32_t capacity = 0; - if (rlen != 0) { - capacity = nBufferSizes / rlen; - } - - pModel = createColumnModel(pSchema, (int32_t)size, capacity); - tfree(pSchema); - if (pModel == NULL){ - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int32_t pg = DEFAULT_PAGE_SIZE; - int32_t overhead = sizeof(tFilePage); - while((pg - overhead) < pModel->rowSize * 2) { - pg *= 2; - } - - assert(numOfSub <= pTableMetaInfo->vgroupList->numOfVgroups); - for (int32_t i = 0; i < numOfSub; ++i) { - (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pg, pModel); - (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; - } - - if (createOrderDescriptor(pOrderDesc, pQueryInfo, pModel) != TSDB_CODE_SUCCESS) { - tfree(pModel); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - return TSDB_CODE_SUCCESS; -} - -/** - * @param pMemBuffer - * @param pDesc - * @param numOfVnodes - */ -void tscDestroyGlobalMergerEnv(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDesc, int32_t numOfVnodes) { - tOrderDescDestroy(pDesc); - for (int32_t i = 0; i < numOfVnodes; ++i) { - pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]); - } - - tfree(pMemBuffer); -} - -/** - * - * @param pMerger - * @param pOneInterDataSrc - * @param treeList - * @return the number of remain input source. if ret == 0, all data has been handled - */ -int32_t loadNewDataFromDiskFor(SGlobalMerger *pMerger, SLocalDataSource *pOneInterDataSrc, - bool *needAdjustLoserTree) { - pOneInterDataSrc->rowIdx = 0; - pOneInterDataSrc->pageId += 1; - - if ((uint32_t)pOneInterDataSrc->pageId < - pOneInterDataSrc->pMemBuffer->fileMeta.flushoutData.pFlushoutInfo[pOneInterDataSrc->flushoutIdx].numOfPages) { - tExtMemBufferLoadData(pOneInterDataSrc->pMemBuffer, &(pOneInterDataSrc->filePage), pOneInterDataSrc->flushoutIdx, - pOneInterDataSrc->pageId); - -#if defined(_DEBUG_VIEW) - printf("new page load to buffer\n"); - tColModelDisplay(pOneInterDataSrc->pMemBuffer->pColumnModel, pOneInterDataSrc->filePage.data, - pOneInterDataSrc->filePage.num, pOneInterDataSrc->pMemBuffer->pColumnModel->capacity); -#endif - *needAdjustLoserTree = true; - } else { - pMerger->numOfCompleted += 1; - - pOneInterDataSrc->rowIdx = -1; - pOneInterDataSrc->pageId = -1; - *needAdjustLoserTree = true; - } - - return pMerger->numOfBuffer; -} - -void adjustLoserTreeFromNewData(SGlobalMerger *pMerger, SLocalDataSource *pOneInterDataSrc, - SLoserTreeInfo *pTree) { - /* - * load a new data page into memory for intermediate dataset source, - * since it's last record in buffer has been chosen to be processed, as the winner of loser-tree - */ - bool needToAdjust = true; - if (pOneInterDataSrc->filePage.num <= pOneInterDataSrc->rowIdx) { - loadNewDataFromDiskFor(pMerger, pOneInterDataSrc, &needToAdjust); - } - - /* - * adjust loser tree otherwise, according to new candidate data - * if the loser tree is rebuild completed, we do not need to adjust - */ - if (needToAdjust) { - int32_t leafNodeIdx = pTree->pNode[0].index + pMerger->numOfBuffer; - -#ifdef _DEBUG_VIEW - printf("before adjust:\t"); - tLoserTreeDisplay(pTree); -#endif - - tLoserTreeAdjust(pTree, leafNodeIdx); - -#ifdef _DEBUG_VIEW - printf("\nafter adjust:\t"); - tLoserTreeDisplay(pTree); - printf("\n"); -#endif - } -} - -//TODO it is not ordered, fix it -static void savePrevOrderColumns(char** prevRow, SArray* pColumnList, SSDataBlock* pBlock, int32_t rowIndex, bool* hasPrev) { - int32_t size = (int32_t) taosArrayGetSize(pColumnList); - - for(int32_t i = 0; i < size; ++i) { - SColIndex* index = taosArrayGet(pColumnList, i); - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, index->colIndex); - assert(index->colId == pColInfo->info.colId); - - memcpy(prevRow[i], pColInfo->pData + pColInfo->info.bytes * rowIndex, pColInfo->info.bytes); - } - - (*hasPrev) = true; -} - -// tsdb_func_tag function only produce one row of result. Therefore, we need to copy the -// output value to multiple rows -static void setTagValueForMultipleRows(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t numOfRows) { - if (numOfRows <= 1) { - return; - } - - for (int32_t k = 0; k < numOfOutput; ++k) { - if (pCtx[k].functionId != TSDB_FUNC_TAG) { - continue; - } - - char* src = pCtx[k].pOutput; - char* dst = pCtx[k].pOutput + pCtx[k].outputBytes; - - // Let's start from the second row, as the first row has result value already. - for (int32_t i = 1; i < numOfRows; ++i) { - memcpy(dst, src, (size_t)pCtx[k].outputBytes); - dst += pCtx[k].outputBytes; - } - } -} - -static void doMergeResultImpl(SMultiwayMergeInfo* pInfo, SQLFunctionCtx *pCtx, int32_t numOfExpr, int32_t rowIndex, char** pDataPtr) { - for (int32_t j = 0; j < numOfExpr; ++j) { - pCtx[j].pInput = pDataPtr[j] + pCtx[j].inputBytes * rowIndex; - } - - for (int32_t j = 0; j < numOfExpr; ++j) { - int32_t functionId = pCtx[j].functionId; - if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { - continue; - } - - if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); - doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE); - } else { - aAggs[functionId].mergeFunc(&pCtx[j]); - } - } -} - -static void doFinalizeResultImpl(SMultiwayMergeInfo* pInfo, SQLFunctionCtx *pCtx, int32_t numOfExpr) { - for(int32_t j = 0; j < numOfExpr; ++j) { - int32_t functionId = pCtx[j].functionId; - if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { - continue; - } - - if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); - doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); - } else { - aAggs[functionId].xFinalize(&pCtx[j]); - } - } -} - -static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock* pBlock) { - SMultiwayMergeInfo* pInfo = pOperator->info; - SQLFunctionCtx* pCtx = pInfo->binfo.pCtx; - - char** addrPtr = calloc(pBlock->info.numOfCols, POINTER_BYTES); - for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - addrPtr[i] = pCtx[i].pInput; - pCtx[i].size = 1; - } - - for(int32_t i = 0; i < pBlock->info.rows; ++i) { - if (pInfo->hasPrev) { - if (needToMerge(pBlock, pInfo->orderColumnList, i, pInfo->prevRow)) { - doMergeResultImpl(pInfo, pCtx, numOfExpr, i, addrPtr); - } else { - doFinalizeResultImpl(pInfo, pCtx, numOfExpr); - - int32_t numOfRows = getNumOfResult(pOperator->pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput); - setTagValueForMultipleRows(pCtx, pOperator->numOfOutput, numOfRows); - - pInfo->binfo.pRes->info.rows += numOfRows; - - for(int32_t j = 0; j < numOfExpr; ++j) { - pCtx[j].pOutput += (pCtx[j].outputBytes * numOfRows); - if (pCtx[j].functionId == TSDB_FUNC_TOP || pCtx[j].functionId == TSDB_FUNC_BOTTOM) { - if(j > 0) pCtx[j].ptsOutputBuf = pCtx[j - 1].pOutput; - } - } - - for(int32_t j = 0; j < numOfExpr; ++j) { - if (pCtx[j].functionId < 0) { - continue; - } - - aAggs[pCtx[j].functionId].init(&pCtx[j], pCtx[j].resultInfo); - } - - doMergeResultImpl(pInfo, pCtx, numOfExpr, i, addrPtr); - } - } else { - doMergeResultImpl(pInfo, pCtx, numOfExpr, i, addrPtr); - } - - savePrevOrderColumns(pInfo->prevRow, pInfo->orderColumnList, pBlock, i, &pInfo->hasPrev); - } - - { - for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - pCtx[i].pInput = addrPtr[i]; - } - } - - tfree(addrPtr); -} - -static bool isAllSourcesCompleted(SGlobalMerger *pMerger) { - return (pMerger->numOfBuffer == pMerger->numOfCompleted); -} - -SGlobalMerger* tscInitResObjForLocalQuery(int32_t numOfRes, int32_t rowLen, uint64_t id) { - SGlobalMerger *pMerger = calloc(1, sizeof(SGlobalMerger)); - if (pMerger == NULL) { - tscDebug("0x%"PRIx64" free local reducer finished", id); - return NULL; - } - - /* - * One more byte space is required, since the sprintf function needs one additional space to put '\0' at - * the end of string - */ - size_t size = numOfRes * rowLen + 1; - pMerger->buf = calloc(1, size); - return pMerger; -} - -// todo remove it -int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) { - int32_t maxRowSize = TMAX(rowSize, finalRowSize); - char* pbuf = calloc(1, (size_t)(pOutput->num * maxRowSize)); - - size_t size = tscNumOfFields(pQueryInfo); - SArithmeticSupport arithSup = {0}; - - // todo refactor - arithSup.offset = 0; - arithSup.numOfCols = (int32_t) tscNumOfExprs(pQueryInfo); - arithSup.exprList = pQueryInfo->exprList; - arithSup.data = calloc(arithSup.numOfCols, POINTER_BYTES); - - for(int32_t k = 0; k < arithSup.numOfCols; ++k) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, k); - arithSup.data[k] = (pOutput->data + pOutput->num* pExpr->base.offset); - } - - int32_t offset = 0; - - for (int i = 0; i < size; ++i) { - SInternalField* pSup = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); - - // calculate the result from several other columns - if (pSup->pExpr->pExpr != NULL) { - arithSup.pExprInfo = pSup->pExpr; - arithmeticTreeTraverse(arithSup.pExprInfo->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc); - } else { - SExprInfo* pExpr = pSup->pExpr; - memcpy(pbuf + pOutput->num * offset, pExpr->base.offset * pOutput->num + pOutput->data, (size_t)(pExpr->base.resBytes * pOutput->num)); - } - - offset += pSup->field.bytes; - } - - memcpy(pOutput->data, pbuf, (size_t)(pOutput->num * offset)); - - tfree(pbuf); - tfree(arithSup.data); - - return offset; -} - -static void appendOneRowToDataBlock(SSDataBlock *pBlock, char *buf, SColumnModel *pModel, int32_t rowIndex, - int32_t maxRows) { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - char* p = pColInfo->pData + pBlock->info.rows * pColInfo->info.bytes; - - char *src = COLMODEL_GET_VAL(buf, pModel, rowIndex, i); - memmove(p, src, pColInfo->info.bytes); - } - - pBlock->info.rows += 1; -} - -SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SMultiwayMergeInfo *pInfo = pOperator->info; - - SGlobalMerger *pMerger = pInfo->pMerge; - SLoserTreeInfo *pTree = pMerger->pLoserTree; - - pInfo->binfo.pRes->info.rows = 0; - - while(1) { - if (isAllSourcesCompleted(pMerger)) { - break; - } - -#ifdef _DEBUG_VIEW - printf("chosen data in pTree[0] = %d\n", pTree->pNode[0].index); -#endif - - assert((pTree->pNode[0].index < pMerger->numOfBuffer) && (pTree->pNode[0].index >= 0)); - - // chosen from loser tree - SLocalDataSource *pOneDataSrc = pMerger->pLocalDataSrc[pTree->pNode[0].index]; - bool sameGroup = true; - if (pInfo->hasPrev) { - - // todo refactor extract method - int32_t numOfCols = (int32_t)taosArrayGetSize(pInfo->orderColumnList); - - // if this row belongs to current result set group - for (int32_t i = 0; i < numOfCols; ++i) { - SColIndex * pIndex = taosArrayGet(pInfo->orderColumnList, i); - SColumnInfoData *pColInfo = taosArrayGet(pInfo->binfo.pRes->pDataBlock, pIndex->colIndex); - - char *newRow = COLMODEL_GET_VAL(pOneDataSrc->filePage.data, pOneDataSrc->pMemBuffer->pColumnModel, - pOneDataSrc->rowIdx, pIndex->colIndex); - - char *data = pInfo->prevRow[i]; - int32_t ret = columnValueAscendingComparator(data, newRow, pColInfo->info.type, pColInfo->info.bytes); - if (ret == 0) { - continue; - } else { - sameGroup = false; - *newgroup = true; - break; - } - } - } - - if (!sameGroup || !pInfo->hasPrev) { //save the data - int32_t numOfCols = (int32_t)taosArrayGetSize(pInfo->orderColumnList); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColIndex * pIndex = taosArrayGet(pInfo->orderColumnList, i); - SColumnInfoData *pColInfo = taosArrayGet(pInfo->binfo.pRes->pDataBlock, pIndex->colIndex); - - char *curCol = COLMODEL_GET_VAL(pOneDataSrc->filePage.data, pOneDataSrc->pMemBuffer->pColumnModel, - pOneDataSrc->rowIdx, pIndex->colIndex); - memcpy(pInfo->prevRow[i], curCol, pColInfo->info.bytes); - } - - pInfo->hasPrev = true; - } - - if (!sameGroup && pInfo->binfo.pRes->info.rows > 0) { - return pInfo->binfo.pRes; - } - - appendOneRowToDataBlock(pInfo->binfo.pRes, pOneDataSrc->filePage.data, pOneDataSrc->pMemBuffer->pColumnModel, - pOneDataSrc->rowIdx, pOneDataSrc->pMemBuffer->pColumnModel->capacity); - -#if defined(_DEBUG_VIEW) - printf("chosen row:\t"); - SSrcColumnInfo colInfo[256] = {0}; - tscGetSrcColumnInfo(colInfo, pQueryInfo); - - tColModelDisplayEx(pModel, tmpBuffer->data, tmpBuffer->num, pModel->capacity, colInfo); -#endif - - pOneDataSrc->rowIdx += 1; - adjustLoserTreeFromNewData(pMerger, pOneDataSrc, pTree); - - if (pInfo->binfo.pRes->info.rows >= pInfo->bufCapacity) { - return pInfo->binfo.pRes; - } - } - - pOperator->status = OP_EXEC_DONE; - return (pInfo->binfo.pRes->info.rows > 0)? pInfo->binfo.pRes:NULL; -} - -static bool isSameGroup(SArray* orderColumnList, SSDataBlock* pBlock, char** dataCols) { - int32_t numOfCols = (int32_t) taosArrayGetSize(orderColumnList); - for (int32_t i = 0; i < numOfCols; ++i) { - SColIndex *pIndex = taosArrayGet(orderColumnList, i); - - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, pIndex->colIndex); - assert(pIndex->colId == pColInfo->info.colId); - - char *data = dataCols[i]; - int32_t ret = columnValueAscendingComparator(data, pColInfo->pData, pColInfo->info.type, pColInfo->info.bytes); - if (ret == 0) { - continue; - } else { - return false; - } - } - - return true; -} - -SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SMultiwayMergeInfo *pAggInfo = pOperator->info; - SOperatorInfo *upstream = pOperator->upstream[0]; - - *newgroup = false; - bool handleData = false; - pAggInfo->binfo.pRes->info.rows = 0; - - { - if (pAggInfo->hasDataBlockForNewGroup) { - pAggInfo->hasPrev = false; // now we start from a new group data set. - - // not belongs to the same group, return the result of current group; - setInputDataBlock(pOperator, pAggInfo->binfo.pCtx, pAggInfo->pExistBlock, TSDB_ORDER_ASC); - updateOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity, pAggInfo->pExistBlock->info.rows); - - { // reset output buffer - for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { - SQLFunctionCtx* pCtx = &pAggInfo->binfo.pCtx[j]; - if (pCtx->functionId < 0) { - clearOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity); - continue; - } - - aAggs[pCtx->functionId].init(pCtx, pCtx->resultInfo); - } - } - - doExecuteFinalMerge(pOperator, pOperator->numOfOutput, pAggInfo->pExistBlock); - - savePrevOrderColumns(pAggInfo->currentGroupColData, pAggInfo->groupColumnList, pAggInfo->pExistBlock, 0, - &pAggInfo->hasGroupColData); - pAggInfo->pExistBlock = NULL; - pAggInfo->hasDataBlockForNewGroup = false; - handleData = true; - *newgroup = true; - } - } - - SSDataBlock* pBlock = NULL; - while(1) { - bool prev = *newgroup; - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - if (pBlock == NULL) { - *newgroup = prev; - break; - } - - bool sameGroup = true; - if (pAggInfo->hasGroupColData) { - sameGroup = isSameGroup(pAggInfo->groupColumnList, pBlock, pAggInfo->currentGroupColData); - if (!sameGroup && !pAggInfo->multiGroupResults) { - *newgroup = true; - pAggInfo->hasDataBlockForNewGroup = true; - pAggInfo->pExistBlock = pBlock; - savePrevOrderColumns(pAggInfo->prevRow, pAggInfo->groupColumnList, pBlock, 0, &pAggInfo->hasPrev); - break; - } - } - - // not belongs to the same group, return the result of current group - setInputDataBlock(pOperator, pAggInfo->binfo.pCtx, pBlock, TSDB_ORDER_ASC); - updateOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity, pBlock->info.rows * pAggInfo->resultRowFactor); - - doExecuteFinalMerge(pOperator, pOperator->numOfOutput, pBlock); - savePrevOrderColumns(pAggInfo->currentGroupColData, pAggInfo->groupColumnList, pBlock, 0, &pAggInfo->hasGroupColData); - handleData = true; - } - - if (handleData) { // data in current group is all handled - doFinalizeResultImpl(pAggInfo, pAggInfo->binfo.pCtx, pOperator->numOfOutput); - int32_t numOfRows = getNumOfResult(pOperator->pRuntimeEnv, pAggInfo->binfo.pCtx, pOperator->numOfOutput); - - pAggInfo->binfo.pRes->info.rows += numOfRows; - setTagValueForMultipleRows(pAggInfo->binfo.pCtx, pOperator->numOfOutput, numOfRows); - } - - SSDataBlock* pRes = pAggInfo->binfo.pRes; - { - SColumnInfoData* pInfoData = taosArrayGet(pRes->pDataBlock, 0); - - if (pInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && pRes->info.rows > 0) { - STimeWindow* w = &pRes->info.window; - - w->skey = *(int64_t*)pInfoData->pData; - w->ekey = *(int64_t*)(((char*)pInfoData->pData) + TSDB_KEYSIZE * (pRes->info.rows - 1)); - - if (pOperator->pRuntimeEnv->pQueryAttr->order.order == TSDB_ORDER_DESC) { - TSWAP(w->skey, w->ekey, TSKEY); - assert(w->skey <= w->ekey); - } - } - } - - return (pRes->info.rows != 0)? pRes:NULL; -} - -static void doHandleDataInCurrentGroup(SSLimitOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowIndex) { - if (pInfo->currentOffset > 0) { - pInfo->currentOffset -= 1; - } else { - // discard the data rows in current group - if (pInfo->limit.limit < 0 || (pInfo->limit.limit >= 0 && pInfo->rowsTotal < pInfo->limit.limit)) { - size_t num1 = taosArrayGetSize(pInfo->pRes->pDataBlock); - for (int32_t i = 0; i < num1; ++i) { - SColumnInfoData *pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - SColumnInfoData *pDstInfoData = taosArrayGet(pInfo->pRes->pDataBlock, i); - - SColumnInfo *pColInfo = &pColInfoData->info; - - char *pSrc = rowIndex * pColInfo->bytes + (char *)pColInfoData->pData; - char *pDst = (char *)pDstInfoData->pData + (pInfo->pRes->info.rows * pColInfo->bytes); - - memcpy(pDst, pSrc, pColInfo->bytes); - } - - pInfo->rowsTotal += 1; - pInfo->pRes->info.rows += 1; - } - } -} - -static void ensureOutputBuf(SSLimitOperatorInfo * pInfo, SSDataBlock *pResultBlock, int32_t numOfRows) { - if (pInfo->capacity < pResultBlock->info.rows + numOfRows) { - int32_t total = pResultBlock->info.rows + numOfRows; - - size_t num = taosArrayGetSize(pResultBlock->pDataBlock); - for (int32_t i = 0; i < num; ++i) { - SColumnInfoData *pInfoData = taosArrayGet(pResultBlock->pDataBlock, i); - - char *tmp = realloc(pInfoData->pData, total * pInfoData->info.bytes); - if (tmp != NULL) { - pInfoData->pData = tmp; - } else { - // todo handle the malloc failure - } - - pInfo->capacity = total; - pInfo->threshold = (int64_t)(total * 0.8); - } - } -} - -enum { - BLOCK_NEW_GROUP = 1, - BLOCK_NO_GROUP = 2, - BLOCK_SAME_GROUP = 3, -}; - -static int32_t doSlimitImpl(SOperatorInfo* pOperator, SSLimitOperatorInfo* pInfo, SSDataBlock* pBlock) { - int32_t rowIndex = 0; - - while (rowIndex < pBlock->info.rows) { - int32_t numOfCols = (int32_t)taosArrayGetSize(pInfo->orderColumnList); - - bool samegroup = true; - if (pInfo->hasPrev) { - for (int32_t i = 0; i < numOfCols; ++i) { - SColIndex *pIndex = taosArrayGet(pInfo->orderColumnList, i); - SColumnInfoData *pColInfoData = taosArrayGet(pBlock->pDataBlock, pIndex->colIndex); - - SColumnInfo *pColInfo = &pColInfoData->info; - - char *d = rowIndex * pColInfo->bytes + (char *)pColInfoData->pData; - int32_t ret = columnValueAscendingComparator(pInfo->prevRow[i], d, pColInfo->type, pColInfo->bytes); - if (ret != 0) { // it is a new group - samegroup = false; - break; - } - } - } - - if (!samegroup || !pInfo->hasPrev) { - pInfo->ignoreCurrentGroup = false; - savePrevOrderColumns(pInfo->prevRow, pInfo->orderColumnList, pBlock, rowIndex, &pInfo->hasPrev); - - pInfo->currentOffset = pInfo->limit.offset; // reset the offset value for a new group - pInfo->rowsTotal = 0; - - if (pInfo->currentGroupOffset > 0) { - pInfo->ignoreCurrentGroup = true; - pInfo->currentGroupOffset -= 1; // now we are in the next group data - rowIndex += 1; - continue; - } - - // A new group has arrived according to the result rows, and the group limitation has already reached. - // Let's jump out of current loop and return immediately. - if (pInfo->slimit.limit >= 0 && pInfo->groupTotal >= pInfo->slimit.limit) { - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); - pOperator->status = OP_EXEC_DONE; - return BLOCK_NO_GROUP; - } - - pInfo->groupTotal += 1; - - // data in current group not allowed, return if current result does not belong to the previous group.And there - // are results exists in current SSDataBlock - if (!pInfo->multigroupResult && !samegroup && pInfo->pRes->info.rows > 0) { - return BLOCK_NEW_GROUP; - } - - doHandleDataInCurrentGroup(pInfo, pBlock, rowIndex); - - } else { // handle the offset in the same group - // All the data in current group needs to be discarded, due to the limit parameter in the SQL statement - if (pInfo->ignoreCurrentGroup) { - rowIndex += 1; - continue; - } - - doHandleDataInCurrentGroup(pInfo, pBlock, rowIndex); - } - - rowIndex += 1; - } - - return BLOCK_SAME_GROUP; -} - -SSDataBlock* doSLimit(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo *)param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SSLimitOperatorInfo *pInfo = pOperator->info; - pInfo->pRes->info.rows = 0; - - if (pInfo->pPrevBlock != NULL) { - ensureOutputBuf(pInfo, pInfo->pRes, pInfo->pPrevBlock->info.rows); - int32_t ret = doSlimitImpl(pOperator, pInfo, pInfo->pPrevBlock); - assert(ret != BLOCK_NEW_GROUP); - - pInfo->pPrevBlock = NULL; - } - - assert(pInfo->currentGroupOffset >= 0); - - while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock *pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; - } - - ensureOutputBuf(pInfo, pInfo->pRes, pBlock->info.rows); - int32_t ret = doSlimitImpl(pOperator, pInfo, pBlock); - if (ret == BLOCK_NEW_GROUP) { - pInfo->pPrevBlock = pBlock; - return pInfo->pRes; - } - - if (pOperator->status == OP_EXEC_DONE) { - return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; - } - - // now the number of rows in current group is enough, let's return to the invoke function - if (pInfo->pRes->info.rows > pInfo->threshold) { - return pInfo->pRes; - } - } -} diff --git a/2.0/src/client/src/tscLocal.c b/2.0/src/client/src/tscLocal.c deleted file mode 100644 index 31e60ff2790ff260454a1621e476bd0e5db481b8..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscLocal.c +++ /dev/null @@ -1,955 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tmsg.h" - -#include "../../../include/client/taos.h" -#include "qTableMeta.h" -#include "taosdef.h" -#include "tname.h" -#include "tscLog.h" -#include "tscSubquery.h" -#include "tscUtil.h" -#include "tsclient.h" - -#define STR_NOCASE_EQUAL(str1, len1, str2, len2) ((len1 == len2) && 0 == strncasecmp(str1, str2, len1)) - -typedef enum BuildType { - SCREATE_BUILD_TABLE = 1, - SCREATE_BUILD_DB = 2, -} BuildType; - -typedef enum Stage { - SCREATE_CALLBACK_QUERY = 1, - SCREATE_CALLBACK_RETRIEVE = 2, -} Stage; - -// support 'show create table' -typedef struct SCreateBuilder { - char sTableName[TSDB_TABLE_FNAME_LEN]; - char buf[TSDB_TABLE_FNAME_LEN]; - SSqlObj *pParentSql; - SSqlObj *pInterSql; - int32_t (*fp)(void *para, char* result); - Stage callStage; -} SCreateBuilder; - -static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength); - -static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { - SSqlRes *pRes = &pSql->res; - - // one column for each row - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pMeta = pTableMetaInfo->pTableMeta; - - /* - * tagValueCnt is to denote the number of tags columns for meter, not metric. and is to show the column data. - * for meter, which is created according to metric, the value of tagValueCnt is not 0, and the numOfTags must be 0. - * for metric, the value of tagValueCnt must be 0, but the numOfTags is not 0 - */ - - int32_t numOfRows = tscGetNumOfColumns(pMeta); - int32_t totalNumOfRows = numOfRows + tscGetNumOfTags(pMeta); - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - numOfRows = numOfRows + tscGetNumOfTags(pMeta); - } - - pSql->res.pMerger = tscInitResObjForLocalQuery(totalNumOfRows, rowLen, pSql->self); - tscInitResForMerge(&pSql->res); - - SSchema *pSchema = tscGetTableSchema(pMeta); - - for (int32_t i = 0; i < numOfRows; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); - char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i; - STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, pField->bytes); - - char *type = tDataTypes[pSchema[i].type].name; - - pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); - dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i; - - STR_WITH_MAXSIZE_TO_VARSTR(dst, type, pField->bytes); - - int32_t bytes = pSchema[i].bytes; - if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - bytes -= VARSTR_HEADER_SIZE; - - if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes / TSDB_NCHAR_SIZE; - } - } - - pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2); - *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes; - - pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3); - if (i >= tscGetNumOfColumns(pMeta) && tscGetNumOfTags(pMeta) != 0) { - char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i; - const char *src = "TAG"; - STR_WITH_MAXSIZE_TO_VARSTR(output, src, pField->bytes); - } - } - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return 0; - } - - // the following is handle display tags for table created according to super table - for (int32_t i = numOfRows; i < totalNumOfRows; ++i) { - // field name - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); - char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i; - STR_WITH_MAXSIZE_TO_VARSTR(output, pSchema[i].name, pField->bytes); - - // type name - pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); - char *type = tDataTypes[pSchema[i].type].name; - - output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i; - STR_WITH_MAXSIZE_TO_VARSTR(output, type, pField->bytes); - - // type length - int32_t bytes = pSchema[i].bytes; - pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2); - if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - bytes -= VARSTR_HEADER_SIZE; - - if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes / TSDB_NCHAR_SIZE; - } - } - - *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes; - - // tag value - pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3); - char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i; - const char *src = "TAG"; - STR_WITH_MAXSIZE_TO_VARSTR(target, src, pField->bytes); - } - - return 0; -} - -static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, int32_t typeColLength, - int32_t noteColLength) { - int32_t rowLen = 0; - SColumnIndex index = {0}; - - pSql->cmd.numOfCols = numOfCols; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - pQueryInfo->order.order = TSDB_ORDER_ASC; - - TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE}; - tstrncpy(f.name, "Field", sizeof(f.name)); - - SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, - (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, -1000, (TSDB_COL_NAME_LEN - 1), false); - - rowLen += ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE); - - f.bytes = (int16_t)(typeColLength + VARSTR_HEADER_SIZE); - f.type = TSDB_DATA_TYPE_BINARY; - tstrncpy(f.name, "Type", sizeof(f.name)); - - pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(typeColLength + VARSTR_HEADER_SIZE), - -1000, typeColLength, false); - - rowLen += typeColLength + VARSTR_HEADER_SIZE; - - f.bytes = sizeof(int32_t); - f.type = TSDB_DATA_TYPE_INT; - tstrncpy(f.name, "Length", sizeof(f.name)); - - pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), - -1000, sizeof(int32_t), false); - - rowLen += sizeof(int32_t); - - f.bytes = (int16_t)(noteColLength + VARSTR_HEADER_SIZE); - f.type = TSDB_DATA_TYPE_BINARY; - tstrncpy(f.name, "Note", sizeof(f.name)); - - pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(noteColLength + VARSTR_HEADER_SIZE), - -1000, noteColLength, false); - - rowLen += noteColLength + VARSTR_HEADER_SIZE; - return rowLen; -} - -static int32_t tscProcessDescribeTable(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - assert(tscGetMetaInfo(pQueryInfo, 0)->pTableMeta != NULL); - - const int32_t NUM_OF_DESC_TABLE_COLUMNS = 4; - const int32_t TYPE_COLUMN_LENGTH = 20; - const int32_t NOTE_COLUMN_MIN_LENGTH = 8; - - int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH; - - int32_t rowLen = tscBuildTableSchemaResultFields(pSql, NUM_OF_DESC_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, noteFieldLen); - tscFieldInfoUpdateOffset(pQueryInfo); - return tscSetValueToResObj(pSql, rowLen); -} -static int32_t tscGetNthFieldResult(TAOS_ROW row, TAOS_FIELD* fields, int *lengths, int idx, char *result) { - const char *val = (const char*)row[idx]; - if (val == NULL) { - sprintf(result, "%s", TSDB_DATA_NULL_STR); - return -1; - } - uint8_t type = fields[idx].type; - int32_t length = lengths[idx]; - - switch (type) { - case TSDB_DATA_TYPE_BOOL: - sprintf(result, "%s", ((((int32_t)(*((char *)val))) == 1) ? "true" : "false")); - break; - case TSDB_DATA_TYPE_TINYINT: - sprintf(result, "%d", *((int8_t *)val)); - break; - case TSDB_DATA_TYPE_SMALLINT: - sprintf(result, "%d", *((int16_t *)val)); - break; - case TSDB_DATA_TYPE_INT: - sprintf(result, "%d", *((int32_t *)val)); - break; - case TSDB_DATA_TYPE_BIGINT: - sprintf(result, "%"PRId64, *((int64_t *)val)); - break; - case TSDB_DATA_TYPE_FLOAT: - sprintf(result, "%f", GET_FLOAT_VAL(val)); - break; - case TSDB_DATA_TYPE_DOUBLE: - sprintf(result, "%f", GET_DOUBLE_VAL(val)); - break; - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_BINARY: - memcpy(result, val, length); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - ///formatTimestamp(buf, *(int64_t*)val, TSDB_TIME_PRECISION_MICRO); - //memcpy(result, val, strlen(buf)); - sprintf(result, "%"PRId64, *((int64_t *)val)); - break; - default: - break; - } - return 0; -} - -void tscSCreateCallBack(void *param, TAOS_RES *tres, int code) { - if (param == NULL || tres == NULL) { - return; - } - SCreateBuilder *builder = (SCreateBuilder *)(param); - SSqlObj *pParentSql = builder->pParentSql; - SSqlObj *pSql = (SSqlObj *)tres; - - SSqlRes *pRes = &pParentSql->res; - pRes->code = taos_errno(pSql); - if (pRes->code != TSDB_CODE_SUCCESS) { - taos_free_result(pSql); - free(builder); - tscAsyncResultOnError(pParentSql); - return; - } - - if (builder->callStage == SCREATE_CALLBACK_QUERY) { - taos_fetch_rows_a(tres, tscSCreateCallBack, param); - builder->callStage = SCREATE_CALLBACK_RETRIEVE; - } else { - char *result = calloc(1, TSDB_MAX_BINARY_LEN); - pRes->code = builder->fp(builder, result); - - taos_free_result(pSql); - free(builder); - free(result); - - if (pRes->code == TSDB_CODE_SUCCESS) { - (*pParentSql->fp)(pParentSql->param, pParentSql, code); - } else { - tscAsyncResultOnError(pParentSql); - } - } -} - -TAOS_ROW tscFetchRow(void *param) { - SCreateBuilder *builder = (SCreateBuilder *)param; - if (builder == NULL) { - return NULL; - } - SSqlObj *pSql = builder->pInterSql; - if (pSql == NULL || pSql->signature != pSql) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pRes->qId == 0 || - pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || - pCmd->command == TSDB_SQL_INSERT) { - return NULL; - } - - // set the sql object owner - tscSetSqlOwner(pSql); - - // current data set are exhausted, fetch more data from node - if (pRes->row >= pRes->numOfRows && (pRes->completed != true || hasMoreVnodesToTry(pSql) || hasMoreClauseToTry(pSql)) && - (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || - pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE || - pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE || - pCmd->command == TSDB_SQL_FETCH || - pCmd->command == TSDB_SQL_SHOW || - pCmd->command == TSDB_SQL_SHOW_CREATE_TABLE || - pCmd->command == TSDB_SQL_SHOW_CREATE_STABLE || - pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE || - pCmd->command == TSDB_SQL_SELECT || - pCmd->command == TSDB_SQL_DESCRIBE_TABLE || - pCmd->command == TSDB_SQL_SERV_STATUS || - pCmd->command == TSDB_SQL_CURRENT_DB || - pCmd->command == TSDB_SQL_SERV_VERSION || - pCmd->command == TSDB_SQL_CLI_VERSION || - pCmd->command == TSDB_SQL_CURRENT_USER )) { - taos_fetch_rows_a(pSql, tscSCreateCallBack, param); - return NULL; - } - - void* data = doSetResultRowData(pSql); - - tscClearSqlOwner(pSql); - return data; -} -static int32_t tscGetTableTagValue(SCreateBuilder *builder, char *result) { - TAOS_ROW row = tscFetchRow(builder); - SSqlObj* pSql = builder->pInterSql; - - if (row == NULL) { - return TSDB_CODE_TSC_INVALID_TABLE_NAME; - } - - int32_t* lengths = taos_fetch_lengths(pSql); - int num_fields = taos_num_fields(pSql); - TAOS_FIELD *fields = taos_fetch_fields(pSql); - - char buf[TSDB_COL_NAME_LEN + 16]; - for (int i = 0; i < num_fields; i++) { - memset(buf, 0, sizeof(buf)); - int32_t ret = tscGetNthFieldResult(row, fields, lengths, i, buf); - - if (i == 0) { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s", "("); - } - if ((fields[i].type == TSDB_DATA_TYPE_NCHAR - || fields[i].type == TSDB_DATA_TYPE_BINARY - || fields[i].type == TSDB_DATA_TYPE_TIMESTAMP) && 0 == ret) { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "\"%s\",", buf); - } else { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s,", buf); - } - if (i == num_fields - 1) { - sprintf(result + strlen(result) - 1, "%s", ")"); - } - } - - if (0 == strlen(result)) { - return TSDB_CODE_TSC_INVALID_TABLE_NAME; - } - return TSDB_CODE_SUCCESS; -} - - -// build 'show create table/database' result fields -static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const char *ddl) { - int32_t rowLen = 0; - int16_t ddlLen = (int16_t)strlen(ddl); - SColumnIndex index = {0}; - pSql->cmd.numOfCols = 2; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - pQueryInfo->order.order = TSDB_ORDER_ASC; - - TAOS_FIELD f; - if (type == SCREATE_BUILD_TABLE) { - f.type = TSDB_DATA_TYPE_BINARY; - f.bytes = (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE; - tstrncpy(f.name, "Table", sizeof(f.name)); - } else { - f.type = TSDB_DATA_TYPE_BINARY; - f.bytes = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE; - tstrncpy(f.name, "Database", sizeof(f.name)); - } - - SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, f.bytes, -1000, f.bytes - VARSTR_HEADER_SIZE, false); - - rowLen += f.bytes; - - f.bytes = (int16_t)(ddlLen + VARSTR_HEADER_SIZE); - f.type = TSDB_DATA_TYPE_BINARY; - if (type == SCREATE_BUILD_TABLE) { - tstrncpy(f.name, "Create Table", sizeof(f.name)); - } else { - tstrncpy(f.name, "Create Database", sizeof(f.name)); - } - - pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, - (int16_t)(ddlLen + VARSTR_HEADER_SIZE), -1000, ddlLen, false); - - rowLen += ddlLen + VARSTR_HEADER_SIZE; - - return rowLen; -} -static int32_t tscSCreateSetValueToResObj(SSqlObj *pSql, int32_t rowLen, const char *tableName, const char *ddl) { - SSqlRes *pRes = &pSql->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - int32_t numOfRows = 1; - if (strlen(ddl) == 0) { - - } - pSql->res.pMerger = tscInitResObjForLocalQuery(numOfRows, rowLen, pSql->self); - tscInitResForMerge(&pSql->res); - - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); - char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(dst, tableName, pField->bytes); - - pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); - dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(dst, ddl, pField->bytes); - return 0; -} -static int32_t tscSCreateBuildResult(SSqlObj *pSql, BuildType type, const char *str, const char *result) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - int32_t rowLen = tscSCreateBuildResultFields(pSql, type, result); - - tscFieldInfoUpdateOffset(pQueryInfo); - return tscSCreateSetValueToResObj(pSql, rowLen, str, result); -} -int32_t tscRebuildCreateTableStatement(void *param,char *result) { - SCreateBuilder *builder = (SCreateBuilder *)param; - int32_t code = TSDB_CODE_SUCCESS; - - char *buf = calloc(1,TSDB_MAX_BINARY_LEN); - if (buf == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - code = tscGetTableTagValue(builder, buf); - if (code == TSDB_CODE_SUCCESS) { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "CREATE TABLE %s USING %s TAGS %s", builder->buf, builder->sTableName, buf); - code = tscSCreateBuildResult(builder->pParentSql, SCREATE_BUILD_TABLE, builder->buf, result); - } - free(buf); - return code; -} - -static int32_t tscGetDBInfo(SCreateBuilder *builder, char *result) { - TAOS_ROW row = tscFetchRow(builder); - if (row == NULL) { - return TSDB_CODE_TSC_DB_NOT_SELECTED; - } - const char *showColumns[] = {"REPLICA", "QUORUM", "DAYS", "KEEP", "BLOCKS", NULL}; - - SSqlObj *pSql = builder->pInterSql; - TAOS_FIELD *fields = taos_fetch_fields(pSql); - int num_fields = taos_num_fields(pSql); - - char buf[TSDB_DB_NAME_LEN + 64] = {0}; - do { - memset(buf, 0, sizeof(buf)); - int32_t* lengths = taos_fetch_lengths(pSql); - int32_t ret = tscGetNthFieldResult(row, fields, lengths, 0, buf); - if (0 == ret && STR_NOCASE_EQUAL(buf, strlen(buf), builder->buf, strlen(builder->buf))) { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "CREATE DATABASE %s", buf); - for (int i = 1; i < num_fields; i++) { - for (int j = 0; showColumns[j] != NULL; j++) { - if (STR_NOCASE_EQUAL(fields[i].name, strlen(fields[i].name), showColumns[j], strlen(showColumns[j]))) { - memset(buf, 0, sizeof(buf)); - ret = tscGetNthFieldResult(row, fields, lengths, i, buf); - if (ret == 0) { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), " %s %s", showColumns[j], buf); - } - } - } - } - break; - } - - row = tscFetchRow(builder); - } while (row != NULL); - - if (0 == strlen(result)) { - return TSDB_CODE_TSC_DB_NOT_SELECTED; - } - - return TSDB_CODE_SUCCESS; -} -int32_t tscRebuildCreateDBStatement(void *param,char *result) { - SCreateBuilder *builder = (SCreateBuilder *)param; - int32_t code = TSDB_CODE_SUCCESS; - - char *buf = calloc(1, TSDB_MAX_BINARY_LEN); - if (buf == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - code = tscGetDBInfo(param, buf); - if (code == TSDB_CODE_SUCCESS) { - code = tscSCreateBuildResult(builder->pParentSql, SCREATE_BUILD_DB, builder->buf, buf); - } - free(buf); - return code; -} - -static int32_t tscGetTableTagColumnName(SSqlObj *pSql, char **result) { - char *buf = (char *)malloc(TSDB_MAX_BINARY_LEN); - if (buf == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - buf[0] = 0; - - STableMeta *pMeta = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0)->pTableMeta; - if (pMeta->tableType == TSDB_SUPER_TABLE || pMeta->tableType == TSDB_NORMAL_TABLE || - pMeta->tableType == TSDB_STREAM_TABLE) { - free(buf); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - SSchema *pTagsSchema = tscGetTableTagSchema(pMeta); - int32_t numOfTags = tscGetNumOfTags(pMeta); - for (int32_t i = 0; i < numOfTags; i++) { - if (i != numOfTags - 1) { - snprintf(buf + strlen(buf), TSDB_MAX_BINARY_LEN - strlen(buf), "%s,", pTagsSchema[i].name); - } else { - snprintf(buf + strlen(buf), TSDB_MAX_BINARY_LEN - strlen(buf), "%s", pTagsSchema[i].name); - } - } - - *result = buf; - return TSDB_CODE_SUCCESS; -} -static int32_t tscRebuildDDLForSubTable(SSqlObj *pSql, const char *tableName, char *ddl) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pMeta = pTableMetaInfo->pTableMeta; - - SSqlObj *pInterSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (pInterSql == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateBuilder *param = (SCreateBuilder *)malloc(sizeof(SCreateBuilder)); - if (param == NULL) { - free(pInterSql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - char fullName[TSDB_TABLE_FNAME_LEN * 2] = {0}; - tNameGetDbName(&pTableMetaInfo->name, fullName); - - extractTableName(pMeta->sTableName, param->sTableName); - snprintf(fullName + strlen(fullName), TSDB_TABLE_FNAME_LEN - strlen(fullName), ".%s", param->sTableName); - - strncpy(param->buf, tNameGetTableName(&pTableMetaInfo->name), TSDB_TABLE_NAME_LEN); - - param->pParentSql = pSql; - param->pInterSql = pInterSql; - param->fp = tscRebuildCreateTableStatement; - param->callStage = SCREATE_CALLBACK_QUERY; - - char *query = (char *)calloc(1, TSDB_MAX_BINARY_LEN); - if (query == NULL) { - free(param); - free(pInterSql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - char *columns = NULL; - int32_t code = tscGetTableTagColumnName(pSql, &columns) ; - if (code != TSDB_CODE_SUCCESS) { - free(param); - free(pInterSql); - free(query); - return code; - } - - snprintf(query + strlen(query), TSDB_MAX_BINARY_LEN - strlen(query), "SELECT %s FROM %s WHERE TBNAME IN(\'%s\')", columns, fullName, param->buf); - doAsyncQuery(pSql->pTscObj, pInterSql, tscSCreateCallBack, param, query, strlen(query)); - free(query); - free(columns); - - return TSDB_CODE_TSC_ACTION_IN_PROGRESS; -} - -static int32_t tscRebuildDDLForNormalTable(SSqlObj *pSql, const char *tableName, char *ddl) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pMeta = pTableMetaInfo->pTableMeta; - - int32_t numOfRows = tscGetNumOfColumns(pMeta); - SSchema *pSchema = tscGetTableSchema(pMeta); - - char *result = ddl; - sprintf(result, "create table %s (", tableName); - for (int32_t i = 0; i < numOfRows; ++i) { - uint8_t type = pSchema[i].type; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE; - if (type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes/TSDB_NCHAR_SIZE; - } - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name, tDataTypes[pSchema[i].type].name, bytes); - } else { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[pSchema[i].type].name); - } - } - sprintf(result + strlen(result) - 1, "%s", ")"); - - return TSDB_CODE_SUCCESS; -} -static int32_t tscRebuildDDLForSuperTable(SSqlObj *pSql, const char *tableName, char *ddl) { - char *result = ddl; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pMeta = pTableMetaInfo->pTableMeta; - - int32_t numOfRows = tscGetNumOfColumns(pMeta); - int32_t totalRows = numOfRows + tscGetNumOfTags(pMeta); - SSchema *pSchema = tscGetTableSchema(pMeta); - - sprintf(result, "create table %s (", tableName); - for (int32_t i = 0; i < numOfRows; ++i) { - uint8_t type = pSchema[i].type; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE; - if (type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes/TSDB_NCHAR_SIZE; - } - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result),"%s %s(%d),", pSchema[i].name,tDataTypes[pSchema[i].type].name, bytes); - } else { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[type].name); - } - } - snprintf(result + strlen(result) - 1, TSDB_MAX_BINARY_LEN - strlen(result), "%s %s", ")", "TAGS ("); - - for (int32_t i = numOfRows; i < totalRows; i++) { - uint8_t type = pSchema[i].type; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE; - if (type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes/TSDB_NCHAR_SIZE; - } - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name,tDataTypes[pSchema[i].type].name, bytes); - } else { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[type].name); - } - } - sprintf(result + strlen(result) - 1, "%s", ")"); - - return TSDB_CODE_SUCCESS; -} - -static int32_t tscProcessShowCreateTable(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - assert(pTableMetaInfo->pTableMeta != NULL); - - const char* tableName = tNameGetTableName(&pTableMetaInfo->name); - if (pSql->cmd.command == TSDB_SQL_SHOW_CREATE_STABLE && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - - char *result = (char *)calloc(1, TSDB_MAX_BINARY_LEN); - int32_t code = TSDB_CODE_SUCCESS; - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - code = tscRebuildDDLForSuperTable(pSql, tableName, result); - } else if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - code = tscRebuildDDLForNormalTable(pSql, tableName, result); - } else if (UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) { - code = tscRebuildDDLForSubTable(pSql, tableName, result); - } else { - code = TSDB_CODE_TSC_INVALID_VALUE; - } - - if (code == TSDB_CODE_SUCCESS) { - code = tscSCreateBuildResult(pSql, SCREATE_BUILD_TABLE, tableName, result); - } - free(result); - return code; -} - -static int32_t tscProcessShowCreateDatabase(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SSqlObj *pInterSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (pInterSql == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateBuilder *param = (SCreateBuilder *)calloc(1, sizeof(SCreateBuilder)); - if (param == NULL) { - free(pInterSql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - tNameGetDbName(&pTableMetaInfo->name, param->buf); - - param->pParentSql = pSql; - param->pInterSql = pInterSql; - param->fp = tscRebuildCreateDBStatement; - param->callStage = SCREATE_CALLBACK_QUERY; - - const char *query = "show databases"; - doAsyncQuery(pSql->pTscObj, pInterSql, tscSCreateCallBack, param, query, strlen(query)); - return TSDB_CODE_TSC_ACTION_IN_PROGRESS; -} -static int32_t tscProcessCurrentUser(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); - pExpr->resBytes = TSDB_USER_LEN + TSDB_DATA_TYPE_BINARY; - pExpr->resType = TSDB_DATA_TYPE_BINARY; - - char* vx = calloc(1, pExpr->resBytes); - if (vx == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - size_t size = sizeof(pSql->pTscObj->user); - STR_WITH_MAXSIZE_TO_VARSTR(vx, pSql->pTscObj->user, size); - - tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes); - free(vx); - - return TSDB_CODE_SUCCESS; -} - -static int32_t tscProcessCurrentDB(SSqlObj *pSql) { - char db[TSDB_DB_NAME_LEN] = {0}; - - pthread_mutex_lock(&pSql->pTscObj->mutex); - extractDBName(pSql->pTscObj->db, db); - pthread_mutex_unlock(&pSql->pTscObj->mutex); - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); - pExpr->resType = TSDB_DATA_TYPE_BINARY; - - size_t t = strlen(db); - pExpr->resBytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE; - - char* vx = calloc(1, pExpr->resBytes); - if (vx == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - if (t == 0) { - setVardataNull(vx, TSDB_DATA_TYPE_BINARY); - } else { - STR_WITH_SIZE_TO_VARSTR(vx, db, (VarDataLenT)t); - } - - tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes); - free(vx); - - return TSDB_CODE_SUCCESS; -} - -static int32_t tscProcessServerVer(SSqlObj *pSql) { - const char* v = pSql->pTscObj->sversion; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); - pExpr->resType = TSDB_DATA_TYPE_BINARY; - - size_t t = strlen(v); - pExpr->resBytes = (int16_t)(t + VARSTR_HEADER_SIZE); - - char* vx = calloc(1, pExpr->resBytes); - if (vx == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - - } - - STR_WITH_SIZE_TO_VARSTR(vx, v, (VarDataLenT)t); - tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes); - - free(vx); - return TSDB_CODE_SUCCESS; - -} - -static int32_t tscProcessClientVer(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); - pExpr->resType = TSDB_DATA_TYPE_BINARY; - - size_t t = strlen(version); - pExpr->resBytes = (int16_t)(t + VARSTR_HEADER_SIZE); - - char* v = calloc(1, pExpr->resBytes); - if (v == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - - } - - STR_WITH_SIZE_TO_VARSTR(v, version, (VarDataLenT)t); - tscSetLocalQueryResult(pSql, v, pExpr->aliasName, pExpr->resType, pExpr->resBytes); - - free(v); - return TSDB_CODE_SUCCESS; - -} - -// TODO add test cases. -static int32_t checkForOnlineNode(SSqlObj* pSql) { - int32_t* data = pSql->res.length; - if (data == NULL) { - return TSDB_CODE_SUCCESS; - } - - int32_t total = data[0]; - int32_t online = data[1]; - return (online < total)? TSDB_CODE_RPC_NETWORK_UNAVAIL:TSDB_CODE_SUCCESS; -} - -static int32_t tscProcessServStatus(SSqlObj *pSql) { - STscObj* pObj = pSql->pTscObj; - - SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); - if (pHb != NULL) { - pSql->res.code = pHb->res.code; - } - - if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - taosReleaseRef(tscObjRef, pObj->hbrid); - return pSql->res.code; - } - - if (pHb != NULL) { - pSql->res.code = checkForOnlineNode(pHb); - taosReleaseRef(tscObjRef, pObj->hbrid); - } - - if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - return pSql->res.code; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); - - int32_t val = 1; - tscSetLocalQueryResult(pSql, (char*) &val, pExpr->aliasName, TSDB_DATA_TYPE_INT, sizeof(int32_t)); - return TSDB_CODE_SUCCESS; -} - -void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - pCmd->numOfCols = 1; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - pQueryInfo->order.order = TSDB_ORDER_ASC; - - tscFieldInfoClear(&pQueryInfo->fieldsInfo); - pQueryInfo->fieldsInfo.internalField = taosArrayInit(1, sizeof(SInternalField)); - - TAOS_FIELD f = tscCreateField((int8_t)type, columnName, (int16_t)valueLength); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - - pSql->res.pMerger = tscInitResObjForLocalQuery(1, (int32_t)valueLength, pSql->self); - tscInitResForMerge(&pSql->res); - - SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, 0); - pInfo->pExpr = taosArrayGetP(pQueryInfo->exprList, 0); - - memcpy(pRes->data, val, pInfo->field.bytes); -} - -int tscProcessLocalCmd(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pCmd->command == TSDB_SQL_CFG_LOCAL) { - if (taosCfgDynamicOptions(pCmd->payload)) { - pRes->code = TSDB_CODE_SUCCESS; - } else { - pRes->code = TSDB_CODE_COM_INVALID_CFG_MSG; - } - pRes->numOfRows = 0; - } else if (pCmd->command == TSDB_SQL_DESCRIBE_TABLE) { - pRes->code = (uint8_t)tscProcessDescribeTable(pSql); - } else if (pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - /* - * set the qhandle to be 1 in order to pass the qhandle check, and to call partial release function to - * free allocated resources and remove the SqlObj from sql query linked list - */ - pRes->qId = 0x1; - pRes->numOfRows = 0; - } else if (pCmd->command == TSDB_SQL_SHOW_CREATE_TABLE || pCmd->command == TSDB_SQL_SHOW_CREATE_STABLE) { - pRes->code = tscProcessShowCreateTable(pSql); - } else if (pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE) { - pRes->code = tscProcessShowCreateDatabase(pSql); - } else if (pCmd->command == TSDB_SQL_RESET_CACHE) { - taosHashClear(tscTableMetaMap); - taosCacheEmpty(tscVgroupListBuf); - pRes->code = TSDB_CODE_SUCCESS; - } else if (pCmd->command == TSDB_SQL_SERV_VERSION) { - pRes->code = tscProcessServerVer(pSql); - } else if (pCmd->command == TSDB_SQL_CLI_VERSION) { - pRes->code = tscProcessClientVer(pSql); - } else if (pCmd->command == TSDB_SQL_CURRENT_USER) { - pRes->code = tscProcessCurrentUser(pSql); - } else if (pCmd->command == TSDB_SQL_CURRENT_DB) { - pRes->code = tscProcessCurrentDB(pSql); - } else if (pCmd->command == TSDB_SQL_SERV_STATUS) { - pRes->code = tscProcessServStatus(pSql); - } else { - pRes->code = TSDB_CODE_TSC_INVALID_OPERATION; - tscError("0x%"PRIx64" not support command:%d", pSql->self, pCmd->command); - } - - // keep the code in local variable in order to avoid invalid read in case of async query - - int32_t code = pRes->code; - if (code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, code); - } else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS){ - } else { - tscAsyncResultOnError(pSql); - } - return code; -} diff --git a/2.0/src/client/src/tscParseInsert.c b/2.0/src/client/src/tscParseInsert.c deleted file mode 100644 index a1f5899cee39e3866c6f3efcd636e296d14ad385..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscParseInsert.c +++ /dev/null @@ -1,1803 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE /* See feature_test_macros(7) */ -#define _GNU_SOURCE - -#define _XOPEN_SOURCE - -#include "os.h" - -#include "ttype.h" -#include "hash.h" -#include "tscUtil.h" -#include "qTableMeta.h" -#include "tsclient.h" -#include "ttokendef.h" -#include "taosdef.h" - -#include "tscLog.h" -#include "ttoken.h" - -#include "tdataformat.h" - -enum { - TSDB_USE_SERVER_TS = 0, - TSDB_USE_CLI_TS = 1, -}; - -static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t *numOfRows); -static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo *pColInfo, SSchema *pSchema, - char *str, char **end); -int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, SParsedDataColInfo *pColInfo) { - ASSERT(nRows >= 0 && pColInfo->numOfCols > 0 && (pColInfo->numOfBound <= pColInfo->numOfCols)); - - uint32_t dataLen = TD_MEM_ROW_DATA_HEAD_SIZE + pColInfo->allNullLen; - uint32_t kvLen = TD_MEM_ROW_KV_HEAD_SIZE + pColInfo->numOfBound * sizeof(SColIdx) + pColInfo->boundNullLen; - if (isUtilizeKVRow(kvLen, dataLen)) { - pBuilder->memRowType = SMEM_ROW_KV; - } else { - pBuilder->memRowType = SMEM_ROW_DATA; - } - - return TSDB_CODE_SUCCESS; -} - -int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { - int32_t index = 0; - SStrToken sToken; - int64_t interval; - int64_t useconds = 0; - char * pTokenEnd = *next; - - if (pToken->type == TK_NOW) { - useconds = taosGetTimestamp(timePrec); - } else if (strncmp(pToken->z, "0", 1) == 0 && pToken->n == 1) { - // do nothing - } else if (pToken->type == TK_INTEGER) { - useconds = taosStr2int64(pToken->z); - } else { - // strptime("2001-11-12 18:31:01", "%Y-%m-%d %H:%M:%S", &tm); - if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(error, "invalid timestamp format", pToken->z); - } - - return TSDB_CODE_SUCCESS; - } - - for (int k = pToken->n; pToken->z[k] != '\0'; k++) { - if (pToken->z[k] == ' ' || pToken->z[k] == '\t') continue; - if (pToken->z[k] == ',') { - *next = pTokenEnd; - *time = useconds; - return 0; - } - - break; - } - - /* - * time expression: - * e.g., now+12a, now-5h - */ - SStrToken valueToken; - index = 0; - sToken = tStrGetToken(pTokenEnd, &index, false); - pTokenEnd += index; - - if (sToken.type == TK_MINUS || sToken.type == TK_PLUS) { - index = 0; - valueToken = tStrGetToken(pTokenEnd, &index, false); - pTokenEnd += index; - - if (valueToken.n < 2) { - return tscInvalidOperationMsg(error, "value expected in timestamp", sToken.z); - } - - char unit = 0; - if (parseAbsoluteDuration(valueToken.z, valueToken.n, &interval, &unit, timePrec) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (sToken.type == TK_PLUS) { - useconds += interval; - } else { - useconds = useconds - interval; - } - - *next = pTokenEnd; - } - - *time = useconds; - return TSDB_CODE_SUCCESS; -} - -int32_t tsParseOneColumn(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, bool primaryKey, - int16_t timePrec) { - int64_t iv; - int32_t ret; - char *endptr = NULL; - - if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { - return tscInvalidOperationMsg(msg, "invalid numeric data", pToken->z); - } - - switch (pSchema->type) { - case TSDB_DATA_TYPE_BOOL: { // bool - if (isNullStr(pToken)) { - *((uint8_t *)payload) = TSDB_DATA_BOOL_NULL; - } else { - if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { - if (strncmp(pToken->z, "true", pToken->n) == 0) { - *(uint8_t *)payload = TSDB_TRUE; - } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - *(uint8_t *)payload = TSDB_FALSE; - } else { - return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z); - } - } else if (pToken->type == TK_INTEGER) { - iv = strtoll(pToken->z, NULL, 10); - *(uint8_t *)payload = (int8_t)((iv == 0) ? TSDB_FALSE : TSDB_TRUE); - } else if (pToken->type == TK_FLOAT) { - double dv = strtod(pToken->z, NULL); - *(uint8_t *)payload = (int8_t)((dv == 0) ? TSDB_FALSE : TSDB_TRUE); - } else { - return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z); - } - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: - if (isNullStr(pToken)) { - *((uint8_t *)payload) = TSDB_DATA_TINYINT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid tinyint data", pToken->z); - } else if (!IS_VALID_TINYINT(iv)) { - return tscInvalidOperationMsg(msg, "data overflow", pToken->z); - } - - *((uint8_t *)payload) = (uint8_t)iv; - } - - break; - - case TSDB_DATA_TYPE_UTINYINT: - if (isNullStr(pToken)) { - *((uint8_t *)payload) = TSDB_DATA_UTINYINT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned tinyint data", pToken->z); - } else if (!IS_VALID_UTINYINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned tinyint data overflow", pToken->z); - } - - *((uint8_t *)payload) = (uint8_t)iv; - } - - break; - - case TSDB_DATA_TYPE_SMALLINT: - if (isNullStr(pToken)) { - *((int16_t *)payload) = TSDB_DATA_SMALLINT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid smallint data", pToken->z); - } else if (!IS_VALID_SMALLINT(iv)) { - return tscInvalidOperationMsg(msg, "smallint data overflow", pToken->z); - } - - *((int16_t *)payload) = (int16_t)iv; - } - - break; - - case TSDB_DATA_TYPE_USMALLINT: - if (isNullStr(pToken)) { - *((uint16_t *)payload) = TSDB_DATA_USMALLINT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned smallint data", pToken->z); - } else if (!IS_VALID_USMALLINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned smallint data overflow", pToken->z); - } - - *((uint16_t *)payload) = (uint16_t)iv; - } - - break; - - case TSDB_DATA_TYPE_INT: - if (isNullStr(pToken)) { - *((int32_t *)payload) = TSDB_DATA_INT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid int data", pToken->z); - } else if (!IS_VALID_INT(iv)) { - return tscInvalidOperationMsg(msg, "int data overflow", pToken->z); - } - - *((int32_t *)payload) = (int32_t)iv; - } - - break; - - case TSDB_DATA_TYPE_UINT: - if (isNullStr(pToken)) { - *((uint32_t *)payload) = TSDB_DATA_UINT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned int data", pToken->z); - } else if (!IS_VALID_UINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned int data overflow", pToken->z); - } - - *((uint32_t *)payload) = (uint32_t)iv; - } - - break; - - case TSDB_DATA_TYPE_BIGINT: - if (isNullStr(pToken)) { - *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z); - } - - *((int64_t *)payload) = iv; - } - break; - - case TSDB_DATA_TYPE_UBIGINT: - if (isNullStr(pToken)) { - *((uint64_t *)payload) = TSDB_DATA_UBIGINT_NULL; - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned bigint data", pToken->z); - } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { - return tscInvalidOperationMsg(msg, "unsigned bigint data overflow", pToken->z); - } - - *((uint64_t *)payload) = iv; - } - break; - - case TSDB_DATA_TYPE_FLOAT: - if (isNullStr(pToken)) { - *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; - } else { - double dv; - if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { - return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); - } - - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { - return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); - } - -// *((float *)payload) = (float)dv; - SET_FLOAT_VAL(payload, dv); - } - break; - - case TSDB_DATA_TYPE_DOUBLE: - if (isNullStr(pToken)) { - *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; - } else { - double dv; - if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { - return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); - } - - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { - return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); - } - - *((double *)payload) = dv; - } - break; - - case TSDB_DATA_TYPE_BINARY: - // binary data cannot be null-terminated char string, otherwise the last char of the string is lost - if (pToken->type == TK_NULL) { - setVardataNull(payload, TSDB_DATA_TYPE_BINARY); - } else { // too long values will return invalid sql, not be truncated automatically - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { //todo refactor - return tscInvalidOperationMsg(msg, "string data overflow", pToken->z); - } - - STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); - } - - break; - - case TSDB_DATA_TYPE_NCHAR: - if (pToken->type == TK_NULL) { - setVardataNull(payload, TSDB_DATA_TYPE_NCHAR); - } else { - // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - int32_t output = 0; - if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(payload), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { - char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); - return tscInvalidOperationMsg(msg, buf, pToken->z); - } - - varDataSetLen(payload, output); - } - break; - - case TSDB_DATA_TYPE_TIMESTAMP: { - if (pToken->type == TK_NULL) { - if (primaryKey) { - *((int64_t *)payload) = 0; - } else { - *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; - } - } else { - int64_t temp; - if (tsParseTime(pToken, &temp, str, msg, timePrec) != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z); - } - - *((int64_t *)payload) = temp; - } - - break; - } - } - - return TSDB_CODE_SUCCESS; -} - -/* - * The server time/client time should not be mixed up in one sql string - * Do not employ sort operation is not involved if server time is used. - */ -int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start) { - // once the data block is disordered, we do NOT keep previous timestamp any more - if (!pDataBlocks->ordered) { - return TSDB_CODE_SUCCESS; - } - - TSKEY k = *(TSKEY *)start; - - if (k == INT64_MIN) { - if (pDataBlocks->tsSource == TSDB_USE_CLI_TS) { - return -1; - } else if (pDataBlocks->tsSource == -1) { - pDataBlocks->tsSource = TSDB_USE_SERVER_TS; - } - } else { - if (pDataBlocks->tsSource == TSDB_USE_SERVER_TS) { - return -1; // client time/server time can not be mixed - - } else if (pDataBlocks->tsSource == -1) { - pDataBlocks->tsSource = TSDB_USE_CLI_TS; - } - } - - if (k <= pDataBlocks->prevTS && (pDataBlocks->tsSource == TSDB_USE_CLI_TS)) { - pDataBlocks->ordered = false; - tscWarn("NOT ordered input timestamp"); - } - - pDataBlocks->prevTS = k; - return TSDB_CODE_SUCCESS; -} - -int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, int32_t *len, char *tmpTokenBuf, - SInsertStatementParam *pInsertParam) { - int32_t index = 0; - SStrToken sToken = {0}; - - char *row = pDataBlocks->pData + pDataBlocks->size; // skip the SSubmitBlk header - - SParsedDataColInfo *spd = &pDataBlocks->boundColumnInfo; - STableMeta * pTableMeta = pDataBlocks->pTableMeta; - SSchema * schema = tscGetTableSchema(pTableMeta); - SMemRowBuilder * pBuilder = &pDataBlocks->rowBuilder; - bool isParseBindParam = false; - - initSMemRow(row, pBuilder->memRowType, pDataBlocks, spd->numOfBound); - - // 1. set the parsed value from sql string - for (int i = 0; i < spd->numOfBound; ++i) { - // the start position in data block buffer of current value in sql - int32_t colIndex = spd->boundColumns[i]; - - char *start = row + spd->cols[colIndex].offset; - - SSchema *pSchema = &schema[colIndex]; // get colId here - - index = 0; - sToken = tStrGetToken(*str, &index, true); - *str += index; - - if (sToken.type == TK_QUESTION) { - if (!isParseBindParam) { - isParseBindParam = true; - } - if (pInsertParam->insertType != TSDB_QUERY_TYPE_STMT_INSERT) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "? only allowed in binding insertion", *str); - } - - uint32_t offset = (uint32_t)(start - pDataBlocks->pData); - if (tscAddParamToDataBlock(pDataBlocks, pSchema->type, (uint8_t)timePrec, pSchema->bytes, offset) != NULL) { - continue; - } - - strcpy(pInsertParam->msg, "client out of memory"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int16_t type = sToken.type; - if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && - type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || - (sToken.n == 0) || (type == TK_RP)) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "invalid data or symbol", sToken.z); - } - - // Remove quotation marks - if (TK_STRING == sToken.type) { - // delete escape character: \\, \', \" - char delim = sToken.z[0]; - - int32_t cnt = 0; - int32_t j = 0; - if (sToken.n >= TSDB_MAX_BYTES_PER_ROW) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "too long string", sToken.z); - } - - for (uint32_t k = 1; k < sToken.n - 1; ++k) { - if (sToken.z[k] == '\\' || (sToken.z[k] == delim && sToken.z[k + 1] == delim)) { - tmpTokenBuf[j] = sToken.z[k + 1]; - - cnt++; - j++; - k++; - continue; - } - - tmpTokenBuf[j] = sToken.z[k]; - j++; - } - - tmpTokenBuf[j] = 0; - sToken.z = tmpTokenBuf; - sToken.n -= 2 + cnt; - } - - bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); - int32_t toffset = -1; - int16_t colId = -1; - tscGetSTSRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId); - - int32_t ret = - tsParseOneColumnKV(pSchema, &sToken, row, pInsertParam->msg, str, isPrimaryKey, timePrec, toffset, colId); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - if (isPrimaryKey) { - TSKEY tsKey = memRowKey(row); - if (tsCheckTimestamp(pDataBlocks, (const char *)&tsKey) != TSDB_CODE_SUCCESS) { - tscInvalidOperationMsg(pInsertParam->msg, "client time/server time can not be mixed up", sToken.z); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - } - } - - if (!isParseBindParam) { - // set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && isDataRow(row)) { - SDataRow dataRow = memRowDataBody(row); - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { - tdAppendDataColVal(dataRow, getNullValue(schema[i].type), true, schema[i].type, spd->cols[i].toffset); - } - } - } - } - - *len = pBuilder->rowSize; - - return TSDB_CODE_SUCCESS; -} - -static int32_t rowDataCompar(const void *lhs, const void *rhs) { - TSKEY left = *(TSKEY *)lhs; - TSKEY right = *(TSKEY *)rhs; - - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; - } -} -int32_t schemaIdxCompar(const void *lhs, const void *rhs) { - uint16_t left = *(uint16_t *)lhs; - uint16_t right = *(uint16_t *)rhs; - - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; - } -} - -int32_t boundIdxCompar(const void *lhs, const void *rhs) { - uint16_t left = *(uint16_t *)POINTER_SHIFT(lhs, sizeof(uint16_t)); - uint16_t right = *(uint16_t *)POINTER_SHIFT(rhs, sizeof(uint16_t)); - - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; - } -} - -int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SInsertStatementParam *pInsertParam, - int32_t* numOfRows, char *tmpTokenBuf) { - int32_t index = 0; - int32_t code = 0; - - (*numOfRows) = 0; - - SStrToken sToken; - - STableMeta* pTableMeta = pDataBlock->pTableMeta; - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - - int32_t precision = tinfo.precision; - - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); - - if (TSDB_CODE_SUCCESS != (code = initMemRowBuilder(&pDataBlock->rowBuilder, 0, &pDataBlock->boundColumnInfo))) { - return code; - } - pDataBlock->rowBuilder.rowSize = extendedRowSize; - while (1) { - index = 0; - sToken = tStrGetToken(*str, &index, false); - if (sToken.n == 0 || sToken.type != TK_LP) break; - - *str += index; - if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { - int32_t tSize; - code = tscAllocateMemIfNeed(pDataBlock, extendedRowSize, &tSize); - if (code != TSDB_CODE_SUCCESS) { //TODO pass the correct error code to client - strcpy(pInsertParam->msg, "client out of memory"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - ASSERT(tSize >= maxRows); - maxRows = tSize; - } - - int32_t len = 0; - code = tsParseOneRow(str, pDataBlock, precision, &len, tmpTokenBuf, pInsertParam); - if (code != TSDB_CODE_SUCCESS) { // error message has been set in tsParseOneRow, return directly - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - - pDataBlock->size += len; - - index = 0; - sToken = tStrGetToken(*str, &index, false); - if (sToken.n == 0 || sToken.type != TK_RP) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, ") expected", *str); - } - - *str += index; - - (*numOfRows)++; - } - - if ((*numOfRows) <= 0) { - strcpy(pInsertParam->msg, "no any data points"); - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } else { - return TSDB_CODE_SUCCESS; - } -} - -void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols) { - pColInfo->numOfCols = numOfCols; - pColInfo->numOfBound = numOfCols; - pColInfo->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode - pColInfo->boundColumns = calloc(pColInfo->numOfCols, sizeof(int32_t)); - pColInfo->cols = calloc(pColInfo->numOfCols, sizeof(SBoundColumn)); - pColInfo->colIdxInfo = NULL; - pColInfo->flen = 0; - pColInfo->allNullLen = 0; - - int32_t nVar = 0; - for (int32_t i = 0; i < pColInfo->numOfCols; ++i) { - uint8_t type = pSchema[i].type; - if (i > 0) { - pColInfo->cols[i].offset = pSchema[i - 1].bytes + pColInfo->cols[i - 1].offset; - pColInfo->cols[i].toffset = pColInfo->flen; - } - pColInfo->flen += TYPE_BYTES[type]; - switch (type) { - case TSDB_DATA_TYPE_BINARY: - pColInfo->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); - ++nVar; - break; - case TSDB_DATA_TYPE_NCHAR: - pColInfo->allNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - ++nVar; - break; - default: - break; - } - pColInfo->boundColumns[i] = i; - } - pColInfo->allNullLen += pColInfo->flen; - pColInfo->boundNullLen = pColInfo->allNullLen; // default set allNullLen - pColInfo->extendedVarLen = (uint16_t)(nVar * sizeof(VarDataOffsetT)); -} - -int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows) { - size_t remain = pDataBlock->nAllocSize - pDataBlock->size; - const int factor = 5; - uint32_t nAllocSizeOld = pDataBlock->nAllocSize; - - // expand the allocated size - if (remain < rowSize * factor) { - while (remain < rowSize * factor) { - pDataBlock->nAllocSize = (uint32_t)(pDataBlock->nAllocSize * 1.5); - remain = pDataBlock->nAllocSize - pDataBlock->size; - } - - char *tmp = realloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); - if (tmp != NULL) { - pDataBlock->pData = tmp; - memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); - } else { - // do nothing, if allocate more memory failed - pDataBlock->nAllocSize = nAllocSizeOld; - *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; - return TSDB_CODE_SUCCESS; -} - -int32_t FORCE_INLINE tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { - pBlocks->tid = pTableMeta->id.tid; - pBlocks->uid = pTableMeta->id.uid; - pBlocks->sversion = pTableMeta->sversion; - - if (pBlocks->numOfRows + numOfRows >= INT16_MAX) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } else { - pBlocks->numOfRows += numOfRows; - return TSDB_CODE_SUCCESS; - } -} - -// data block is disordered, sort it in ascending order -void tscSortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) { - SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; - - // size is less than the total size, since duplicated rows may be removed yet. - assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SSubmitBlk) == dataBuf->size); - - // if use server time, this block must be ordered - if (dataBuf->tsSource == TSDB_USE_SERVER_TS) { - assert(dataBuf->ordered); - } - - if (!dataBuf->ordered) { - char *pBlockData = pBlocks->data; - qsort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar); - - int32_t i = 0; - int32_t j = 1; - - while (j < pBlocks->numOfRows) { - TSKEY ti = *(TSKEY *)(pBlockData + dataBuf->rowSize * i); - TSKEY tj = *(TSKEY *)(pBlockData + dataBuf->rowSize * j); - - if (ti == tj) { - ++j; - continue; - } - - int32_t nextPos = (++i); - if (nextPos != j) { - memmove(pBlockData + dataBuf->rowSize * nextPos, pBlockData + dataBuf->rowSize * j, dataBuf->rowSize); - } - - ++j; - } - - dataBuf->ordered = true; - - pBlocks->numOfRows = i + 1; - dataBuf->size = sizeof(SSubmitBlk) + dataBuf->rowSize * pBlocks->numOfRows; - } - - dataBuf->prevTS = INT64_MIN; -} - -// data block is disordered, sort it in ascending order -int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKeyInfo) { - SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; - int16_t nRows = pBlocks->numOfRows; - - // size is less than the total size, since duplicated rows may be removed yet. - - // if use server time, this block must be ordered - if (dataBuf->tsSource == TSDB_USE_SERVER_TS) { - assert(dataBuf->ordered); - } - // allocate memory - size_t nAlloc = nRows * sizeof(SBlockKeyTuple); - if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { - char *tmp = trealloc(pBlkKeyInfo->pKeyTuple, nAlloc); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple *)tmp; - pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; - } - memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); - - int32_t extendedRowSize = getExtendedRowSize(dataBuf); - SBlockKeyTuple *pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - char * pBlockData = pBlocks->data; - int n = 0; - while (n < nRows) { - pBlkKeyTuple->skey = memRowKey(pBlockData); - pBlkKeyTuple->payloadAddr = pBlockData; - - // next loop - pBlockData += extendedRowSize; - ++pBlkKeyTuple; - ++n; - } - - if (!dataBuf->ordered) { - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataCompar); - - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - int32_t i = 0; - int32_t j = 1; - while (j < nRows) { - TSKEY ti = (pBlkKeyTuple + i)->skey; - TSKEY tj = (pBlkKeyTuple + j)->skey; - - if (ti == tj) { - ++j; - continue; - } - - int32_t nextPos = (++i); - if (nextPos != j) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); - } - ++j; - } - - dataBuf->ordered = true; - pBlocks->numOfRows = i + 1; - } - - dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; - dataBuf->prevTS = INT64_MIN; - - return 0; -} - -static int32_t doParseInsertStatement(SInsertStatementParam *pInsertParam, char **str, STableDataBlocks* dataBuf, int32_t *totalNum) { - int32_t maxNumOfRows; - int32_t code = tscAllocateMemIfNeed(dataBuf, getExtendedRowSize(dataBuf), &maxNumOfRows); - if (TSDB_CODE_SUCCESS != code) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - code = TSDB_CODE_TSC_INVALID_OPERATION; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" - - int32_t numOfRows = 0; - code = tsParseValues(str, dataBuf, maxNumOfRows, pInsertParam, &numOfRows, tmpTokenBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - for (uint32_t i = 0; i < dataBuf->numOfParams; ++i) { - SParamInfo *param = dataBuf->params + i; - if (param->idx == -1) { - param->idx = pInsertParam->numOfParams++; - param->offset -= sizeof(SSubmitBlk); - } - } - - SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData); - code = tsSetBlockInfo(pBlocks, dataBuf->pTableMeta, numOfRows); - if (code != TSDB_CODE_SUCCESS) { - tscInvalidOperationMsg(pInsertParam->msg, "too many rows in sql, total number of rows should be less than 32767", *str); - return code; - } - - dataBuf->numOfTables = 1; - *totalNum += numOfRows; - return TSDB_CODE_SUCCESS; -} - -static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundColumn) { - int32_t index = 0; - SStrToken sToken = {0}; - SStrToken tableToken = {0}; - int32_t code = TSDB_CODE_SUCCESS; - - const int32_t TABLE_INDEX = 0; - const int32_t STABLE_INDEX = 1; - - SSqlCmd * pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - SInsertStatementParam* pInsertParam = &pCmd->insertParam; - - char *sql = *sqlstr; - - // get the token of specified table - index = 0; - tableToken = tStrGetToken(sql, &index, false); - sql += index; - - // skip possibly exists column list - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; - - int32_t numOfColList = 0; - - // Bind table columns list in string, skip it and continue - if (sToken.type == TK_LP) { - *boundColumn = &sToken.z[0]; - - while (1) { - index = 0; - sToken = tStrGetToken(sql, &index, false); - - if (sToken.type == TK_ILLEGAL) { - return tscSQLSyntaxErrMsg(pCmd->payload, "unrecognized token", sToken.z); - } - - if (sToken.type == TK_RP) { - break; - } - - sql += index; - ++numOfColList; - } - - sToken = tStrGetToken(sql, &index, false); - sql += index; - } - - if (numOfColList == 0 && (*boundColumn) != NULL) { - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); - - if (sToken.type == TK_USING) { // create table if not exists according to the super table - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; - - //the source super table is moved to the secondary position of the pTableMetaInfo list - if (pQueryInfo->numOfTables < 2) { - tscAddEmptyMetaInfo(pQueryInfo); - } - - STableMetaInfo *pSTableMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); - code = tscSetTableFullName(&pSTableMetaInfo->name, &sToken, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - tNameExtractFullName(&pSTableMetaInfo->name, pInsertParam->tagData.name); - pInsertParam->tagData.dataLen = 0; - - code = tscGetTableMeta(pSql, pSTableMetaInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (!UTIL_TABLE_IS_SUPER_TABLE(pSTableMetaInfo)) { - return tscInvalidOperationMsg(pInsertParam->msg, "create table only from super table is allowed", sToken.z); - } - - SSchema *pTagSchema = tscGetTableTagSchema(pSTableMetaInfo->pTableMeta); - STableComInfo tinfo = tscGetTableInfo(pSTableMetaInfo->pTableMeta); - - SParsedDataColInfo spd = {0}; - tscSetBoundColumnInfo(&spd, pTagSchema, tscGetNumOfTags(pSTableMetaInfo->pTableMeta)); - - index = 0; - sToken = tStrGetToken(sql, &index, false); - if (sToken.type != TK_TAGS && sToken.type != TK_LP) { - tscDestroyBoundColumnInfo(&spd); - return tscSQLSyntaxErrMsg(pInsertParam->msg, "keyword TAGS expected", sToken.z); - } - - // parse the bound tags column - if (sToken.type == TK_LP) { - /* - * insert into tablename (col1, col2,..., coln) using superTableName (tagName1, tagName2, ..., tagNamen) - * tags(tagVal1, tagVal2, ..., tagValn) values(v1, v2,... vn); - */ - char* end = NULL; - code = parseBoundColumns(pInsertParam, &spd, pTagSchema, sql, &end); - if (code != TSDB_CODE_SUCCESS) { - tscDestroyBoundColumnInfo(&spd); - return code; - } - - sql = end; - - index = 0; // keywords of "TAGS" - sToken = tStrGetToken(sql, &index, false); - sql += index; - } else { - sql += index; - } - - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; - - if (sToken.type != TK_LP) { - tscDestroyBoundColumnInfo(&spd); - return tscSQLSyntaxErrMsg(pInsertParam->msg, "( is expected", sToken.z); - } - - SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { - tscDestroyBoundColumnInfo(&spd); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int i = 0; i < spd.numOfBound; ++i) { - SSchema *pSchema = &pTagSchema[spd.boundColumns[i]]; - - index = 0; - sToken = tStrGetToken(sql, &index, true); - sql += index; - - if (TK_ILLEGAL == sToken.type) { - tdDestroyKVRowBuilder(&kvRowBuilder); - tscDestroyBoundColumnInfo(&spd); - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - - if (sToken.n == 0 || sToken.type == TK_RP) { - break; - } - - // Remove quotation marks - if (TK_STRING == sToken.type) { - sToken.z++; - sToken.n -= 2; - } - - char tagVal[TSDB_MAX_TAGS_LEN]; - code = tsParseOneColumn(pSchema, &sToken, tagVal, pInsertParam->msg, &sql, false, tinfo.precision); - if (code != TSDB_CODE_SUCCESS) { - tdDestroyKVRowBuilder(&kvRowBuilder); - tscDestroyBoundColumnInfo(&spd); - return code; - } - - tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); - } - - tscDestroyBoundColumnInfo(&spd); - - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - tdDestroyKVRowBuilder(&kvRowBuilder); - if (row == NULL) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "tag value expected", NULL); - } - tdSortKVRowByColIdx(row); - - pInsertParam->tagData.dataLen = kvRowLen(row); - if (pInsertParam->tagData.dataLen <= 0){ - return tscSQLSyntaxErrMsg(pInsertParam->msg, "tag value expected", NULL); - } - - char* pTag = realloc(pInsertParam->tagData.data, pInsertParam->tagData.dataLen); - if (pTag == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - kvRowCpy(pTag, row); - free(row); - pInsertParam->tagData.data = pTag; - - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; - if (sToken.n == 0 || sToken.type != TK_RP) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, ") expected", sToken.z); - } - - /* parse columns after super table tags values. - * insert into table_name using super_table(tag_name1, tag_name2) tags(tag_val1, tag_val2) - * (normal_col1, normal_col2) values(normal_col1_val, normal_col2_val); - * */ - index = 0; - sToken = tStrGetToken(sql, &index, false); - sql += index; - int numOfColsAfterTags = 0; - if (sToken.type == TK_LP) { - if (*boundColumn != NULL) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "bind columns again", sToken.z); - } else { - *boundColumn = &sToken.z[0]; - } - - while (1) { - index = 0; - sToken = tStrGetToken(sql, &index, false); - - if (sToken.type == TK_RP) { - break; - } - - if (sToken.n == 0 || sToken.type == TK_SEMI || index == 0) { - return tscSQLSyntaxErrMsg(pCmd->payload, "unexpected token", sql); - } - - sql += index; - ++numOfColsAfterTags; - } - - if (numOfColsAfterTags == 0 && (*boundColumn) != NULL) { - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - - sToken = tStrGetToken(sql, &index, false); - } - - sql = sToken.z; - - if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(pInsertParam->msg, "invalid table name", *sqlstr); - } - - int32_t ret = tscSetTableFullName(&pTableMetaInfo->name, &tableToken, pSql); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - if (sql == NULL) { - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - - code = tscGetTableMetaEx(pSql, pTableMetaInfo, true, false); - if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) { - return code; - } - - } else { - if (sToken.z == NULL) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "", sql); - } - - sql = sToken.z; - code = tscGetTableMetaEx(pSql, pTableMetaInfo, false, false); - if (pInsertParam->sql == NULL) { - assert(code == TSDB_CODE_TSC_ACTION_IN_PROGRESS); - } - } - - *sqlstr = sql; - return code; -} - -int validateTableName(char *tblName, int len, SStrToken* psTblToken) { - tstrncpy(psTblToken->z, tblName, TSDB_TABLE_FNAME_LEN); - - psTblToken->n = len; - psTblToken->type = TK_ID; - tGetToken(psTblToken->z, &psTblToken->type); - - return tscValidateName(psTblToken); -} - -static int32_t validateDataSource(SInsertStatementParam *pInsertParam, int32_t type, const char *sql) { - uint32_t *insertType = &pInsertParam->insertType; - if (*insertType == TSDB_QUERY_TYPE_STMT_INSERT && type == TSDB_QUERY_TYPE_INSERT) { - return TSDB_CODE_SUCCESS; - } - - if ((*insertType) != 0 && (*insertType) != type) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "keyword VALUES and FILE are not allowed to mixed up", sql); - } - - *insertType = type; - return TSDB_CODE_SUCCESS; -} - -static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo* pColInfo, SSchema* pSchema, - char* str, char **end) { - int32_t nCols = pColInfo->numOfCols; - - pColInfo->numOfBound = 0; - pColInfo->boundNullLen = 0; - memset(pColInfo->boundColumns, 0, sizeof(int32_t) * nCols); - for (int32_t i = 0; i < nCols; ++i) { - pColInfo->cols[i].valStat = VAL_STAT_NONE; - } - - int32_t code = TSDB_CODE_SUCCESS; - - int32_t index = 0; - SStrToken sToken = tStrGetToken(str, &index, false); - str += index; - - if (sToken.type != TK_LP) { - code = tscSQLSyntaxErrMsg(pInsertParam->msg, "( is expected", sToken.z); - goto _clean; - } - - bool isOrdered = true; - int32_t lastColIdx = -1; // last column found - while (1) { - index = 0; - sToken = tStrGetToken(str, &index, false); - str += index; - - if (TK_STRING == sToken.type) { - tscDequoteAndTrimToken(&sToken); - } - - if (sToken.type == TK_RP) { - if (end != NULL) { // set the end position - *end = str; - } - - break; - } - - bool findColumnIndex = false; - - // todo speedup by using hash list - int32_t nScanned = 0, t = lastColIdx + 1; - while (t < nCols) { - if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) { - if (pColInfo->cols[t].valStat == VAL_STAT_HAS) { - code = tscInvalidOperationMsg(pInsertParam->msg, "duplicated column name", sToken.z); - goto _clean; - } - - pColInfo->cols[t].valStat = VAL_STAT_HAS; - pColInfo->boundColumns[pColInfo->numOfBound] = t; - ++pColInfo->numOfBound; - switch (pSchema[t].type) { - case TSDB_DATA_TYPE_BINARY: - pColInfo->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); - break; - case TSDB_DATA_TYPE_NCHAR: - pColInfo->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - break; - default: - pColInfo->boundNullLen += TYPE_BYTES[pSchema[t].type]; - break; - } - findColumnIndex = true; - if (isOrdered && (lastColIdx > t)) { - isOrdered = false; - } - lastColIdx = t; - break; - } - ++t; - ++nScanned; - } - if (!findColumnIndex) { - t = 0; - int32_t nRemain = nCols - nScanned; - while (t < nRemain) { - if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) { - if (pColInfo->cols[t].valStat == VAL_STAT_HAS) { - code = tscInvalidOperationMsg(pInsertParam->msg, "duplicated column name", sToken.z); - goto _clean; - } - - pColInfo->cols[t].valStat = VAL_STAT_HAS; - pColInfo->boundColumns[pColInfo->numOfBound] = t; - ++pColInfo->numOfBound; - switch (pSchema[t].type) { - case TSDB_DATA_TYPE_BINARY: - pColInfo->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); - break; - case TSDB_DATA_TYPE_NCHAR: - pColInfo->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - break; - default: - pColInfo->boundNullLen += TYPE_BYTES[pSchema[t].type]; - break; - } - findColumnIndex = true; - if (isOrdered && (lastColIdx > t)) { - isOrdered = false; - } - lastColIdx = t; - break; - } - ++t; - } - } - - if (!findColumnIndex) { - code = tscInvalidOperationMsg(pInsertParam->msg, "invalid column/tag name", sToken.z); - goto _clean; - } - } - - pColInfo->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; - - if (!isOrdered) { - pColInfo->colIdxInfo = tcalloc(pColInfo->numOfBound, sizeof(SBoundIdxInfo)); - if (pColInfo->colIdxInfo == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _clean; - } - SBoundIdxInfo *pColIdx = pColInfo->colIdxInfo; - for (uint16_t i = 0; i < pColInfo->numOfBound; ++i) { - pColIdx[i].schemaColIdx = (uint16_t)pColInfo->boundColumns[i]; - pColIdx[i].boundIdx = i; - } - qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); - for (uint16_t i = 0; i < pColInfo->numOfBound; ++i) { - pColIdx[i].finalIdx = i; - } - qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); - } - - memset(&pColInfo->boundColumns[pColInfo->numOfBound], 0, - sizeof(int32_t) * (pColInfo->numOfCols - pColInfo->numOfBound)); - - return TSDB_CODE_SUCCESS; - -_clean: - pInsertParam->sql = NULL; - return code; -} - -static int32_t getFileFullPath(SStrToken* pToken, char* output) { - char path[PATH_MAX] = {0}; - strncpy(path, pToken->z, pToken->n); - strdequote(path); - - wordexp_t full_path; - if (wordexp(path, &full_path, 0) != 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - tstrncpy(output, full_path.we_wordv[0], PATH_MAX); - wordfree(&full_path); - return TSDB_CODE_SUCCESS; -} - -/** - * parse insert sql - * @param pSql - * @return - */ -int tsParseInsertSql(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - - SInsertStatementParam* pInsertParam = &pCmd->insertParam; - pInsertParam->objectId = pSql->self; - char* str = pInsertParam->sql; - - int32_t totalNum = 0; - int32_t code = TSDB_CODE_SUCCESS; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - assert(pQueryInfo != NULL); - - STableMetaInfo *pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo):tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - code = terrno; - return code; - } - - if (NULL == pInsertParam->pTableBlockHashList) { - pInsertParam->pTableBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - if (NULL == pInsertParam->pTableBlockHashList) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _clean; - } - } else { - str = pInsertParam->sql; - } - - tscDebug("0x%"PRIx64" create data block list hashList:%p", pSql->self, pInsertParam->pTableBlockHashList); - - while (1) { - int32_t index = 0; - SStrToken sToken = tStrGetToken(str, &index, false); - - // no data in the sql string anymore. - if (sToken.n == 0) { - /* - * if the data is from the data file, no data has been generated yet. So, there no data to - * merge or submit, save the file path and parse the file in other routines. - */ - if (TSDB_QUERY_HAS_TYPE(pInsertParam->insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { - goto _clean; - } - - /* - * if no data has been generated during parsing the sql string, error msg will return - * Otherwise, create the first submit block and submit to virtual node. - */ - if (totalNum == 0) { - code = TSDB_CODE_TSC_INVALID_OPERATION; - goto _clean; - } else { - break; - } - } - - pInsertParam->sql = sToken.z; - char buf[TSDB_TABLE_FNAME_LEN]; - SStrToken sTblToken; - sTblToken.z = buf; - // Check if the table name available or not - if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) { - code = tscInvalidOperationMsg(pInsertParam->msg, "table name invalid", sToken.z); - goto _clean; - } - - if ((code = tscSetTableFullName(&pTableMetaInfo->name, &sTblToken, pSql)) != TSDB_CODE_SUCCESS) { - goto _clean; - } - - char *bindedColumns = NULL; - if ((code = tscCheckIfCreateTable(&str, pSql, &bindedColumns)) != TSDB_CODE_SUCCESS) { - /* - * After retrieving the table meta from server, the sql string will be parsed from the paused position. - * And during the getTableMetaCallback function, the sql string will be parsed from the paused position. - */ - if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) { - return code; - } - - tscError("0x%"PRIx64" async insert parse error, code:%s", pSql->self, tstrerror(code)); - pInsertParam->sql = NULL; - goto _clean; - } - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - code = tscInvalidOperationMsg(pInsertParam->msg, "insert data into super table is not supported", NULL); - goto _clean; - } - - index = 0; - sToken = tStrGetToken(str, &index, false); - str += index; - - if (sToken.n == 0 || (sToken.type != TK_FILE && sToken.type != TK_VALUES)) { - code = tscSQLSyntaxErrMsg(pInsertParam->msg, "keyword VALUES or FILE required", sToken.z); - goto _clean; - } - - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - if (sToken.type == TK_FILE) { - if (validateDataSource(pInsertParam, TSDB_QUERY_TYPE_FILE_INSERT, sToken.z) != TSDB_CODE_SUCCESS) { - goto _clean; - } - - index = 0; - sToken = tStrGetToken(str, &index, false); - if (sToken.type != TK_STRING && sToken.type != TK_ID) { - code = tscSQLSyntaxErrMsg(pInsertParam->msg, "file path is required following keyword FILE", sToken.z); - goto _clean; - } - str += index; - if (sToken.n == 0) { - code = tscSQLSyntaxErrMsg(pInsertParam->msg, "file path is required following keyword FILE", sToken.z); - goto _clean; - } - - code = getFileFullPath(&sToken, pCmd->payload); - if (code != TSDB_CODE_SUCCESS) { - tscInvalidOperationMsg(pInsertParam->msg, "invalid filename", sToken.z); - goto _clean; - } - } else { - if (bindedColumns == NULL) { - STableMeta *pTableMeta = pTableMetaInfo->pTableMeta; - if (validateDataSource(pInsertParam, TSDB_QUERY_TYPE_INSERT, sToken.z) != TSDB_CODE_SUCCESS) { - goto _clean; - } - - STableDataBlocks *dataBuf = NULL; - int32_t ret = tscGetDataBlockFromList(pInsertParam->pTableBlockHashList, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, - &dataBuf, NULL); - if (ret != TSDB_CODE_SUCCESS) { - goto _clean; - } - - code = doParseInsertStatement(pInsertParam, &str, dataBuf, &totalNum); - if (code != TSDB_CODE_SUCCESS) { - goto _clean; - } - } else { // bindedColumns != NULL - // insert into tablename(col1, col2,..., coln) values(v1, v2,... vn); - STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, 0)->pTableMeta; - - if (validateDataSource(pInsertParam, TSDB_QUERY_TYPE_INSERT, sToken.z) != TSDB_CODE_SUCCESS) { - goto _clean; - } - - STableDataBlocks *dataBuf = NULL; - int32_t ret = tscGetDataBlockFromList(pInsertParam->pTableBlockHashList, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, - &dataBuf, NULL); - if (ret != TSDB_CODE_SUCCESS) { - goto _clean; - } - - SSchema *pSchema = tscGetTableSchema(pTableMeta); - code = parseBoundColumns(pInsertParam, &dataBuf->boundColumnInfo, pSchema, bindedColumns, NULL); - if (code != TSDB_CODE_SUCCESS) { - goto _clean; - } - - if (dataBuf->boundColumnInfo.cols[0].valStat == VAL_STAT_NONE) { - code = tscInvalidOperationMsg(pInsertParam->msg, "primary timestamp column can not be null", NULL); - goto _clean; - } - - if (sToken.type != TK_VALUES) { - code = tscSQLSyntaxErrMsg(pInsertParam->msg, "keyword VALUES is expected", sToken.z); - goto _clean; - } - - code = doParseInsertStatement(pInsertParam, &str, dataBuf, &totalNum); - if (code != TSDB_CODE_SUCCESS) { - goto _clean; - } - } - } - } - - // we need to keep the data blocks if there are parameters in the sql - if (pInsertParam->numOfParams > 0) { - goto _clean; - } - - // merge according to vgId - if (!TSDB_QUERY_HAS_TYPE(pInsertParam->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pInsertParam->pTableBlockHashList) > 0) { - if ((code = tscMergeTableDataBlocks(pInsertParam, true)) != TSDB_CODE_SUCCESS) { - goto _clean; - } - } - - code = TSDB_CODE_SUCCESS; - goto _clean; - -_clean: - pInsertParam->sql = NULL; - return code; -} - -int tsInsertInitialCheck(SSqlObj *pSql) { - if (!pSql->pTscObj->writeAuth) { - return TSDB_CODE_TSC_NO_WRITE_AUTH; - } - - int32_t index = 0; - SSqlCmd *pCmd = &pSql->cmd; - - SStrToken sToken = tStrGetToken(pSql->sqlstr, &index, false); - assert(sToken.type == TK_INSERT || sToken.type == TK_IMPORT); - - pCmd->count = 0; - pCmd->command = TSDB_SQL_INSERT; - SInsertStatementParam* pInsertParam = &pCmd->insertParam; - - SQueryInfo *pQueryInfo = tscGetQueryInfoS(pCmd); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT); - - sToken = tStrGetToken(pSql->sqlstr, &index, false); - if (sToken.type != TK_INTO) { - return tscSQLSyntaxErrMsg(pInsertParam->msg, "keyword INTO is expected", sToken.z); - } - - pInsertParam->sql = sToken.z + sToken.n; - return TSDB_CODE_SUCCESS; -} - -int tsParseSql(SSqlObj *pSql, bool initial) { - int32_t ret = TSDB_CODE_SUCCESS; - SSqlCmd* pCmd = &pSql->cmd; - if (!initial) { - tscDebug("0x%"PRIx64" resume to parse sql: %s", pSql->self, pCmd->insertParam.sql); - } - - ret = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); - if (TSDB_CODE_SUCCESS != ret) { - return ret; - } - - if (tscIsInsertData(pSql->sqlstr)) { - if (initial && ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS)) { - strncpy(pCmd->payload, pCmd->insertParam.msg, TSDB_DEFAULT_PAYLOAD_SIZE); - return ret; - } - - ret = tsParseInsertSql(pSql); - if (pSql->parseRetry < 1 && (ret == TSDB_CODE_TSC_SQL_SYNTAX_ERROR || ret == TSDB_CODE_TSC_INVALID_OPERATION)) { - tscDebug("0x%"PRIx64 " parse insert sql statement failed, code:%s, clear meta cache and retry ", pSql->self, tstrerror(ret)); - - tscResetSqlCmd(pCmd, true, pSql->self); - pSql->parseRetry++; - - if ((ret = tsInsertInitialCheck(pSql)) == TSDB_CODE_SUCCESS) { - ret = tsParseInsertSql(pSql); - } - } - - if (ret != TSDB_CODE_SUCCESS) { - strncpy(pCmd->payload, pCmd->insertParam.msg, TSDB_DEFAULT_PAYLOAD_SIZE); - } - } else { - SSqlInfo sqlInfo = qSqlParse(pSql->sqlstr); - ret = tscValidateSqlInfo(pSql, &sqlInfo); - if (ret == TSDB_CODE_TSC_INVALID_OPERATION && pSql->parseRetry < 1 && sqlInfo.type == TSDB_SQL_SELECT) { - tscDebug("0x%"PRIx64 " parse query sql statement failed, code:%s, clear meta cache and retry ", pSql->self, tstrerror(ret)); - - tscResetSqlCmd(pCmd, true, pSql->self); - pSql->parseRetry++; - - ret = tscValidateSqlInfo(pSql, &sqlInfo); - } - - SqlInfoDestroy(&sqlInfo); - } - - /* - * the pRes->code may be modified or released by another thread in tscTableMetaCallBack function, - * so do NOT use pRes->code to determine if the getTableMeta function - * invokes new threads to get data from mgmt node or simply retrieves data from cache. - * do NOT assign return code to pRes->code for the same reason since it may be released by another thread already. - */ - return ret; -} - -static int doPackSendDataBlock(SSqlObj* pSql, SInsertStatementParam *pInsertParam, STableMeta* pTableMeta, int32_t numOfRows, STableDataBlocks *pTableDataBlocks) { - int32_t code = TSDB_CODE_SUCCESS; - - SSubmitBlk *pBlocks = (SSubmitBlk *)(pTableDataBlocks->pData); - code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); - if (code != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(pInsertParam->msg, "too many rows in sql, total number of rows should be less than 32767", NULL); - } - - if ((code = tscMergeTableDataBlocks(pInsertParam, true)) != TSDB_CODE_SUCCESS) { - return code; - } - - STableDataBlocks *pDataBlock = taosArrayGetP(pInsertParam->pDataBlocks, 0); - if ((code = tscCopyDataBlockToPayload(pSql, pDataBlock)) != TSDB_CODE_SUCCESS) { - return code; - } - - return TSDB_CODE_SUCCESS; -} - -typedef struct SImportFileSupport { - SSqlObj *pSql; - FILE *fp; -} SImportFileSupport; - -static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRows) { - assert(param != NULL && tres != NULL); - - char * tokenBuf = NULL; - size_t n = 0; - ssize_t readLen = 0; - char * line = NULL; - int32_t count = 0; - int32_t maxRows = 0; - FILE * fp = NULL; - - SSqlObj *pSql = tres; - SSqlCmd *pCmd = &pSql->cmd; - - SImportFileSupport *pSupporter = (SImportFileSupport *)param; - - SSqlObj *pParentSql = pSupporter->pSql; - fp = pSupporter->fp; - - int32_t code = pSql->res.code; - - // retry parse data from file and import data from the begining again - if (code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - assert(pSql->res.numOfRows == 0); - int32_t ret = fseek(fp, 0, SEEK_SET); - if (ret < 0) { - tscError("0x%"PRIx64" failed to seek SEEK_SET since:%s", pSql->self, tstrerror(errno)); - code = TAOS_SYSTEM_ERROR(errno); - goto _error; - } - } else if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - // accumulate the total submit records - pParentSql->res.numOfRows += pSql->res.numOfRows; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - - SInsertStatementParam *pInsertParam = &pCmd->insertParam; - destroyTableNameList(pInsertParam); - - pInsertParam->pDataBlocks = tscDestroyBlockArrayList(pInsertParam->pDataBlocks); - - if (pInsertParam->pTableBlockHashList == NULL) { - pInsertParam->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - if (pInsertParam->pTableBlockHashList == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - } - - STableDataBlocks *pTableDataBlock = NULL; - int32_t ret = tscGetDataBlockFromList(pInsertParam->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, - sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, - &pTableDataBlock, NULL); - if (ret != TSDB_CODE_SUCCESS) { - pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - int32_t extendedRowSize = getExtendedRowSize(pTableDataBlock); - tscAllocateMemIfNeed(pTableDataBlock, extendedRowSize, &maxRows); - tokenBuf = calloc(1, TSDB_MAX_BYTES_PER_ROW); - if (tokenBuf == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - // insert from .csv means full and ordered columns, thus use SDataRow all the time - ASSERT(SMEM_ROW_DATA == pTableDataBlock->rowBuilder.memRowType); - pTableDataBlock->rowBuilder.rowSize = extendedRowSize; - - while ((readLen = tgetline(&line, &n, fp)) != -1) { - if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { - line[--readLen] = 0; - } - - if (readLen == 0) { - continue; - } - - char *lineptr = line; - strtolower(line, line); - - int32_t len = 0; - code = tsParseOneRow(&lineptr, pTableDataBlock, tinfo.precision, &len, tokenBuf, pInsertParam); - if (code != TSDB_CODE_SUCCESS || pTableDataBlock->numOfParams > 0) { - pSql->res.code = code; - break; - } - - pTableDataBlock->size += len; - - if (++count >= maxRows) { - break; - } - } - - tfree(tokenBuf); - tfree(line); - - pParentSql->res.code = code; - if (code == TSDB_CODE_SUCCESS) { - if (count > 0) { - pSql->res.numOfRows = 0; - code = doPackSendDataBlock(pSql, pInsertParam, pTableMeta, count, pTableDataBlock); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - tscBuildAndSendRequest(pSql, NULL); - return; - } else { - taos_free_result(pSql); - tfree(pSupporter); - fclose(fp); - - pParentSql->fp = pParentSql->fetchFp; - - // all data has been sent to vnode, call user function - int32_t v = (int32_t)pParentSql->res.numOfRows; - (*pParentSql->fp)(pParentSql->param, pParentSql, v); - return; - } - } - -_error: - pParentSql->res.code = code; - tfree(tokenBuf); - tfree(line); - taos_free_result(pSql); - tfree(pSupporter); - fclose(fp); - - tscAsyncResultOnError(pParentSql); -} - -void tscImportDataFromFile(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - if (pCmd->command != TSDB_SQL_INSERT) { - return; - } - - SInsertStatementParam* pInsertParam = &pCmd->insertParam; - assert(TSDB_QUERY_HAS_TYPE(pInsertParam->insertType, TSDB_QUERY_TYPE_FILE_INSERT) && strlen(pCmd->payload) != 0); - pCmd->active = pCmd->pQueryInfo; - - SImportFileSupport *pSupporter = calloc(1, sizeof(SImportFileSupport)); - SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL); - - FILE *fp = fopen(pCmd->payload, "rb"); - if (fp == NULL) { - pSql->res.code = TAOS_SYSTEM_ERROR(errno); - tscError("0x%"PRIx64" failed to open file %s to load data from file, code:%s", pSql->self, pCmd->payload, tstrerror(pSql->res.code)); - - tfree(pSupporter); - taos_free_result(pNew); - tscAsyncResultOnError(pSql); - return; - } - - pSupporter->pSql = pSql; - pSupporter->fp = fp; - - parseFileSendDataBlock(pSupporter, pNew, TSDB_CODE_SUCCESS); -} diff --git a/2.0/src/client/src/tscParseLineProtocol.c b/2.0/src/client/src/tscParseLineProtocol.c deleted file mode 100644 index 2225b2c580a2a53817175b7f3073bca7db160bae..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscParseLineProtocol.c +++ /dev/null @@ -1,2184 +0,0 @@ -#include -#include -#include -#include - -#include "os.h" -#include "osString.h" -#include "ttype.h" -#include "tmd5.h" -#include "tstrbuild.h" -#include "tname.h" -#include "hash.h" -#include "tskiplist.h" - -#include "tscUtil.h" -#include "tsclient.h" -#include "tscLog.h" - -#include "../../../include/client/taos.h" -#include "tscParseLine.h" - -typedef struct { - char sTableName[TSDB_TABLE_NAME_LEN]; - SHashObj* tagHash; - SHashObj* fieldHash; - SArray* tags; //SArray - SArray* fields; //SArray - uint8_t precision; -} SSmlSTableSchema; - -//================================================================================================= - -static uint64_t linesSmlHandleId = 0; - -uint64_t genLinesSmlId() { - uint64_t id; - - do { - id = atomic_add_fetch_64(&linesSmlHandleId, 1); - } while (id == 0); - - return id; -} - -int compareSmlColKv(const void* p1, const void* p2) { - TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; - TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; - int kvLen1 = (int)strlen(kv1->key); - int kvLen2 = (int)strlen(kv2->key); - int res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); - if (res != 0) { - return res; - } else { - return kvLen1-kvLen2; - } -} - -typedef enum { - SCHEMA_ACTION_CREATE_STABLE, - SCHEMA_ACTION_ADD_COLUMN, - SCHEMA_ACTION_ADD_TAG, - SCHEMA_ACTION_CHANGE_COLUMN_SIZE, - SCHEMA_ACTION_CHANGE_TAG_SIZE, -} ESchemaAction; - -typedef struct { - char sTableName[TSDB_TABLE_NAME_LEN]; - SArray* tags; //SArray - SArray* fields; //SArray -} SCreateSTableActionInfo; - -typedef struct { - char sTableName[TSDB_TABLE_NAME_LEN]; - SSchema* field; -} SAlterSTableActionInfo; - -typedef struct { - ESchemaAction action; - union { - SCreateSTableActionInfo createSTable; - SAlterSTableActionInfo alterSTable; - }; -} SSchemaAction; - -static int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes, uint64_t id) { - if (!IS_VAR_DATA_TYPE(kv->type)) { - *bytes = tDataTypes[kv->type].bytes; - } else { - if (kv->type == TSDB_DATA_TYPE_NCHAR) { - char* ucs = malloc(kv->length * TSDB_NCHAR_SIZE + 1); - int32_t bytesNeeded = 0; - bool succ = taosMbsToUcs4(kv->value, kv->length, ucs, kv->length * TSDB_NCHAR_SIZE, &bytesNeeded); - if (!succ) { - free(ucs); - tscError("SML:0x%"PRIx64" convert nchar string to UCS4_LE failed:%s", id, kv->value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - free(ucs); - *bytes = bytesNeeded + VARSTR_HEADER_SIZE; - } else if (kv->type == TSDB_DATA_TYPE_BINARY) { - *bytes = kv->length + VARSTR_HEADER_SIZE; - } - } - return 0; -} - -static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* array, SSmlLinesInfo* info) { - SSchema* pField = NULL; - size_t* pFieldIdx = taosHashGet(hash, smlKv->key, strlen(smlKv->key)); - size_t fieldIdx = -1; - int32_t code = 0; - if (pFieldIdx) { - fieldIdx = *pFieldIdx; - pField = taosArrayGet(array, fieldIdx); - - if (pField->type != smlKv->type) { - tscError("SML:0x%"PRIx64" type mismatch. key %s, type %d. type before %d", info->id, smlKv->key, smlKv->type, pField->type); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - int32_t bytes = 0; - code = getFieldBytesFromSmlKv(smlKv, &bytes, info->id); - if (code != 0) { - return code; - } - pField->bytes = TMAX(pField->bytes, bytes); - - } else { - SSchema field = {0}; - size_t tagKeyLen = strlen(smlKv->key); - strncpy(field.name, smlKv->key, tagKeyLen); - field.name[tagKeyLen] = '\0'; - field.type = smlKv->type; - - int32_t bytes = 0; - code = getFieldBytesFromSmlKv(smlKv, &bytes, info->id); - if (code != 0) { - return code; - } - field.bytes = bytes; - - pField = taosArrayPush(array, &field); - fieldIdx = taosArrayGetSize(array) - 1; - taosHashPut(hash, field.name, tagKeyLen, &fieldIdx, sizeof(fieldIdx)); - } - - uintptr_t valPointer = (uintptr_t)smlKv; - taosHashPut(info->smlDataToSchema, &valPointer, sizeof(uintptr_t), &fieldIdx, sizeof(fieldIdx)); - - return 0; -} - -static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen, - SSmlLinesInfo* info) { - tscDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id); - qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); - - SStringBuilder sb; memset(&sb, 0, sizeof(sb)); - char sTableName[TSDB_TABLE_NAME_LEN] = {0}; - strtolower(sTableName, point->stableName); - taosStringBuilderAppendString(&sb, sTableName); - for (int j = 0; j < point->tagNum; ++j) { - taosStringBuilderAppendChar(&sb, ','); - TAOS_SML_KV* tagKv = point->tags + j; - char tagName[TSDB_COL_NAME_LEN] = {0}; - strtolower(tagName, tagKv->key); - taosStringBuilderAppendString(&sb, tagName); - taosStringBuilderAppendChar(&sb, '='); - taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); - } - size_t len = 0; - char* keyJoined = taosStringBuilderGetResult(&sb, &len); - MD5_CTX context; - MD5Init(&context); - MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); - MD5Final(&context); - *tableNameLen = snprintf(tableName, *tableNameLen, - "t_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], - context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], - context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], - context.digest[12], context.digest[13], context.digest[14], context.digest[15]); - taosStringBuilderDestroy(&sb); - tscDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName); - return 0; -} - -static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, SArray* stableSchemas, SSmlLinesInfo* info) { - int32_t code = 0; - SHashObj* sname2shema = taosHashInit(32, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - for (int i = 0; i < numPoint; ++i) { - TAOS_SML_DATA_POINT* point = &points[i]; - size_t stableNameLen = strlen(point->stableName); - size_t* pStableIdx = taosHashGet(sname2shema, point->stableName, stableNameLen); - SSmlSTableSchema* pStableSchema = NULL; - size_t stableIdx = -1; - if (pStableIdx) { - pStableSchema= taosArrayGet(stableSchemas, *pStableIdx); - stableIdx = *pStableIdx; - } else { - SSmlSTableSchema schema; - strncpy(schema.sTableName, point->stableName, stableNameLen); - schema.sTableName[stableNameLen] = '\0'; - schema.fields = taosArrayInit(64, sizeof(SSchema)); - schema.tags = taosArrayInit(8, sizeof(SSchema)); - schema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - schema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - pStableSchema = taosArrayPush(stableSchemas, &schema); - stableIdx = taosArrayGetSize(stableSchemas) - 1; - taosHashPut(sname2shema, schema.sTableName, stableNameLen, &stableIdx, sizeof(size_t)); - } - - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* tagKv = point->tags + j; - if (!point->childTableName) { - char childTableName[TSDB_TABLE_NAME_LEN]; - int32_t tableNameLen = TSDB_TABLE_NAME_LEN; - getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info); - point->childTableName = calloc(1, tableNameLen+1); - strncpy(point->childTableName, childTableName, tableNameLen); - point->childTableName[tableNameLen] = '\0'; - } - - code = buildSmlKvSchema(tagKv, pStableSchema->tagHash, pStableSchema->tags, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, tagKv->key); - return code; - } - } - - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* fieldKv = point->fields + j; - code = buildSmlKvSchema(fieldKv, pStableSchema->fieldHash, pStableSchema->fields, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, fieldKv->key); - return code; - } - } - - uintptr_t valPointer = (uintptr_t)point; - taosHashPut(info->smlDataToSchema, &valPointer, sizeof(uintptr_t), &stableIdx, sizeof(stableIdx)); - } - - size_t numStables = taosArrayGetSize(stableSchemas); - for (int32_t i = 0; i < numStables; ++i) { - SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); - taosHashCleanup(schema->tagHash); - taosHashCleanup(schema->fieldHash); - } - taosHashCleanup(sname2shema); - - tscDebug("SML:0x%"PRIx64" build point schema succeed. num of super table: %zu", info->id, numStables); - for (int32_t i = 0; i < numStables; ++i) { - SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); - tscDebug("\ttable name: %s, tags number: %zu, fields number: %zu", schema->sTableName, - taosArrayGetSize(schema->tags), taosArrayGetSize(schema->fields)); - } - - return 0; -} - -static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[], - SSchemaAction* action, bool* actionNeeded, SSmlLinesInfo* info) { - char fieldNameLowerCase[TSDB_COL_NAME_LEN] = {0}; - strtolower(fieldNameLowerCase, pointColField->name); - - size_t* pDbIndex = taosHashGet(dbAttrHash, fieldNameLowerCase, strlen(fieldNameLowerCase)); - if (pDbIndex) { - SSchema* dbAttr = taosArrayGet(dbAttrArray, *pDbIndex); - assert(strcasecmp(dbAttr->name, pointColField->name) == 0); - if (pointColField->type != dbAttr->type) { - tscError("SML:0x%"PRIx64" point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, pointColField->name, - pointColField->type, dbAttr->type); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (IS_VAR_DATA_TYPE(pointColField->type) && (pointColField->bytes > dbAttr->bytes)) { - if (isTag) { - action->action = SCHEMA_ACTION_CHANGE_TAG_SIZE; - } else { - action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; - } - memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); - memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); - action->alterSTable.field = pointColField; - *actionNeeded = true; - } - } else { - if (isTag) { - action->action = SCHEMA_ACTION_ADD_TAG; - } else { - action->action = SCHEMA_ACTION_ADD_COLUMN; - } - memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); - memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); - action->alterSTable.field = pointColField; - *actionNeeded = true; - } - if (*actionNeeded) { - tscDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, fieldNameLowerCase, - action->action); - } - return 0; -} - -static int32_t buildColumnDescription(SSchema* field, - char* buf, int32_t bufSize, int32_t* outBytes) { - uint8_t type = field->type; - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t bytes = field->bytes - VARSTR_HEADER_SIZE; - if (type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes/TSDB_NCHAR_SIZE; - } - int out = snprintf(buf, bufSize,"%s %s(%d)", - field->name,tDataTypes[field->type].name, bytes); - *outBytes = out; - } else { - int out = snprintf(buf, bufSize, "%s %s", - field->name, tDataTypes[type].name); - *outBytes = out; - } - - return 0; -} - - -static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInfo* info) { - int32_t code = 0; - int32_t outBytes = 0; - char *result = (char *)calloc(1, tsMaxSQLStringLen+1); - int32_t capacity = tsMaxSQLStringLen + 1; - - tscDebug("SML:0x%"PRIx64" apply schema action. action: %d", info->id, action->action); - switch (action->action) { - case SCHEMA_ACTION_ADD_COLUMN: { - int n = sprintf(result, "alter stable %s add column ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery - code = taos_errno(res); - char* errStr = taos_errstr(res); - char* begin = strstr(errStr, "duplicated column names"); - bool tscDupColNames = (begin != NULL); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%"PRIx64" apply schema action. error: %s", info->id, errStr); - } - taos_free_result(res); - - if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || code == TSDB_CODE_MND_TAG_ALREAY_EXIST || tscDupColNames) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - } - break; - } - case SCHEMA_ACTION_ADD_TAG: { - int n = sprintf(result, "alter stable %s add tag ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, - result+n, capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery - code = taos_errno(res); - char* errStr = taos_errstr(res); - char* begin = strstr(errStr, "duplicated column names"); - bool tscDupColNames = (begin != NULL); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); - } - taos_free_result(res); - - if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST || code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || tscDupColNames) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - } - break; - } - case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { - int n = sprintf(result, "alter stable %s modify column ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, result+n, - capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery - code = taos_errno(res); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); - } - taos_free_result(res); - - if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - } - break; - } - case SCHEMA_ACTION_CHANGE_TAG_SIZE: { - int n = sprintf(result, "alter stable %s modify tag ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, result+n, - capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery - code = taos_errno(res); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); - } - taos_free_result(res); - - if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - } - break; - } - case SCHEMA_ACTION_CREATE_STABLE: { - int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); - char* pos = result + n; int freeBytes = capacity - n; - size_t numCols = taosArrayGetSize(action->createSTable.fields); - for (int32_t i = 0; i < numCols; ++i) { - SSchema* field = taosArrayGet(action->createSTable.fields, i); - buildColumnDescription(field, pos, freeBytes, &outBytes); - pos += outBytes; freeBytes -= outBytes; - *pos = ','; ++pos; --freeBytes; - } - --pos; ++freeBytes; - - outBytes = snprintf(pos, freeBytes, ") tags ("); - pos += outBytes; freeBytes -= outBytes; - - size_t numTags = taosArrayGetSize(action->createSTable.tags); - for (int32_t i = 0; i < numTags; ++i) { - SSchema* field = taosArrayGet(action->createSTable.tags, i); - buildColumnDescription(field, pos, freeBytes, &outBytes); - pos += outBytes; freeBytes -= outBytes; - *pos = ','; ++pos; --freeBytes; - } - pos--; ++freeBytes; - outBytes = snprintf(pos, freeBytes, ")"); - TAOS_RES* res = taos_query(taos, result); - code = taos_errno(res); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); - } - taos_free_result(res); - - if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - code = taos_errno(res2); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - } - break; - } - - default: - break; - } - - free(result); - if (code != 0) { - tscError("SML:0x%"PRIx64 " apply schema action failure. %s", info->id, tstrerror(code)); - } - return code; -} - -static int32_t destroySmlSTableSchema(SSmlSTableSchema* schema) { - taosHashCleanup(schema->tagHash); - taosHashCleanup(schema->fieldHash); - taosArrayDestroy(schema->tags); - taosArrayDestroy(schema->fields); - return 0; -} - -static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { - schema->tags = taosArrayInit(8, sizeof(SSchema)); - schema->fields = taosArrayInit(64, sizeof(SSchema)); - schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); - schema->precision = tableMeta->tableInfo.precision; - for (int i=0; itableInfo.numOfColumns; ++i) { - SSchema field; - tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)+1); - field.type = tableMeta->schema[i].type; - field.bytes = tableMeta->schema[i].bytes; - taosArrayPush(schema->fields, &field); - size_t fieldIndex = taosArrayGetSize(schema->fields) - 1; - taosHashPut(schema->fieldHash, field.name, strlen(field.name), &fieldIndex, sizeof(fieldIndex)); - } - - for (int i=0; itableInfo.numOfTags; ++i) { - int j = i + tableMeta->tableInfo.numOfColumns; - SSchema field; - tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)+1); - field.type = tableMeta->schema[j].type; - field.bytes = tableMeta->schema[j].bytes; - taosArrayPush(schema->tags, &field); - size_t tagIndex = taosArrayGetSize(schema->tags) - 1; - taosHashPut(schema->tagHash, field.name, strlen(field.name), &tagIndex, sizeof(tagIndex)); - } - tscDebug("SML:0x%"PRIx64 " load table schema succeed. table name: %s, columns number: %d, tag number: %d, precision: %d", - info->id, tableName, tableMeta->tableInfo.numOfColumns, tableMeta->tableInfo.numOfTags, schema->precision); - return TSDB_CODE_SUCCESS; -} - -static int32_t retrieveTableMeta(TAOS* taos, char* tableName, STableMeta** pTableMeta, SSmlLinesInfo* info) { - int32_t code = 0; - int32_t retries = 0; - STableMeta* tableMeta = NULL; - while (retries++ < TSDB_MAX_REPLICA && tableMeta == NULL) { - STscObj* pObj = (STscObj*)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - tscDebug("SML:0x%" PRIx64 " retrieve table meta. super table name: %s", info->id, tableName); - - char tableNameLowerCase[TSDB_TABLE_NAME_LEN]; - strtolower(tableNameLowerCase, tableName); - - char sql[256]; - snprintf(sql, 256, "describe %s", tableNameLowerCase); - TAOS_RES* res = taos_query(taos, sql); - code = taos_errno(res); - if (code != 0) { - tscError("SML:0x%" PRIx64 " describe table failure. %s", info->id, taos_errstr(res)); - taos_free_result(res); - return code; - } - taos_free_result(res); - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - tscError("SML:0x%" PRIx64 " failed to allocate memory, reason:%s", info->id, strerror(errno)); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return code; - } - pSql->pTscObj = taos; - pSql->signature = pSql; - pSql->fp = NULL; - - registerSqlObj(pSql); - SStrToken tableToken = {.z = tableNameLowerCase, .n = (uint32_t)strlen(tableNameLowerCase), .type = TK_ID}; - tGetToken(tableNameLowerCase, &tableToken.type); - // Check if the table name available or not - if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pSql->cmd.payload, "table name is invalid"); - tscFreeRegisteredSqlObj(pSql); - return code; - } - - SName sname = {0}; - if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { - tscFreeRegisteredSqlObj(pSql); - return code; - } - char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; - memset(fullTableName, 0, tListLen(fullTableName)); - tNameExtractFullName(&sname, fullTableName); - tscFreeRegisteredSqlObj(pSql); - - size_t size = 0; - taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void**)&tableMeta, &size); - } - - if (tableMeta != NULL) { - *pTableMeta = tableMeta; - return TSDB_CODE_SUCCESS; - } else { - tscError("SML:0x%" PRIx64 " failed to retrieve table meta. super table name: %s", info->id, tableName); - return TSDB_CODE_TSC_NO_META_CACHED; - } -} - -static int32_t loadTableSchemaFromDB(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { - int32_t code = 0; - STableMeta* tableMeta = NULL; - code = retrieveTableMeta(taos, tableName, &tableMeta, info); - if (code == TSDB_CODE_SUCCESS) { - assert(tableMeta != NULL); - fillDbSchema(tableMeta, tableName, schema, info); - free(tableMeta); - tableMeta = NULL; - } - - return code; -} - -static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* info) { - int32_t code = 0; - size_t numStable = taosArrayGetSize(stableSchemas); - for (int i = 0; i < numStable; ++i) { - SSmlSTableSchema* pointSchema = taosArrayGet(stableSchemas, i); - SSmlSTableSchema dbSchema; - memset(&dbSchema, 0, sizeof(SSmlSTableSchema)); - - code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); - if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { - SSchemaAction schemaAction = {0}; - schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; - memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); - memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); - schemaAction.createSTable.tags = pointSchema->tags; - schemaAction.createSTable.fields = pointSchema->fields; - applySchemaAction(taos, &schemaAction, info); - code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, pointSchema->sTableName); - return code; - } else { - pointSchema->precision = dbSchema.precision; - destroySmlSTableSchema(&dbSchema); - } - } else if (code == TSDB_CODE_SUCCESS) { - size_t pointTagSize = taosArrayGetSize(pointSchema->tags); - size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); - - SHashObj* dbTagHash = dbSchema.tagHash; - SHashObj* dbFieldHash = dbSchema.fieldHash; - - for (int j = 0; j < pointTagSize; ++j) { - SSchema* pointTag = taosArrayGet(pointSchema->tags, j); - SSchemaAction schemaAction = {0}; - bool actionNeeded = false; - generateSchemaAction(pointTag, dbTagHash, dbSchema.tags, true, pointSchema->sTableName, - &schemaAction, &actionNeeded, info); - if (actionNeeded) { - code = applySchemaAction(taos, &schemaAction, info); - if (code != 0) { - destroySmlSTableSchema(&dbSchema); - return code; - } - } - } - - SSchema* pointColTs = taosArrayGet(pointSchema->fields, 0); - SSchema* dbColTs = taosArrayGet(dbSchema.fields, 0); - memcpy(pointColTs->name, dbColTs->name, TSDB_COL_NAME_LEN); - - for (int j = 1; j < pointFieldSize; ++j) { - SSchema* pointCol = taosArrayGet(pointSchema->fields, j); - SSchemaAction schemaAction = {0}; - bool actionNeeded = false; - generateSchemaAction(pointCol, dbFieldHash, dbSchema.fields,false, pointSchema->sTableName, - &schemaAction, &actionNeeded, info); - if (actionNeeded) { - code = applySchemaAction(taos, &schemaAction, info); - if (code != 0) { - destroySmlSTableSchema(&dbSchema); - return code; - } - } - } - - pointSchema->precision = dbSchema.precision; - - destroySmlSTableSchema(&dbSchema); - } else { - tscError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code)); - return code; - } - } - return 0; -} - -static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, const char* sTableName, - SArray* tagsSchema, SArray* tagsBind, SSmlLinesInfo* info) { - size_t numTags = taosArrayGetSize(tagsSchema); - char* sql = malloc(tsMaxSQLStringLen+1); - if (sql == NULL) { - tscError("malloc sql memory error"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - int freeBytes = tsMaxSQLStringLen + 1; - sprintf(sql, "create table if not exists %s using %s", cTableName, sTableName); - - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "("); - for (int i = 0; i < numTags; ++i) { - SSchema* tagSchema = taosArrayGet(tagsSchema, i); - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", tagSchema->name); - } - snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ")"); - - snprintf(sql + strlen(sql), freeBytes-strlen(sql), " tags ("); - - for (int i = 0; i < numTags; ++i) { - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); - } - snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ")"); - sql[strlen(sql)] = '\0'; - - tscDebug("SML:0x%"PRIx64" create table : %s", info->id, sql); - - TAOS_STMT* stmt = taos_stmt_init(taos); - if (stmt == NULL) { - free(sql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - int32_t code; - code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); - free(sql); - - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_prepare returns %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - code = taos_stmt_bind_param(stmt, TARRAY_GET_START(tagsBind)); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_bind_param returns %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - code = taos_stmt_execute(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_execute returns %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - code = taos_stmt_close(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_close return %d:%s", info->id, code, tstrerror(code)); - return code; - } - return code; -} - -static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* colsSchema, SArray* rowsBind, SSmlLinesInfo* info) { - size_t numCols = taosArrayGetSize(colsSchema); - char* sql = malloc(tsMaxSQLStringLen+1); - if (sql == NULL) { - tscError("malloc sql memory error"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int32_t freeBytes = tsMaxSQLStringLen + 1 ; - sprintf(sql, "insert into ? ("); - - for (int i = 0; i < numCols; ++i) { - SSchema* colSchema = taosArrayGet(colsSchema, i); - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", colSchema->name); - } - snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ") values ("); - - for (int i = 0; i < numCols; ++i) { - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); - } - snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ")"); - sql[strlen(sql)] = '\0'; - - tscDebug("SML:0x%"PRIx64" insert rows into child table %s. num of rows: %zu", info->id, cTableName, taosArrayGetSize(rowsBind)); - - int32_t code = 0; - - TAOS_STMT* stmt = taos_stmt_init(taos); - if (stmt == NULL) { - tfree(sql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); - tfree(sql); - - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - bool tryAgain = false; - int32_t try = 0; - do { - code = taos_stmt_set_tbname(stmt, cTableName); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_set_tbname return %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - size_t rows = taosArrayGetSize(rowsBind); - for (int32_t i = 0; i < rows; ++i) { - TAOS_BIND* colsBinds = taosArrayGetP(rowsBind, i); - code = taos_stmt_bind_param(stmt, colsBinds); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - code = taos_stmt_add_batch(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_add_batch return %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - } - - code = taos_stmt_execute(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s, try:%d", info->id, code, tstrerror(code), try); - } - - tryAgain = false; - if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID - || code == TSDB_CODE_VND_INVALID_VGROUP_ID - || code == TSDB_CODE_TDB_TABLE_RECONFIGURE - || code == TSDB_CODE_APP_NOT_READY - || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) { - tryAgain = true; - } - - if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - int32_t code2 = taos_errno(res2); - if (code2 != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - if (tryAgain) { - taosMsleep(50 * (2 << try)); - } - } - if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - if (tryAgain) { - taosMsleep( 50 * (2 << try)); - } - } - } while (tryAgain); - - - taos_stmt_close(stmt); - return code; -} - -static int32_t arrangePointsByChildTableName(TAOS_SML_DATA_POINT* points, int numPoints, - SHashObj* cname2points, SArray* stableSchemas, SSmlLinesInfo* info) { - for (int32_t i = 0; i < numPoints; ++i) { - TAOS_SML_DATA_POINT * point = points + i; - uintptr_t valPointer = (uintptr_t)point; - size_t* pSchemaIndex = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); - assert(pSchemaIndex != NULL); - SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, *pSchemaIndex); - - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* kv = point->tags + j; - if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t ts = *(int64_t*)(kv->value); - ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision); - *(int64_t*)(kv->value) = ts; - } - } - - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* kv = point->fields + j; - if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t ts = *(int64_t*)(kv->value); - ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision); - *(int64_t*)(kv->value) = ts; - } - } - - SArray* cTablePoints = NULL; - SArray** pCTablePoints = taosHashGet(cname2points, point->childTableName, strlen(point->childTableName)); - if (pCTablePoints) { - cTablePoints = *pCTablePoints; - } else { - cTablePoints = taosArrayInit(64, sizeof(point)); - taosHashPut(cname2points, point->childTableName, strlen(point->childTableName), &cTablePoints, POINTER_BYTES); - } - taosArrayPush(cTablePoints, &point); - } - - return 0; -} - -static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableName, - SSmlSTableSchema* sTableSchema, SArray* cTablePoints, SSmlLinesInfo* info) { - size_t numTags = taosArrayGetSize(sTableSchema->tags); - size_t rows = taosArrayGetSize(cTablePoints); - - TAOS_SML_KV* tagKVs[TSDB_MAX_TAGS] = {0}; - for (int i= 0; i < rows; ++i) { - TAOS_SML_DATA_POINT * pDataPoint = taosArrayGetP(cTablePoints, i); - for (int j = 0; j < pDataPoint->tagNum; ++j) { - TAOS_SML_KV* kv = pDataPoint->tags + j; - uintptr_t valPointer = (uintptr_t)kv; - size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); - assert(pFieldSchemaIdx != NULL); - tagKVs[*pFieldSchemaIdx] = kv; - } - } - - SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); - taosArraySetSize(tagBinds, numTags); - int isNullColBind = TSDB_TRUE; - for (int j = 0; j < numTags; ++j) { - TAOS_BIND* bind = taosArrayGet(tagBinds, j); - bind->is_null = &isNullColBind; - } - for (int j = 0; j < numTags; ++j) { - if (tagKVs[j] == NULL) continue; - TAOS_SML_KV* kv = tagKVs[j]; - uintptr_t valPointer = (uintptr_t)kv; - size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); - assert(pFieldSchemaIdx != NULL); - TAOS_BIND* bind = taosArrayGet(tagBinds, *pFieldSchemaIdx); - bind->buffer_type = kv->type; - bind->length = malloc(sizeof(uintptr_t*)); - *bind->length = kv->length; - bind->buffer = kv->value; - bind->is_null = NULL; - } - - int32_t code = creatChildTableIfNotExists(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, info); - - for (int i = 0; i < taosArrayGetSize(tagBinds); ++i) { - TAOS_BIND* bind = taosArrayGet(tagBinds, i); - free(bind->length); - } - taosArrayDestroy(tagBinds); - return code; -} - -static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, char* cTableName, - SArray* cTablePoints, SSmlLinesInfo* info) { - int32_t code = TSDB_CODE_SUCCESS; - - size_t numCols = taosArrayGetSize(sTableSchema->fields); - size_t rows = taosArrayGetSize(cTablePoints); - SArray* rowsBind = taosArrayInit(rows, POINTER_BYTES); - - for (int i = 0; i < rows; ++i) { - TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, i); - - TAOS_BIND* colBinds = calloc(numCols, sizeof(TAOS_BIND)); - if (colBinds == NULL) { - tscError("SML:0x%"PRIx64" taos_sml_insert insert points, failed to allocated memory for TAOS_BIND, " - "num of rows: %zu, num of cols: %zu", info->id, rows, numCols); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int isNullColBind = TSDB_TRUE; - for (int j = 0; j < numCols; ++j) { - TAOS_BIND* bind = colBinds + j; - bind->is_null = &isNullColBind; - } - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* kv = point->fields + j; - uintptr_t valPointer = (uintptr_t)kv; - size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); - assert(pFieldSchemaIdx != NULL); - TAOS_BIND* bind = colBinds + *pFieldSchemaIdx; - bind->buffer_type = kv->type; - bind->length = malloc(sizeof(uintptr_t*)); - *bind->length = kv->length; - bind->buffer = kv->value; - bind->is_null = NULL; - } - taosArrayPush(rowsBind, &colBinds); - } - - code = insertChildTableBatch(taos, cTableName, sTableSchema->fields, rowsBind, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" insert into child table %s failed. error %s", info->id, cTableName, tstrerror(code)); - } - - for (int i = 0; i < rows; ++i) { - TAOS_BIND* colBinds = taosArrayGetP(rowsBind, i); - for (int j = 0; j < numCols; ++j) { - TAOS_BIND* bind = colBinds + j; - free(bind->length); - } - free(colBinds); - } - taosArrayDestroy(rowsBind); - return code; -} - -static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints, SArray* stableSchemas, SSmlLinesInfo* info) { - int32_t code = TSDB_CODE_SUCCESS; - - SHashObj* cname2points = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - arrangePointsByChildTableName(points, numPoints, cname2points, stableSchemas, info); - - SArray** pCTablePoints = taosHashIterate(cname2points, NULL); - while (pCTablePoints) { - SArray* cTablePoints = *pCTablePoints; - - TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, 0); - uintptr_t valPointer = (uintptr_t)point; - size_t* pSchemaIndex = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); - assert(pSchemaIndex != NULL); - SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, *pSchemaIndex); - - tscDebug("SML:0x%"PRIx64" apply child table tags. child table: %s", info->id, point->childTableName); - code = applyChildTableTags(taos, point->childTableName, point->stableName, sTableSchema, cTablePoints, info); - if (code != 0) { - tscError("apply child table tags failed. child table %s, error %s", point->childTableName, tstrerror(code)); - goto cleanup; - } - - tscDebug("SML:0x%"PRIx64" apply child table points. child table: %s", info->id, point->childTableName); - code = applyChildTableFields(taos, sTableSchema, point->childTableName, cTablePoints, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" Apply child table fields failed. child table %s, error %s", info->id, point->childTableName, tstrerror(code)); - goto cleanup; - } - - tscDebug("SML:0x%"PRIx64" successfully applied data points of child table %s", info->id, point->childTableName); - - pCTablePoints = taosHashIterate(cname2points, pCTablePoints); - } - -cleanup: - pCTablePoints = taosHashIterate(cname2points, NULL); - while (pCTablePoints) { - SArray* pPoints = *pCTablePoints; - taosArrayDestroy(pPoints); - pCTablePoints = taosHashIterate(cname2points, pCTablePoints); - } - taosHashCleanup(cname2points); - return code; -} - -int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info) { - tscDebug("SML:0x%"PRIx64" taos_sml_insert. number of points: %d", info->id, numPoint); - - int32_t code = TSDB_CODE_SUCCESS; - info->smlDataToSchema = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, false); - - tscDebug("SML:0x%"PRIx64" build data point schemas", info->id); - SArray* stableSchemas = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray - code = buildDataPointSchemas(points, numPoint, stableSchemas, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" error building data point schemas : %s", info->id, tstrerror(code)); - goto clean_up; - } - - tscDebug("SML:0x%"PRIx64" modify db schemas", info->id); - code = modifyDBSchemas(taos, stableSchemas, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" error change db schema : %s", info->id, tstrerror(code)); - goto clean_up; - } - - tscDebug("SML:0x%"PRIx64" apply data points", info->id); - code = applyDataPoints(taos, points, numPoint, stableSchemas, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" error apply data points : %s", info->id, tstrerror(code)); - } - -clean_up: - for (int i = 0; i < taosArrayGetSize(stableSchemas); ++i) { - SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); - taosArrayDestroy(schema->fields); - taosArrayDestroy(schema->tags); - } - taosArrayDestroy(stableSchemas); - taosHashCleanup(info->smlDataToSchema); - return code; -} - -int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { - SSmlLinesInfo* info = calloc(1, sizeof(SSmlLinesInfo)); - info->id = genLinesSmlId(); - int code = tscSmlInsert(taos, points, numPoint, info); - free(info); - return code; -} - -//========================================================================= - -/* Field Escape charaters - 1: measurement Comma,Space - 2: tag_key, tag_value, field_key Comma,Equal Sign,Space - 3: field_value Double quote,Backslash -*/ -static void escapeSpecialCharacter(uint8_t field, const char **pos) { - const char *cur = *pos; - if (*cur != '\\') { - return; - } - switch (field) { - case 1: - switch (*(cur + 1)) { - case ',': - case ' ': - cur++; - break; - default: - break; - } - break; - case 2: - switch (*(cur + 1)) { - case ',': - case ' ': - case '=': - cur++; - break; - default: - break; - } - break; - case 3: - switch (*(cur + 1)) { - case '"': - case '\\': - cur++; - break; - default: - break; - } - break; - default: - break; - } - *pos = cur; -} - -static bool isValidInteger(char *str) { - char *c = str; - if (*c != '+' && *c != '-' && !isdigit(*c)) { - return false; - } - c++; - while (*c != '\0') { - if (!isdigit(*c)) { - return false; - } - c++; - } - return true; -} - -static bool isValidFloat(char *str) { - char *c = str; - uint8_t has_dot, has_exp, has_sign; - has_dot = 0; - has_exp = 0; - has_sign = 0; - - if (*c != '+' && *c != '-' && *c != '.' && !isdigit(*c)) { - return false; - } - if (*c == '.' && isdigit(*(c + 1))) { - has_dot = 1; - } - c++; - while (*c != '\0') { - if (!isdigit(*c)) { - switch (*c) { - case '.': { - if (!has_dot && !has_exp && isdigit(*(c + 1))) { - has_dot = 1; - } else { - return false; - } - break; - } - case 'e': - case 'E': { - if (!has_exp && isdigit(*(c - 1)) && - (isdigit(*(c + 1)) || - *(c + 1) == '+' || - *(c + 1) == '-')) { - has_exp = 1; - } else { - return false; - } - break; - } - case '+': - case '-': { - if (!has_sign && has_exp && isdigit(*(c + 1))) { - has_sign = 1; - } else { - return false; - } - break; - } - default: { - return false; - } - } - } - c++; - } //while - return true; -} - -static bool isTinyInt(char *pVal, uint16_t len) { - if (len <= 2) { - return false; - } - if (!strcmp(&pVal[len - 2], "i8")) { - //printf("Type is int8(%s)\n", pVal); - return true; - } - return false; -} - -static bool isTinyUint(char *pVal, uint16_t len) { - if (len <= 2) { - return false; - } - if (pVal[0] == '-') { - return false; - } - if (!strcmp(&pVal[len - 2], "u8")) { - //printf("Type is uint8(%s)\n", pVal); - return true; - } - return false; -} - -static bool isSmallInt(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (!strcmp(&pVal[len - 3], "i16")) { - //printf("Type is int16(%s)\n", pVal); - return true; - } - return false; -} - -static bool isSmallUint(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (pVal[0] == '-') { - return false; - } - if (strcmp(&pVal[len - 3], "u16") == 0) { - //printf("Type is uint16(%s)\n", pVal); - return true; - } - return false; -} - -static bool isInt(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (strcmp(&pVal[len - 3], "i32") == 0) { - //printf("Type is int32(%s)\n", pVal); - return true; - } - return false; -} - -static bool isUint(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (pVal[0] == '-') { - return false; - } - if (strcmp(&pVal[len - 3], "u32") == 0) { - //printf("Type is uint32(%s)\n", pVal); - return true; - } - return false; -} - -static bool isBigInt(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (strcmp(&pVal[len - 3], "i64") == 0) { - //printf("Type is int64(%s)\n", pVal); - return true; - } - return false; -} - -static bool isBigUint(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (pVal[0] == '-') { - return false; - } - if (strcmp(&pVal[len - 3], "u64") == 0) { - //printf("Type is uint64(%s)\n", pVal); - return true; - } - return false; -} - -static bool isFloat(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (strcmp(&pVal[len - 3], "f32") == 0) { - //printf("Type is float(%s)\n", pVal); - return true; - } - return false; -} - -static bool isDouble(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (strcmp(&pVal[len - 3], "f64") == 0) { - //printf("Type is double(%s)\n", pVal); - return true; - } - return false; -} - -static bool isBool(char *pVal, uint16_t len, bool *bVal) { - if ((len == 1) && - (pVal[len - 1] == 't' || - pVal[len - 1] == 'T')) { - //printf("Type is bool(%c)\n", pVal[len - 1]); - *bVal = true; - return true; - } - - if ((len == 1) && - (pVal[len - 1] == 'f' || - pVal[len - 1] == 'F')) { - //printf("Type is bool(%c)\n", pVal[len - 1]); - *bVal = false; - return true; - } - - if((len == 4) && - (!strcmp(&pVal[len - 4], "true") || - !strcmp(&pVal[len - 4], "True") || - !strcmp(&pVal[len - 4], "TRUE"))) { - //printf("Type is bool(%s)\n", &pVal[len - 4]); - *bVal = true; - return true; - } - if((len == 5) && - (!strcmp(&pVal[len - 5], "false") || - !strcmp(&pVal[len - 5], "False") || - !strcmp(&pVal[len - 5], "FALSE"))) { - //printf("Type is bool(%s)\n", &pVal[len - 5]); - *bVal = false; - return true; - } - return false; -} - -static bool isBinary(char *pVal, uint16_t len) { - //binary: "abc" - if (len < 2) { - return false; - } - //binary - if (pVal[0] == '"' && pVal[len - 1] == '"') { - //printf("Type is binary(%s)\n", pVal); - return true; - } - return false; -} - -static bool isNchar(char *pVal, uint16_t len) { - //nchar: L"abc" - if (len < 3) { - return false; - } - if (pVal[0] == 'L' && pVal[1] == '"' && pVal[len - 1] == '"') { - //printf("Type is nchar(%s)\n", pVal); - return true; - } - return false; -} - -static bool isTimeStamp(char *pVal, uint16_t len, SMLTimeStampType *tsType) { - if (len == 0) { - return true; - } - if ((len == 1) && pVal[0] == '0') { - *tsType = SML_TIME_STAMP_NOW; - //printf("Type is timestamp(%s)\n", pVal); - return true; - } - if (len < 2) { - return false; - } - //No appendix use usec as default - if (isdigit(pVal[len - 1]) && isdigit(pVal[len - 2])) { - *tsType = SML_TIME_STAMP_MICRO_SECONDS; - //printf("Type is timestamp(%s)\n", pVal); - return true; - } - if (pVal[len - 1] == 's') { - switch (pVal[len - 2]) { - case 'm': - *tsType = SML_TIME_STAMP_MILLI_SECONDS; - break; - case 'u': - *tsType = SML_TIME_STAMP_MICRO_SECONDS; - break; - case 'n': - *tsType = SML_TIME_STAMP_NANO_SECONDS; - break; - default: - if (isdigit(pVal[len - 2])) { - *tsType = SML_TIME_STAMP_SECONDS; - break; - } else { - return false; - } - } - //printf("Type is timestamp(%s)\n", pVal); - return true; - } - return false; -} - -static bool convertStrToNumber(TAOS_SML_KV *pVal, char*str, SSmlLinesInfo* info) { - errno = 0; - uint8_t type = pVal->type; - int16_t length = pVal->length; - int64_t val_s; - uint64_t val_u; - double val_d; - - if (IS_FLOAT_TYPE(type)) { - val_d = strtod(str, NULL); - } else { - if (IS_SIGNED_NUMERIC_TYPE(type)) { - val_s = strtoll(str, NULL, 10); - } else { - val_u = strtoull(str, NULL, 10); - } - } - - if (errno == ERANGE) { - tscError("SML:0x%"PRIx64" Convert number(%s) out of range", info->id, str); - return false; - } - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - if (!IS_VALID_TINYINT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int8_t *)(pVal->value) = (int8_t)val_s; - break; - case TSDB_DATA_TYPE_UTINYINT: - if (!IS_VALID_UTINYINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint8_t *)(pVal->value) = (uint8_t)val_u; - break; - case TSDB_DATA_TYPE_SMALLINT: - if (!IS_VALID_SMALLINT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int16_t *)(pVal->value) = (int16_t)val_s; - break; - case TSDB_DATA_TYPE_USMALLINT: - if (!IS_VALID_USMALLINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint16_t *)(pVal->value) = (uint16_t)val_u; - break; - case TSDB_DATA_TYPE_INT: - if (!IS_VALID_INT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int32_t *)(pVal->value) = (int32_t)val_s; - break; - case TSDB_DATA_TYPE_UINT: - if (!IS_VALID_UINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint32_t *)(pVal->value) = (uint32_t)val_u; - break; - case TSDB_DATA_TYPE_BIGINT: - if (!IS_VALID_BIGINT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int64_t *)(pVal->value) = (int64_t)val_s; - break; - case TSDB_DATA_TYPE_UBIGINT: - if (!IS_VALID_UBIGINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint64_t *)(pVal->value) = (uint64_t)val_u; - break; - case TSDB_DATA_TYPE_FLOAT: - if (!IS_VALID_FLOAT(val_d)) { - return false; - } - pVal->value = calloc(length, 1); - *(float *)(pVal->value) = (float)val_d; - break; - case TSDB_DATA_TYPE_DOUBLE: - if (!IS_VALID_DOUBLE(val_d)) { - return false; - } - pVal->value = calloc(length, 1); - *(double *)(pVal->value) = (double)val_d; - break; - default: - return false; - } - return true; -} -//len does not include '\0' from value. -bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, - uint16_t len, SSmlLinesInfo* info) { - if (len <= 0) { - return false; - } - - //integer number - if (isTinyInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_TINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 2] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isTinyUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_UTINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 2] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isSmallInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_SMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isSmallUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_USMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_INT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_UINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isBigInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_BIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isBigUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_UBIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - //floating number - if (isFloat(value, len)) { - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidFloat(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isDouble(value, len)) { - pVal->type = TSDB_DATA_TYPE_DOUBLE; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidFloat(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - //binary - if (isBinary(value, len)) { - pVal->type = TSDB_DATA_TYPE_BINARY; - pVal->length = len - 2; - pVal->value = calloc(pVal->length, 1); - //copy after " - memcpy(pVal->value, value + 1, pVal->length); - return true; - } - //nchar - if (isNchar(value, len)) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length = len - 3; - pVal->value = calloc(pVal->length, 1); - //copy after L" - memcpy(pVal->value, value + 2, pVal->length); - return true; - } - //bool - bool bVal; - if (isBool(value, len, &bVal)) { - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = calloc(pVal->length, 1); - memcpy(pVal->value, &bVal, pVal->length); - return true; - } - //Handle default(no appendix) as float - if (isValidInteger(value) || isValidFloat(value)) { - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - if (!convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - return false; -} - -static int32_t getTimeStampValue(char *value, uint16_t len, - SMLTimeStampType type, int64_t *ts) { - - if (len >= 2) { - for (int i = 0; i < len - 2; ++i) { - if(!isdigit(value[i])) { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - } - } - //No appendix or no timestamp given (len = 0) - if (len >= 1 && isdigit(value[len - 1]) && type != SML_TIME_STAMP_NOW) { - type = SML_TIME_STAMP_MICRO_SECONDS; - } - if (len != 0) { - *ts = (int64_t)strtoll(value, NULL, 10); - } else { - type = SML_TIME_STAMP_NOW; - } - switch (type) { - case SML_TIME_STAMP_NOW: { - *ts = taosGetTimestampNs(); - break; - } - case SML_TIME_STAMP_SECONDS: { - *ts = (int64_t)(*ts * 1e9); - break; - } - case SML_TIME_STAMP_MILLI_SECONDS: { - *ts = convertTimePrecision(*ts, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); - break; - } - case SML_TIME_STAMP_MICRO_SECONDS: { - *ts = convertTimePrecision(*ts, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); - break; - } - case SML_TIME_STAMP_NANO_SECONDS: { - *ts = *ts * 1; - break; - } - default: { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - } - return TSDB_CODE_SUCCESS; -} - -int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, - uint16_t len, SSmlLinesInfo* info) { - int32_t ret; - SMLTimeStampType type; - int64_t tsVal; - - if (!isTimeStamp(value, len, &type)) { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - - ret = getTimeStampValue(value, len, type, &tsVal); - if (ret) { - return ret; - } - tscDebug("SML:0x%"PRIx64"Timestamp after conversion:%"PRId64, info->id, tsVal); - - pVal->type = TSDB_DATA_TYPE_TIMESTAMP; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = calloc(pVal->length, 1); - memcpy(pVal->value, &tsVal, pVal->length); - return TSDB_CODE_SUCCESS; -} - -static int32_t parseSmlTimeStamp(TAOS_SML_KV **pTS, const char **index, SSmlLinesInfo* info) { - const char *start, *cur; - int32_t ret = TSDB_CODE_SUCCESS; - int len = 0; - char key[] = "_ts"; - char *value = NULL; - - start = cur = *index; - *pTS = calloc(1, sizeof(TAOS_SML_KV)); - - while(*cur != '\0') { - cur++; - len++; - } - - if (len > 0) { - value = calloc(len + 1, 1); - memcpy(value, start, len); - } - - ret = convertSmlTimeStamp(*pTS, value, len, info); - if (ret) { - free(value); - free(*pTS); - return ret; - } - free(value); - - (*pTS)->key = calloc(sizeof(key), 1); - memcpy((*pTS)->key, key, sizeof(key)); - return ret; -} - -bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { - char *val = NULL; - char *cur = key; - char keyLower[TSDB_COL_NAME_LEN]; - size_t keyLen = 0; - while(*cur != '\0') { - keyLower[keyLen] = tolower(*cur); - keyLen++; - cur++; - } - keyLower[keyLen] = '\0'; - - val = taosHashGet(pHash, keyLower, keyLen); - if (val) { - tscError("SML:0x%"PRIx64" Duplicate key detected:%s", info->id, keyLower); - return true; - } - - uint8_t dummy_val = 0; - taosHashPut(pHash, keyLower, strlen(key), &dummy_val, sizeof(uint8_t)); - - return false; -} - -static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) { - const char *cur = *index; - char key[TSDB_COL_NAME_LEN + 1]; // +1 to avoid key[len] over write - uint16_t len = 0; - - //key field cannot start with digit - if (isdigit(*cur)) { - tscError("SML:0x%"PRIx64" Tag key cannnot start with digit", info->id); - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - while (*cur != '\0') { - if (len > TSDB_COL_NAME_LEN) { - tscError("SML:0x%"PRIx64" Key field cannot exceeds 65 characters", info->id); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - //unescaped '=' identifies a tag key - if (*cur == '=' && *(cur - 1) != '\\') { - break; - } - //Escape special character - if (*cur == '\\') { - escapeSpecialCharacter(2, &cur); - } - key[len] = *cur; - cur++; - len++; - } - key[len] = '\0'; - - if (checkDuplicateKey(key, pHash, info)) { - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - pKV->key = calloc(len + 1, 1); - memcpy(pKV->key, key, len + 1); - //tscDebug("SML:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); - *index = cur + 1; - return TSDB_CODE_SUCCESS; -} - - -static bool parseSmlValue(TAOS_SML_KV *pKV, const char **index, - bool *is_last_kv, SSmlLinesInfo* info) { - const char *start, *cur; - char *value = NULL; - uint16_t len = 0; - start = cur = *index; - - while (1) { - // unescaped ',' or ' ' or '\0' identifies a value - if ((*cur == ',' || *cur == ' ' || *cur == '\0') && *(cur - 1) != '\\') { - //unescaped ' ' or '\0' indicates end of value - *is_last_kv = (*cur == ' ' || *cur == '\0') ? true : false; - break; - } - //Escape special character - if (*cur == '\\') { - escapeSpecialCharacter(2, &cur); - } - cur++; - len++; - } - - value = calloc(len + 1, 1); - memcpy(value, start, len); - value[len] = '\0'; - if (!convertSmlValueType(pKV, value, len, info)) { - tscError("SML:0x%"PRIx64" Failed to convert sml value string(%s) to any type", - info->id, value); - //free previous alocated key field - free(pKV->key); - pKV->key = NULL; - free(value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - free(value); - - *index = (*cur == '\0') ? cur : cur + 1; - return TSDB_CODE_SUCCESS; -} - -static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index, - uint8_t *has_tags, SSmlLinesInfo* info) { - const char *cur = *index; - uint16_t len = 0; - - pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + 1, 1); // +1 to avoid 1772 line over write - if (pSml->stableName == NULL){ - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - if (isdigit(*cur)) { - tscError("SML:0x%"PRIx64" Measurement field cannnot start with digit", info->id); - free(pSml->stableName); - pSml->stableName = NULL; - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - while (*cur != '\0') { - if (len > TSDB_TABLE_NAME_LEN) { - tscError("SML:0x%"PRIx64" Measurement field cannot exceeds 193 characters", info->id); - free(pSml->stableName); - pSml->stableName = NULL; - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - //first unescaped comma or space identifies measurement - //if space detected first, meaning no tag in the input - if (*cur == ',' && *(cur - 1) != '\\') { - *has_tags = 1; - break; - } - if (*cur == ' ' && *(cur - 1) != '\\') { - break; - } - //Comma, Space, Backslash needs to be escaped if any - if (*cur == '\\') { - escapeSpecialCharacter(1, &cur); - } - pSml->stableName[len] = *cur; - cur++; - len++; - } - pSml->stableName[len] = '\0'; - *index = cur + 1; - tscDebug("SML:0x%"PRIx64" Stable name in measurement:%s|len:%d", info->id, pSml->stableName, len); - - return TSDB_CODE_SUCCESS; -} - -//Table name can only contain digits(0-9),alphebet(a-z),underscore(_) -int32_t isValidChildTableName(const char *pTbName, int16_t len) { - const char *cur = pTbName; - for (int i = 0; i < len; ++i) { - if(!isdigit(cur[i]) && !isalpha(cur[i]) && (cur[i] != '_')) { - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - } - return TSDB_CODE_SUCCESS; -} - - -static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs, - const char **index, bool isField, - TAOS_SML_DATA_POINT* smlData, SHashObj *pHash, - SSmlLinesInfo* info) { - const char *cur = *index; - int32_t ret = TSDB_CODE_SUCCESS; - TAOS_SML_KV *pkv; - bool is_last_kv = false; - - int32_t capacity = 0; - if (isField) { - capacity = 64; - *pKVs = calloc(capacity, sizeof(TAOS_SML_KV)); - // leave space for timestamp; - pkv = *pKVs; - pkv++; - } else { - capacity = 8; - *pKVs = calloc(capacity, sizeof(TAOS_SML_KV)); - pkv = *pKVs; - } - - while (*cur != '\0') { - ret = parseSmlKey(pkv, &cur, pHash, info); - if (ret) { - tscError("SML:0x%"PRIx64" Unable to parse key", info->id); - goto error; - } - ret = parseSmlValue(pkv, &cur, &is_last_kv, info); - if (ret) { - tscError("SML:0x%"PRIx64" Unable to parse value", info->id); - goto error; - } - if (!isField && - (strcasecmp(pkv->key, "ID") == 0) && pkv->type == TSDB_DATA_TYPE_BINARY) { - ret = isValidChildTableName(pkv->value, pkv->length); - if (ret) { - goto error; - } - smlData->childTableName = malloc( pkv->length + 1); - memcpy(smlData->childTableName, pkv->value, pkv->length); - smlData->childTableName[pkv->length] = '\0'; - free(pkv->key); - free(pkv->value); - } else { - *num_kvs += 1; - } - if (is_last_kv) { - goto done; - } - - //reallocate addtional memory for more kvs - TAOS_SML_KV *more_kvs = NULL; - - if (isField) { - if ((*num_kvs + 2) > capacity) { - capacity *= 3; capacity /= 2; - more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); - } else { - more_kvs = *pKVs; - } - } else { - if ((*num_kvs + 1) > capacity) { - capacity *= 3; capacity /= 2; - more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); - } else { - more_kvs = *pKVs; - } - } - - if (!more_kvs) { - goto error; - } - *pKVs = more_kvs; - //move pKV points to next TAOS_SML_KV block - if (isField) { - pkv = *pKVs + *num_kvs + 1; - } else { - pkv = *pKVs + *num_kvs; - } - } - goto done; - -error: - return ret; -done: - *index = cur; - return ret; -} - -static void moveTimeStampToFirstKv(TAOS_SML_DATA_POINT** smlData, TAOS_SML_KV *ts) { - TAOS_SML_KV* tsField = (*smlData)->fields; - tsField->length = ts->length; - tsField->type = ts->type; - tsField->value = malloc(ts->length); - tsField->key = malloc(strlen(ts->key) + 1); - memcpy(tsField->key, ts->key, strlen(ts->key) + 1); - memcpy(tsField->value, ts->value, ts->length); - (*smlData)->fieldNum = (*smlData)->fieldNum + 1; - - free(ts->key); - free(ts->value); - free(ts); -} - -int32_t tscParseLine(const char* sql, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) { - const char* index = sql; - int32_t ret = TSDB_CODE_SUCCESS; - uint8_t has_tags = 0; - TAOS_SML_KV *timestamp = NULL; - SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - ret = parseSmlMeasurement(smlData, &index, &has_tags, info); - if (ret) { - tscError("SML:0x%"PRIx64" Unable to parse measurement", info->id); - taosHashCleanup(keyHashTable); - return ret; - } - tscDebug("SML:0x%"PRIx64" Parse measurement finished, has_tags:%d", info->id, has_tags); - - //Parse Tags - if (has_tags) { - ret = parseSmlKvPairs(&smlData->tags, &smlData->tagNum, &index, false, smlData, keyHashTable, info); - if (ret) { - tscError("SML:0x%"PRIx64" Unable to parse tag", info->id); - taosHashCleanup(keyHashTable); - return ret; - } - } - tscDebug("SML:0x%"PRIx64" Parse tags finished, num of tags:%d", info->id, smlData->tagNum); - - //Parse fields - ret = parseSmlKvPairs(&smlData->fields, &smlData->fieldNum, &index, true, smlData, keyHashTable, info); - if (ret) { - tscError("SML:0x%"PRIx64" Unable to parse field", info->id); - taosHashCleanup(keyHashTable); - return ret; - } - tscDebug("SML:0x%"PRIx64" Parse fields finished, num of fields:%d", info->id, smlData->fieldNum); - taosHashCleanup(keyHashTable); - - //Parse timestamp - ret = parseSmlTimeStamp(×tamp, &index, info); - if (ret) { - tscError("SML:0x%"PRIx64" Unable to parse timestamp", info->id); - return ret; - } - moveTimeStampToFirstKv(&smlData, timestamp); - tscDebug("SML:0x%"PRIx64" Parse timestamp finished", info->id); - - return TSDB_CODE_SUCCESS; -} - -//========================================================================= - -void destroySmlDataPoint(TAOS_SML_DATA_POINT* point) { - for (int i=0; itagNum; ++i) { - free((point->tags+i)->key); - free((point->tags+i)->value); - } - free(point->tags); - for (int i=0; ifieldNum; ++i) { - free((point->fields+i)->key); - free((point->fields+i)->value); - } - free(point->fields); - free(point->stableName); - free(point->childTableName); -} - -int32_t tscParseLines(char* lines[], int numLines, SArray* points, SArray* failedLines, SSmlLinesInfo* info) { - for (int32_t i = 0; i < numLines; ++i) { - TAOS_SML_DATA_POINT point = {0}; - int32_t code = tscParseLine(lines[i], &point, info); - if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); - destroySmlDataPoint(&point); - return code; - } else { - tscDebug("SML:0x%"PRIx64" data point line parse success. line %d", info->id, i); - } - - taosArrayPush(points, &point); - } - return TSDB_CODE_SUCCESS; -} - -int taos_insert_lines(TAOS* taos, char* lines[], int numLines) { - int32_t code = 0; - - SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); - info->id = genLinesSmlId(); - - if (numLines <= 0 || numLines > 65536) { - tscError("SML:0x%"PRIx64" taos_insert_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); - tfree(info); - code = TSDB_CODE_TSC_APP_ERROR; - return code; - } - - for (int i = 0; i < numLines; ++i) { - if (lines[i] == NULL) { - tscError("SML:0x%"PRIx64" taos_insert_lines line %d is NULL", info->id, i); - tfree(info); - code = TSDB_CODE_TSC_APP_ERROR; - return code; - } - } - - SArray* lpPoints = taosArrayInit(numLines, sizeof(TAOS_SML_DATA_POINT)); - if (lpPoints == NULL) { - tscError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); - tfree(info); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - tscDebug("SML:0x%"PRIx64" taos_insert_lines begin inserting %d lines, first line: %s", info->id, numLines, lines[0]); - code = tscParseLines(lines, numLines, lpPoints, NULL, info); - size_t numPoints = taosArrayGetSize(lpPoints); - - if (code != 0) { - goto cleanup; - } - - TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); - code = tscSmlInsert(taos, points, (int)numPoints, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_sml_insert error: %s", info->id, tstrerror((code))); - } - -cleanup: - tscDebug("SML:0x%"PRIx64" taos_insert_lines finish inserting %d lines. code: %d", info->id, numLines, code); - points = TARRAY_GET_START(lpPoints); - numPoints = taosArrayGetSize(lpPoints); - for (int i=0; i -#include -#include -#include - -#include "../../../include/client/taos.h" -#include "cJSON.h" -#include "hash.h" - -#include "tscUtil.h" -#include "tsclient.h" -#include "tscLog.h" - -#include "tscParseLine.h" - -#define OTD_MAX_FIELDS_NUM 2 -#define OTD_JSON_SUB_FIELDS_NUM 2 -#define OTD_JSON_FIELDS_NUM 4 - -#define OTD_TIMESTAMP_COLUMN_NAME "ts" -#define OTD_METRIC_VALUE_COLUMN_NAME "value" - -/* telnet style API parser */ -static uint64_t HandleId = 0; - -static uint64_t genUID() { - uint64_t id; - - do { - id = atomic_add_fetch_64(&HandleId, 1); - } while (id == 0); - - return id; -} - -static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index, SSmlLinesInfo* info) { - const char *cur = *index; - uint16_t len = 0; - - pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN + 1, 1); // +1 to avoid 1772 line over write - if (pSml->stableName == NULL){ - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - if (isdigit(*cur)) { - tscError("OTD:0x%"PRIx64" Metric cannnot start with digit", info->id); - tfree(pSml->stableName); - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - while (*cur != '\0') { - if (len > TSDB_TABLE_NAME_LEN) { - tscError("OTD:0x%"PRIx64" Metric cannot exceeds 193 characters", info->id); - tfree(pSml->stableName); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - if (*cur == ' ') { - break; - } - - pSml->stableName[len] = *cur; - cur++; - len++; - } - if (len == 0 || *cur == '\0') { - tfree(pSml->stableName); - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - pSml->stableName[len] = '\0'; - *index = cur + 1; - tscDebug("OTD:0x%"PRIx64" Stable name in metric:%s|len:%d", info->id, pSml->stableName, len); - - return TSDB_CODE_SUCCESS; -} - -static int32_t parseTelnetTimeStamp(TAOS_SML_KV **pTS, int *num_kvs, const char **index, SSmlLinesInfo* info) { - //Timestamp must be the first KV to parse - assert(*num_kvs == 0); - - const char *start, *cur; - int32_t ret = TSDB_CODE_SUCCESS; - int len = 0; - char key[] = OTD_TIMESTAMP_COLUMN_NAME; - char *value = NULL; - - start = cur = *index; - //allocate fields for timestamp and value - *pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); - - while(*cur != '\0') { - if (*cur == ' ') { - break; - } - cur++; - len++; - } - - if (len > 0 && *cur != '\0') { - value = tcalloc(len + 1, 1); - memcpy(value, start, len); - } else { - tfree(*pTS); - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - ret = convertSmlTimeStamp(*pTS, value, len, info); - if (ret) { - tfree(value); - tfree(*pTS); - return ret; - } - tfree(value); - - (*pTS)->key = tcalloc(sizeof(key), 1); - memcpy((*pTS)->key, key, sizeof(key)); - - *num_kvs += 1; - *index = cur + 1; - - return ret; -} - -static int32_t parseTelnetMetricValue(TAOS_SML_KV **pKVs, int *num_kvs, const char **index, SSmlLinesInfo* info) { - //skip timestamp - TAOS_SML_KV *pVal = *pKVs + 1; - const char *start, *cur; - int32_t ret = TSDB_CODE_SUCCESS; - int len = 0; - char key[] = OTD_METRIC_VALUE_COLUMN_NAME; - char *value = NULL; - - start = cur = *index; - - while(*cur != '\0') { - if (*cur == ' ') { - break; - } - cur++; - len++; - } - - if (len > 0 && *cur != '\0') { - value = tcalloc(len + 1, 1); - memcpy(value, start, len); - } else { - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - if (!convertSmlValueType(pVal, value, len, info)) { - tscError("OTD:0x%"PRIx64" Failed to convert metric value string(%s) to any type", - info->id, value); - tfree(value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - tfree(value); - - pVal->key = tcalloc(sizeof(key), 1); - memcpy(pVal->key, key, sizeof(key)); - *num_kvs += 1; - - *index = cur + 1; - return ret; -} - -static int32_t parseTelnetTagKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) { - const char *cur = *index; - char key[TSDB_COL_NAME_LEN + 1]; // +1 to avoid key[len] over write - uint16_t len = 0; - - //key field cannot start with digit - if (isdigit(*cur)) { - tscError("OTD:0x%"PRIx64" Tag key cannnot start with digit", info->id); - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - while (*cur != '\0') { - if (len > TSDB_COL_NAME_LEN) { - tscError("OTD:0x%"PRIx64" Tag key cannot exceeds 65 characters", info->id); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - if (*cur == '=') { - break; - } - - key[len] = *cur; - cur++; - len++; - } - if (len == 0 || *cur == '\0') { - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - key[len] = '\0'; - - if (checkDuplicateKey(key, pHash, info)) { - return TSDB_CODE_TSC_DUP_TAG_NAMES; - } - - pKV->key = tcalloc(len + 1, 1); - memcpy(pKV->key, key, len + 1); - //tscDebug("OTD:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); - *index = cur + 1; - return TSDB_CODE_SUCCESS; -} - - -static int32_t parseTelnetTagValue(TAOS_SML_KV *pKV, const char **index, - bool *is_last_kv, SSmlLinesInfo* info) { - const char *start, *cur; - char *value = NULL; - uint16_t len = 0; - start = cur = *index; - - while (1) { - // ',' or '\0' identifies a value - if (*cur == ',' || *cur == '\0') { - // '\0' indicates end of value - *is_last_kv = (*cur == '\0') ? true : false; - break; - } - cur++; - len++; - } - - if (len == 0) { - tfree(pKV->key); - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - value = tcalloc(len + 1, 1); - memcpy(value, start, len); - value[len] = '\0'; - if (!convertSmlValueType(pKV, value, len, info)) { - tscError("OTD:0x%"PRIx64" Failed to convert sml value string(%s) to any type", - info->id, value); - //free previous alocated key field - tfree(pKV->key); - tfree(value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - tfree(value); - - *index = (*cur == '\0') ? cur : cur + 1; - return TSDB_CODE_SUCCESS; -} - -static int32_t parseTelnetTagKvs(TAOS_SML_KV **pKVs, int *num_kvs, - const char **index, char **childTableName, - SHashObj *pHash, SSmlLinesInfo* info) { - const char *cur = *index; - int32_t ret = TSDB_CODE_SUCCESS; - TAOS_SML_KV *pkv; - bool is_last_kv = false; - - int32_t capacity = 4; - *pKVs = tcalloc(capacity, sizeof(TAOS_SML_KV)); - pkv = *pKVs; - - while (*cur != '\0') { - ret = parseTelnetTagKey(pkv, &cur, pHash, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse key", info->id); - return ret; - } - ret = parseTelnetTagValue(pkv, &cur, &is_last_kv, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse value", info->id); - return ret; - } - if ((strcasecmp(pkv->key, "ID") == 0) && pkv->type == TSDB_DATA_TYPE_BINARY) { - ret = isValidChildTableName(pkv->value, pkv->length); - if (ret) { - return ret; - } - *childTableName = malloc(pkv->length + 1); - memcpy(*childTableName, pkv->value, pkv->length); - (*childTableName)[pkv->length] = '\0'; - tfree(pkv->key); - tfree(pkv->value); - } else { - *num_kvs += 1; - } - - if (is_last_kv) { - break; - } - - //reallocate addtional memory for more kvs - if ((*num_kvs + 1) > capacity) { - TAOS_SML_KV *more_kvs = NULL; - capacity *= 3; capacity /= 2; - more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); - if (!more_kvs) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - *pKVs = more_kvs; - } - - //move pKV points to next TAOS_SML_KV block - pkv = *pKVs + *num_kvs; - } - - return ret; -} - -int32_t tscParseTelnetLine(const char* line, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) { - const char* index = line; - int32_t ret = TSDB_CODE_SUCCESS; - - //Parse metric - ret = parseTelnetMetric(smlData, &index, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse metric", info->id); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse metric finished", info->id); - - //Parse timestamp - ret = parseTelnetTimeStamp(&smlData->fields, &smlData->fieldNum, &index, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse timestamp", info->id); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse timestamp finished", info->id); - - //Parse value - ret = parseTelnetMetricValue(&smlData->fields, &smlData->fieldNum, &index, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse metric value", info->id); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse metric value finished", info->id); - - //Parse tagKVs - SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - ret = parseTelnetTagKvs(&smlData->tags, &smlData->tagNum, &index, &smlData->childTableName, keyHashTable, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse tags", info->id); - taosHashCleanup(keyHashTable); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse tags finished", info->id); - taosHashCleanup(keyHashTable); - - - return TSDB_CODE_SUCCESS; -} - -int32_t tscParseTelnetLines(char* lines[], int numLines, SArray* points, SArray* failedLines, SSmlLinesInfo* info) { - for (int32_t i = 0; i < numLines; ++i) { - TAOS_SML_DATA_POINT point = {0}; - int32_t code = tscParseTelnetLine(lines[i], &point, info); - if (code != TSDB_CODE_SUCCESS) { - tscError("OTD:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); - destroySmlDataPoint(&point); - return code; - } else { - tscDebug("OTD:0x%"PRIx64" data point line parse success. line %d", info->id, i); - } - - taosArrayPush(points, &point); - } - return TSDB_CODE_SUCCESS; -} - -int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines) { - int32_t code = 0; - - SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); - info->id = genUID(); - - if (numLines <= 0 || numLines > 65536) { - tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); - tfree(info); - code = TSDB_CODE_TSC_APP_ERROR; - return code; - } - - for (int i = 0; i < numLines; ++i) { - if (lines[i] == NULL) { - tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines line %d is NULL", info->id, i); - tfree(info); - code = TSDB_CODE_TSC_APP_ERROR; - return code; - } - } - - SArray* lpPoints = taosArrayInit(numLines, sizeof(TAOS_SML_DATA_POINT)); - if (lpPoints == NULL) { - tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines failed to allocate memory", info->id); - tfree(info); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d lines, first line: %s", info->id, numLines, lines[0]); - code = tscParseTelnetLines(lines, numLines, lpPoints, NULL, info); - size_t numPoints = taosArrayGetSize(lpPoints); - - if (code != 0) { - goto cleanup; - } - - TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); - code = tscSmlInsert(taos, points, (int)numPoints, info); - if (code != 0) { - tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines error: %s", info->id, tstrerror((code))); - } - -cleanup: - tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines finish inserting %d lines. code: %d", info->id, numLines, code); - points = TARRAY_GET_START(lpPoints); - numPoints = taosArrayGetSize(lpPoints); - for (int i = 0; i < numPoints; ++i) { - destroySmlDataPoint(points+i); - } - - taosArrayDestroy(lpPoints); - - tfree(info); - return code; -} - -int taos_telnet_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { - SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); - info->id = genUID(); - int code = tscSmlInsert(taos, points, numPoint, info); - tfree(info); - return code; -} - - -/* telnet style API parser */ -int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { - cJSON *metric = cJSON_GetObjectItem(root, "metric"); - if (!cJSON_IsString(metric)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - size_t stableLen = strlen(metric->valuestring); - if (stableLen > TSDB_TABLE_NAME_LEN) { - tscError("OTD:0x%"PRIx64" Metric cannot exceeds 193 characters in JSON", info->id); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - pSml->stableName = tcalloc(stableLen + 1, sizeof(char)); - if (pSml->stableName == NULL){ - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - if (isdigit(metric->valuestring[0])) { - tscError("OTD:0x%"PRIx64" Metric cannnot start with digit in JSON", info->id); - tfree(pSml->stableName); - return TSDB_CODE_TSC_INVALID_JSON; - } - - tstrncpy(pSml->stableName, metric->valuestring, stableLen + 1); - - return TSDB_CODE_SUCCESS; - -} - -int32_t parseTimestampFromJSONObj(cJSON *root, int64_t *tsVal, SSmlLinesInfo* info) { - int32_t size = cJSON_GetArraySize(root); - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (!cJSON_IsNumber(value)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - *tsVal = value->valueint; - //if timestamp value is 0 use current system time - if (*tsVal == 0) { - *tsVal = taosGetTimestampNs(); - return TSDB_CODE_SUCCESS; - } - - size_t typeLen = strlen(type->valuestring); - if (typeLen == 1 && type->valuestring[0] == 's') { - //seconds - *tsVal = (int64_t)(*tsVal * 1e9); - } else if (typeLen == 2 && type->valuestring[1] == 's') { - switch (type->valuestring[0]) { - case 'm': - //milliseconds - *tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); - break; - case 'u': - //microseconds - *tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); - break; - case 'n': - //nanoseconds - *tsVal = *tsVal * 1; - break; - default: - return TSDB_CODE_TSC_INVALID_JSON; - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSmlLinesInfo* info) { - //Timestamp must be the first KV to parse - assert(*num_kvs == 0); - int64_t tsVal; - char key[] = OTD_TIMESTAMP_COLUMN_NAME; - - cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); - if (cJSON_IsNumber(timestamp)) { - //timestamp value 0 indicates current system time - if (timestamp->valueint == 0) { - tsVal = taosGetTimestampNs(); - } else { - tsVal = convertTimePrecision(timestamp->valueint, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); - } - } else if (cJSON_IsObject(timestamp)) { - int32_t ret = parseTimestampFromJSONObj(timestamp, &tsVal, info); - if (ret != TSDB_CODE_SUCCESS) { - tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); - return ret; - } - } else { - return TSDB_CODE_TSC_INVALID_JSON; - } - - //allocate fields for timestamp and value - *pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); - - - (*pTS)->key = tcalloc(sizeof(key), 1); - memcpy((*pTS)->key, key, sizeof(key)); - - (*pTS)->type = TSDB_DATA_TYPE_TIMESTAMP; - (*pTS)->length = (int16_t)tDataTypes[(*pTS)->type].bytes; - (*pTS)->value = tcalloc((*pTS)->length, 1); - memcpy((*pTS)->value, &tsVal, (*pTS)->length); - - *num_kvs += 1; - return TSDB_CODE_SUCCESS; - -} - -int32_t convertJSONBool(TAOS_SML_KV *pVal, char* typeStr, int64_t valueInt, SSmlLinesInfo* info) { - if (strcasecmp(typeStr, "bool") != 0) { - tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Bool", info->id, typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(bool *)(pVal->value) = valueInt ? true : false; - - return TSDB_CODE_SUCCESS; -} - -int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { - //tinyint - if (strcasecmp(typeStr, "i8") == 0 || - strcasecmp(typeStr, "tinyint") == 0) { - if (!IS_VALID_TINYINT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(tinyint)", info->id, value->valueint); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_TINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(int8_t *)(pVal->value) = (int8_t)(value->valueint); - return TSDB_CODE_SUCCESS; - } - //smallint - if (strcasecmp(typeStr, "i16") == 0 || - strcasecmp(typeStr, "smallint") == 0) { - if (!IS_VALID_SMALLINT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(smallint)", info->id, value->valueint); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_SMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(int16_t *)(pVal->value) = (int16_t)(value->valueint); - return TSDB_CODE_SUCCESS; - } - //int - if (strcasecmp(typeStr, "i32") == 0 || - strcasecmp(typeStr, "int") == 0) { - if (!IS_VALID_INT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(int)", info->id, value->valueint); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_INT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(int32_t *)(pVal->value) = (int32_t)(value->valueint); - return TSDB_CODE_SUCCESS; - } - //bigint - if (strcasecmp(typeStr, "i64") == 0 || - strcasecmp(typeStr, "bigint") == 0) { - if (!IS_VALID_BIGINT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(bigint)", info->id, value->valueint); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_BIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(int64_t *)(pVal->value) = (int64_t)(value->valueint); - return TSDB_CODE_SUCCESS; - } - //float - if (strcasecmp(typeStr, "f32") == 0 || - strcasecmp(typeStr, "float") == 0) { - if (!IS_VALID_FLOAT(value->valuedouble)) { - tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(float)", info->id, value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(float *)(pVal->value) = (float)(value->valuedouble); - return TSDB_CODE_SUCCESS; - } - //double - if (strcasecmp(typeStr, "f64") == 0 || - strcasecmp(typeStr, "double") == 0) { - if (!IS_VALID_DOUBLE(value->valuedouble)) { - tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(double)", info->id, value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_DOUBLE; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(double *)(pVal->value) = (double)(value->valuedouble); - return TSDB_CODE_SUCCESS; - } - - //if reach here means type is unsupported - tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Number", info->id, typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; -} - -int32_t convertJSONString(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { - if (strcasecmp(typeStr, "binary") == 0) { - pVal->type = TSDB_DATA_TYPE_BINARY; - } else if (strcasecmp(typeStr, "nchar") == 0) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - } else { - tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON String", info->id, typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->length = (int16_t)strlen(value->valuestring); - pVal->value = tcalloc(pVal->length + 1, 1); - memcpy(pVal->value, value->valuestring, pVal->length); - return TSDB_CODE_SUCCESS; -} - -int32_t parseValueFromJSONObj(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { - int32_t ret = TSDB_CODE_SUCCESS; - int32_t size = cJSON_GetArraySize(root); - - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (value == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - switch (value->type) { - case cJSON_True: - case cJSON_False: { - ret = convertJSONBool(pVal, type->valuestring, value->valueint, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - case cJSON_Number: { - ret = convertJSONNumber(pVal, type->valuestring, value, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - case cJSON_String: { - ret = convertJSONString(pVal, type->valuestring, value, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - default: - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { - int type = root->type; - - switch (type) { - case cJSON_True: - case cJSON_False: { - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(bool *)(pVal->value) = root->valueint ? true : false; - break; - } - case cJSON_Number: { - //convert default JSON Number type to float - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(float *)(pVal->value) = (float)(root->valuedouble); - break; - } - case cJSON_String: { - //convert default JSON String type to nchar - pVal->type = TSDB_DATA_TYPE_NCHAR; - //pVal->length = wcslen((wchar_t *)root->valuestring) * TSDB_NCHAR_SIZE; - pVal->length = (int16_t)strlen(root->valuestring); - pVal->value = tcalloc(pVal->length + 1, 1); - memcpy(pVal->value, root->valuestring, pVal->length); - break; - } - case cJSON_Object: { - int32_t ret = parseValueFromJSONObj(root, pVal, info); - if (ret != TSDB_CODE_SUCCESS) { - tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); - return ret; - } - break; - } - default: - return TSDB_CODE_TSC_INVALID_JSON; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, SSmlLinesInfo* info) { - //skip timestamp - TAOS_SML_KV *pVal = *pKVs + 1; - char key[] = OTD_METRIC_VALUE_COLUMN_NAME; - - cJSON *metricVal = cJSON_GetObjectItem(root, "value"); - if (metricVal == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - int32_t ret = parseValueFromJSON(metricVal, pVal, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - pVal->key = tcalloc(sizeof(key), 1); - memcpy(pVal->key, key, sizeof(key)); - - *num_kvs += 1; - return TSDB_CODE_SUCCESS; - -} - -int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char **childTableName, SSmlLinesInfo* info) { - int32_t ret = TSDB_CODE_SUCCESS; - - cJSON *tags = cJSON_GetObjectItem(root, "tags"); - if (tags == NULL || tags->type != cJSON_Object) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - //only pick up the first ID value as child table name - cJSON *id = cJSON_GetObjectItem(tags, "ID"); - if (id != NULL) { - size_t idLen = strlen(id->valuestring); - ret = isValidChildTableName(id->valuestring, (int16_t)idLen); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - *childTableName = tcalloc(idLen + 1, sizeof(char)); - memcpy(*childTableName, id->valuestring, idLen); - //remove all ID fields from tags list no case sensitive - while (id != NULL) { - cJSON_DeleteItemFromObject(tags, "ID"); - id = cJSON_GetObjectItem(tags, "ID"); - } - } - - int32_t tagNum = cJSON_GetArraySize(tags); - //at least one tag pair required - if (tagNum <= 0) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - //allocate memory for tags - *pKVs = tcalloc(tagNum, sizeof(TAOS_SML_KV)); - TAOS_SML_KV *pkv = *pKVs; - - for (int32_t i = 0; i < tagNum; ++i) { - cJSON *tag = cJSON_GetArrayItem(tags, i); - if (tag == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - //key - size_t keyLen = strlen(tag->string); - pkv->key = tcalloc(keyLen + 1, sizeof(char)); - strncpy(pkv->key, tag->string, keyLen); - //value - ret = parseValueFromJSON(tag, pkv, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - *num_kvs += 1; - pkv++; - } - - return ret; - -} - -int32_t tscParseJSONPayload(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { - int32_t ret = TSDB_CODE_SUCCESS; - - if (!cJSON_IsObject(root)) { - tscError("OTD:0x%"PRIx64" data point needs to be JSON object", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - - int32_t size = cJSON_GetArraySize(root); - //outmost json fields has to be exactly 4 - if (size != OTD_JSON_FIELDS_NUM) { - tscError("OTD:0x%"PRIx64" Invalid number of JSON fields in data point %d", info->id, size); - return TSDB_CODE_TSC_INVALID_JSON; - } - - //Parse metric - ret = parseMetricFromJSON(root, pSml, info); - if (ret != TSDB_CODE_SUCCESS) { - tscError("OTD:0x%"PRIx64" Unable to parse metric from JSON payload", info->id); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse metric from JSON payload finished", info->id); - - //Parse timestamp - ret = parseTimestampFromJSON(root, &pSml->fields, &pSml->fieldNum, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse timestamp from JSON payload", info->id); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse timestamp from JSON payload finished", info->id); - - //Parse metric value - ret = parseMetricValueFromJSON(root, &pSml->fields, &pSml->fieldNum, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse metric value from JSON payload", info->id); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse metric value from JSON payload finished", info->id); - - //Parse tags - ret = parseTagsFromJSON(root, &pSml->tags, &pSml->tagNum, &pSml->childTableName, info); - if (ret) { - tscError("OTD:0x%"PRIx64" Unable to parse tags from JSON payload", info->id); - return ret; - } - tscDebug("OTD:0x%"PRIx64" Parse tags from JSON payload finished", info->id); - - return TSDB_CODE_SUCCESS; -} - -int32_t tscParseMultiJSONPayload(char* payload, SArray* points, SSmlLinesInfo* info) { - int32_t payloadNum, ret; - ret = TSDB_CODE_SUCCESS; - - if (payload == NULL) { - tscError("OTD:0x%"PRIx64" empty JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *root = cJSON_Parse(payload); - //multiple data points must be sent in JSON array - if (cJSON_IsObject(root)) { - payloadNum = 1; - } else if (cJSON_IsArray(root)) { - payloadNum = cJSON_GetArraySize(root); - } else { - tscError("OTD:0x%"PRIx64" Invalid JSON Payload", info->id); - ret = TSDB_CODE_TSC_INVALID_JSON; - goto PARSE_JSON_OVER; - } - - for (int32_t i = 0; i < payloadNum; ++i) { - TAOS_SML_DATA_POINT point = {0}; - cJSON *dataPoint = (payloadNum == 1) ? root : cJSON_GetArrayItem(root, i); - - ret = tscParseJSONPayload(dataPoint, &point, info); - if (ret != TSDB_CODE_SUCCESS) { - tscError("OTD:0x%"PRIx64" JSON data point parse failed", info->id); - destroySmlDataPoint(&point); - goto PARSE_JSON_OVER; - } else { - tscDebug("OTD:0x%"PRIx64" JSON data point parse success", info->id); - } - taosArrayPush(points, &point); - } - -PARSE_JSON_OVER: - cJSON_Delete(root); - return ret; -} - -int taos_insert_json_payload(TAOS* taos, char* payload) { - int32_t code = 0; - - SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); - info->id = genUID(); - - if (payload == NULL) { - tscError("OTD:0x%"PRIx64" taos_insert_json_payload payload is NULL", info->id); - tfree(info); - code = TSDB_CODE_TSC_APP_ERROR; - return code; - } - - SArray* lpPoints = taosArrayInit(1, sizeof(TAOS_SML_DATA_POINT)); - if (lpPoints == NULL) { - tscError("OTD:0x%"PRIx64" taos_insert_json_payload failed to allocate memory", info->id); - tfree(info); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d points", info->id, 1); - code = tscParseMultiJSONPayload(payload, lpPoints, info); - size_t numPoints = taosArrayGetSize(lpPoints); - - if (code != 0) { - goto cleanup; - } - - TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); - code = tscSmlInsert(taos, points, (int)numPoints, info); - if (code != 0) { - tscError("OTD:0x%"PRIx64" taos_insert_json_payload error: %s", info->id, tstrerror((code))); - } - -cleanup: - tscDebug("OTD:0x%"PRIx64" taos_insert_json_payload finish inserting 1 Point. code: %d", info->id, code); - points = TARRAY_GET_START(lpPoints); - numPoints = taosArrayGetSize(lpPoints); - for (int i = 0; i < numPoints; ++i) { - destroySmlDataPoint(points+i); - } - - taosArrayDestroy(lpPoints); - - tfree(info); - return code; -} diff --git a/2.0/src/client/src/tscPrepare.c b/2.0/src/client/src/tscPrepare.c deleted file mode 100644 index ab582ec249ea43bdd78f6794a068481d5ecc7493..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscPrepare.c +++ /dev/null @@ -1,2075 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "os.h" - -#include "../../../include/client/taos.h" -#include "tmsg.h" -#include "tscLog.h" -#include "tscSubquery.h" -#include "tscUtil.h" -#include "tsclient.h" -#include "tstrbuild.h" -#include "ttimer.h" - -int tsParseInsertSql(SSqlObj *pSql); -int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start); - -//////////////////////////////////////////////////////////////////////////////// -// functions for normal statement preparation - -typedef struct SNormalStmtPart { - bool isParam; - uint32_t len; - char* str; -} SNormalStmtPart; - -typedef struct SNormalStmt { - uint16_t sizeParts; - uint16_t numParts; - uint16_t numParams; - char* sql; - SNormalStmtPart* parts; - tVariant* params; -} SNormalStmt; - -typedef struct SMultiTbStmt { - bool nameSet; - bool tagSet; - bool subSet; - uint64_t currentUid; - char *sqlstr; - uint32_t tbNum; - SStrToken tbname; - SStrToken stbname; - SStrToken values; - SArray *tags; - STableDataBlocks *lastBlock; - SHashObj *pTableHash; - SHashObj *pTableBlockHashList; // data block for each table -} SMultiTbStmt; - -typedef enum { - STMT_INIT = 1, - STMT_PREPARE, - STMT_SETTBNAME, - STMT_BIND, - STMT_BIND_COL, - STMT_ADD_BATCH, - STMT_EXECUTE -} STMT_ST; - -typedef struct STscStmt { - bool isInsert; - bool multiTbInsert; - int16_t last; - STscObj* taos; - SSqlObj* pSql; - SMultiTbStmt mtb; - SNormalStmt normal; -} STscStmt; - -#define STMT_RET(c) do { \ - int32_t _code = c; \ - if (pStmt && pStmt->pSql) { pStmt->pSql->res.code = _code; } else {terrno = _code;} \ - return _code; \ -} while (0) - -static int32_t invalidOperationMsg(char* dstBuffer, const char* errMsg) { - return tscInvalidOperationMsg(dstBuffer, errMsg, NULL); -} - -static int normalStmtAddPart(SNormalStmt* stmt, bool isParam, char* str, uint32_t len) { - uint16_t size = stmt->numParts + 1; - if (size > stmt->sizeParts) { - size *= 2; - void* tmp = realloc(stmt->parts, sizeof(SNormalStmtPart) * size); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - stmt->sizeParts = size; - stmt->parts = (SNormalStmtPart*)tmp; - } - - stmt->parts[stmt->numParts].isParam = isParam; - stmt->parts[stmt->numParts].str = str; - stmt->parts[stmt->numParts].len = len; - - ++stmt->numParts; - if (isParam) { - ++stmt->numParams; - } - return TSDB_CODE_SUCCESS; -} - -static int normalStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { - SNormalStmt* normal = &stmt->normal; - - for (uint16_t i = 0; i < normal->numParams; ++i) { - TAOS_BIND* tb = bind + i; - tVariant* var = normal->params + i; - tVariantDestroy(var); - - var->nLen = 0; - if (tb->is_null != NULL && *(tb->is_null)) { - var->nType = TSDB_DATA_TYPE_NULL; - var->i64 = 0; - continue; - } - - var->nType = tb->buffer_type; - switch (tb->buffer_type) { - case TSDB_DATA_TYPE_NULL: - var->i64 = 0; - break; - - case TSDB_DATA_TYPE_BOOL: - var->i64 = (*(int8_t*)tb->buffer) ? 1 : 0; - break; - - case TSDB_DATA_TYPE_TINYINT: - var->i64 = *(int8_t*)tb->buffer; - break; - - case TSDB_DATA_TYPE_SMALLINT: - var->i64 = *(int16_t*)tb->buffer; - break; - - case TSDB_DATA_TYPE_INT: - var->i64 = *(int32_t*)tb->buffer; - break; - - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - var->i64 = *(int64_t*)tb->buffer; - break; - - case TSDB_DATA_TYPE_FLOAT: - var->dKey = GET_FLOAT_VAL(tb->buffer); - break; - - case TSDB_DATA_TYPE_DOUBLE: - var->dKey = GET_DOUBLE_VAL(tb->buffer); - break; - - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - var->pz = (char*)malloc((*tb->length) + 1); - if (var->pz == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - memcpy(var->pz, tb->buffer, (*tb->length)); - var->pz[*tb->length] = 0; - var->nLen = (int32_t)(*tb->length); - break; - - default: - tscError("0x%"PRIx64" bind column%d: type mismatch or invalid", stmt->pSql->self, i); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "bind type mismatch or invalid"); - } - } - - return TSDB_CODE_SUCCESS; -} - - -static int normalStmtPrepare(STscStmt* stmt) { - SNormalStmt* normal = &stmt->normal; - char* sql = stmt->pSql->sqlstr; - uint32_t i = 0, start = 0; - - while (sql[i] != 0) { - SStrToken token = {0}; - token.n = tGetToken(sql + i, &token.type); - - if (token.type == TK_QUESTION) { - sql[i] = 0; - if (i > start) { - int code = normalStmtAddPart(normal, false, sql + start, i - start); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - int code = normalStmtAddPart(normal, true, NULL, 0); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - start = i + token.n; - } else if (token.type == TK_ILLEGAL) { - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "invalid sql"); - } - - i += token.n; - } - - if (i > start) { - int code = normalStmtAddPart(normal, false, sql + start, i - start); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - if (normal->numParams > 0) { - normal->params = calloc(normal->numParams, sizeof(tVariant)); - if (normal->params == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - return TSDB_CODE_SUCCESS; -} - -static char* normalStmtBuildSql(STscStmt* stmt) { - SNormalStmt* normal = &stmt->normal; - SStringBuilder sb; memset(&sb, 0, sizeof(sb)); - - if (taosStringBuilderSetJmp(&sb) != 0) { - taosStringBuilderDestroy(&sb); - return NULL; - } - - taosStringBuilderEnsureCapacity(&sb, 4096); - uint32_t idxParam = 0; - - for(uint16_t i = 0; i < normal->numParts; i++) { - SNormalStmtPart* part = normal->parts + i; - if (!part->isParam) { - taosStringBuilderAppendStringLen(&sb, part->str, part->len); - continue; - } - - tVariant* var = normal->params + idxParam++; - switch (var->nType) - { - case TSDB_DATA_TYPE_NULL: - taosStringBuilderAppendNull(&sb); - break; - - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - taosStringBuilderAppendInteger(&sb, var->i64); - break; - - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - taosStringBuilderAppendDouble(&sb, var->dKey); - break; - - case TSDB_DATA_TYPE_BINARY: - taosStringBuilderAppendChar(&sb, '\''); - for (char* p = var->pz; *p != 0; ++p) { - if (*p == '\'' || *p == '"') { - taosStringBuilderAppendChar(&sb, '\\'); - } - taosStringBuilderAppendChar(&sb, *p); - } - taosStringBuilderAppendChar(&sb, '\''); - break; - - case TSDB_DATA_TYPE_NCHAR: - taosStringBuilderAppendChar(&sb, '\''); - taosStringBuilderAppend(&sb, var->wpz, var->nLen); - taosStringBuilderAppendChar(&sb, '\''); - break; - - default: - assert(false); - break; - } - } - - return taosStringBuilderGetResult(&sb, NULL); -} - -static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) { - SParsedDataColInfo* spd = &pBlock->boundColumnInfo; - int32_t offset = 0; - SSchema *schema = (SSchema*)pBlock->pTableMeta->schema; - - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // current column do not have any value to insert, set it to null - for (int32_t n = 0; n < rowNum; ++n) { - char *ptr = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * n + offset; - - if (schema[i].type == TSDB_DATA_TYPE_BINARY) { - varDataSetLen(ptr, sizeof(int8_t)); - *(uint8_t*) varDataVal(ptr) = TSDB_DATA_BINARY_NULL; - } else if (schema[i].type == TSDB_DATA_TYPE_NCHAR) { - varDataSetLen(ptr, sizeof(int32_t)); - *(uint32_t*) varDataVal(ptr) = TSDB_DATA_NCHAR_NULL; - } else { - setNull(ptr, schema[i].type, schema[i].bytes); - } - } - } - - offset += schema[i].bytes; - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t fillTablesColumnsNull(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - - STableDataBlocks** p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, NULL); - - STableDataBlocks* pOneTableBlock = *p; - while(pOneTableBlock) { - SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; - if (pBlocks->numOfRows > 0 && pOneTableBlock->boundColumnInfo.numOfBound < pOneTableBlock->boundColumnInfo.numOfCols) { - fillColumnsNull(pOneTableBlock, pBlocks->numOfRows); - } - - p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, p); - if (p == NULL) { - break; - } - - pOneTableBlock = *p; - } - - return TSDB_CODE_SUCCESS; -} - - - -//////////////////////////////////////////////////////////////////////////////// -// functions for insertion statement preparation -static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind, int32_t colNum) { - if (bind->is_null != NULL && *(bind->is_null)) { - setNull(data + param->offset, param->type, param->bytes); - return TSDB_CODE_SUCCESS; - } - -#if 0 - if (0) { - // allow user bind param data with different type - union { - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - unsigned char buf[32*1024]; - } u; - switch (param->type) { - case TSDB_DATA_TYPE_BOOL: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BOOL: { - u.v1 = *(int8_t*)bind->buffer; - if (u.v1==0 || u.v1==1) break; - } break; - case TSDB_DATA_TYPE_TINYINT: { - u.v1 = *(int8_t*)bind->buffer; - if (u.v1==0 || u.v1==1) break; - } break; - case TSDB_DATA_TYPE_SMALLINT: { - u.v1 = (int8_t)*(int16_t*)bind->buffer; - if (u.v1==0 || u.v1==1) break; - } break; - case TSDB_DATA_TYPE_INT: { - u.v1 = (int8_t)*(int32_t*)bind->buffer; - if (u.v1==0 || u.v1==1) break; - } break; - case TSDB_DATA_TYPE_BIGINT: { - u.v1 = (int8_t)*(int64_t*)bind->buffer; - if (u.v1==0 || u.v1==1) break; - } break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - // "0", "1" convertible - if (strncmp((const char*)bind->buffer, "0", *bind->length)==0) { - u.v1 = 0; - break; - } - if (strncmp((const char*)bind->buffer, "1", *bind->length)==0) { - u.v1 = 1; - break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - memcpy(data + param->offset, &u.v1, sizeof(u.v1)); - return TSDB_CODE_SUCCESS; - } break; - case TSDB_DATA_TYPE_TINYINT: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: { - int8_t v = *(int8_t*)bind->buffer; - u.v1 = v; - if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; - } break; - case TSDB_DATA_TYPE_SMALLINT: { - int16_t v = *(int16_t*)bind->buffer; - u.v1 = (int8_t)v; - if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_INT: { - int32_t v = *(int32_t*)bind->buffer; - u.v1 = (int8_t)v; - if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t v = *(int64_t*)bind->buffer; - u.v1 = (int8_t)v; - if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - int64_t v; - int n, r; - r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r == 1 && n == strlen((const char*)bind->buffer)) { - u.v1 = (int8_t)v; - if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - memcpy(data + param->offset, &u.v1, sizeof(u.v1)); - return TSDB_CODE_SUCCESS; - } - case TSDB_DATA_TYPE_SMALLINT: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: { - int v = *(int16_t*)bind->buffer; - u.v2 = (int16_t)v; - } break; - case TSDB_DATA_TYPE_INT: { - int32_t v = *(int32_t*)bind->buffer; - u.v2 = (int16_t)v; - if (v >= SHRT_MIN && v <= SHRT_MAX) break; - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t v = *(int64_t*)bind->buffer; - u.v2 = (int16_t)v; - if (v >= SHRT_MIN && v <= SHRT_MAX) break; - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - int64_t v; - int n, r; - r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r == 1 && n == strlen((const char*)bind->buffer)) { - u.v2 = (int16_t)v; - if (v >= SHRT_MIN && v <= SHRT_MAX) break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - memcpy(data + param->offset, &u.v2, sizeof(u.v2)); - return TSDB_CODE_SUCCESS; - } - case TSDB_DATA_TYPE_INT: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: { - u.v4 = *(int32_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_BIGINT: { - int64_t v = *(int64_t*)bind->buffer; - u.v4 = (int32_t)v; - if (v >= INT_MIN && v <= INT_MAX) break; - return TSDB_CODE_TSC_INVALID_VALUE; - } break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - int64_t v; - int n,r; - r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { - u.v4 = (int32_t)v; - if (v >= INT_MIN && v <= INT_MAX) break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } break; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } break; - } - memcpy(data + param->offset, &u.v2, sizeof(u.v2)); - return TSDB_CODE_SUCCESS; - } break; - case TSDB_DATA_TYPE_FLOAT: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: { - u.f4 = *(int8_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_SMALLINT: { - u.f4 = *(int16_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_INT: { - u.f4 = (float)*(int32_t*)bind->buffer; - // shall we check equality? - } break; - case TSDB_DATA_TYPE_BIGINT: { - u.f4 = (float)*(int64_t*)bind->buffer; - // shall we check equality? - } break; - case TSDB_DATA_TYPE_FLOAT: { - u.f4 = *(float*)bind->buffer; - } break; - case TSDB_DATA_TYPE_DOUBLE: { - u.f4 = *(float*)bind->buffer; - // shall we check equality? - } break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - float v; - int n,r; - r = sscanf((const char*)bind->buffer, "%f%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { - u.f4 = v; - break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } break; - case TSDB_DATA_TYPE_TIMESTAMP: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } break; - } - memcpy(data + param->offset, &u.f4, sizeof(u.f4)); - return TSDB_CODE_SUCCESS; - } break; - case TSDB_DATA_TYPE_BIGINT: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: { - u.v8 = *(int8_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_SMALLINT: { - u.v8 = *(int16_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_INT: { - u.v8 = *(int32_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_BIGINT: { - u.v8 = *(int64_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - int64_t v; - int n,r; - r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { - u.v8 = v; - break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - memcpy(data + param->offset, &u.v8, sizeof(u.v8)); - return TSDB_CODE_SUCCESS; - } - case TSDB_DATA_TYPE_DOUBLE: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: { - u.f8 = *(int8_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_SMALLINT: { - u.f8 = *(int16_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_INT: { - u.f8 = *(int32_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_BIGINT: { - u.f8 = (double)*(int64_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_FLOAT: { - u.f8 = *(float*)bind->buffer; - } break; - case TSDB_DATA_TYPE_DOUBLE: { - u.f8 = *(double*)bind->buffer; - } break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - double v; - int n,r; - r = sscanf((const char*)bind->buffer, "%lf%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { - u.f8 = v; - break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } - case TSDB_DATA_TYPE_TIMESTAMP: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - memcpy(data + param->offset, &u.f8, sizeof(u.f8)); - return TSDB_CODE_SUCCESS; - } - case TSDB_DATA_TYPE_TIMESTAMP: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_TIMESTAMP: { - u.v8 = *(int64_t*)bind->buffer; - } break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - // is this the correct way to call taosParseTime? - int32_t len = (int32_t)*bind->length; - if (taosParseTime(bind->buffer, &u.v8, len, 3, tsDaylight) == TSDB_CODE_SUCCESS) { - break; - } - return TSDB_CODE_TSC_INVALID_VALUE; - } break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_DOUBLE: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } break; - }; - memcpy(data + param->offset, &u.v8, sizeof(u.v8)); - return TSDB_CODE_SUCCESS; - } - case TSDB_DATA_TYPE_BINARY: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_BINARY: { - if ((*bind->length) > (uintptr_t)param->bytes) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - short size = (short)*bind->length; - STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer, size); - return TSDB_CODE_SUCCESS; - } - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_NCHAR: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - } - case TSDB_DATA_TYPE_NCHAR: { - switch (bind->buffer_type) { - case TSDB_DATA_TYPE_NCHAR: { - int32_t output = 0; - if (!taosMbsToUcs4(bind->buffer, *bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - varDataSetLen(data + param->offset, output); - return TSDB_CODE_SUCCESS; - } - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BINARY: - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - } - default: { - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - } -#endif - - if (bind->buffer_type != param->type) { - tscError("column type mismatch"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - short size = 0; - switch(param->type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *)(data + param->offset) = *(uint8_t *)bind->buffer; - break; - - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t *)(data + param->offset) = *(uint16_t *)bind->buffer; - break; - - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_FLOAT: - *(uint32_t *)(data + param->offset) = *(uint32_t *)bind->buffer; - break; - - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - *(uint64_t *)(data + param->offset) = *(uint64_t *)bind->buffer; - break; - - case TSDB_DATA_TYPE_BINARY: - if ((*bind->length) > (uintptr_t)param->bytes) { - tscError("column length is too big"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - size = (short)*bind->length; - STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer, size); - return TSDB_CODE_SUCCESS; - - case TSDB_DATA_TYPE_NCHAR: { - int32_t output = 0; - if (!taosMbsToUcs4(bind->buffer, *bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { - tscError("convert nchar failed"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - varDataSetLen(data + param->offset, output); - return TSDB_CODE_SUCCESS; - } - default: - assert(false); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (param->offset == 0) { - if (tsCheckTimestamp(pBlock, data + param->offset) != TSDB_CODE_SUCCESS) { - tscError("invalid timestamp"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t insertStmtGenLastBlock(STableDataBlocks** lastBlock, STableDataBlocks* pBlock) { - *lastBlock = (STableDataBlocks*)malloc(sizeof(STableDataBlocks)); - memcpy(*lastBlock, pBlock, sizeof(STableDataBlocks)); - (*lastBlock)->cloned = true; - - (*lastBlock)->pData = NULL; - (*lastBlock)->ordered = true; - (*lastBlock)->prevTS = INT64_MIN; - (*lastBlock)->size = sizeof(SSubmitBlk); - (*lastBlock)->tsSource = -1; - - return TSDB_CODE_SUCCESS; -} - - -static int32_t insertStmtGenBlock(STscStmt* pStmt, STableDataBlocks** pBlock, STableMeta* pTableMeta, SName* name) { - int32_t code = 0; - - if (pStmt->mtb.lastBlock == NULL) { - tscError("no previous data block"); - return TSDB_CODE_TSC_APP_ERROR; - } - - int32_t msize = tscGetTableMetaSize(pTableMeta); - int32_t tsize = sizeof(STableDataBlocks) + msize; - - void *t = malloc(tsize); - *pBlock = t; - - memcpy(*pBlock, pStmt->mtb.lastBlock, sizeof(STableDataBlocks)); - - t = (char *)t + sizeof(STableDataBlocks); - (*pBlock)->pTableMeta = t; - memcpy((*pBlock)->pTableMeta, pTableMeta, msize); - - (*pBlock)->pData = malloc((*pBlock)->nAllocSize); - - (*pBlock)->vgId = (*pBlock)->pTableMeta->vgId; - - tNameAssign(&(*pBlock)->tableName, name); - - SSubmitBlk* blk = (SSubmitBlk*)(*pBlock)->pData; - memset(blk, 0, sizeof(*blk)); - - code = tsSetBlockInfo(blk, pTableMeta, 0); - if (code != TSDB_CODE_SUCCESS) { - STMT_RET(code); - } - - return TSDB_CODE_SUCCESS; -} - - -static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MULTI_BIND* bind, int32_t rowNum) { - if (bind->buffer_type != param->type || !isValidDataType(param->type)) { - tscError("column mismatch or invalid"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (IS_VAR_DATA_TYPE(param->type) && bind->length == NULL) { - tscError("BINARY/NCHAR no length"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - for (int i = 0; i < bind->num; ++i) { - char* data = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * (rowNum + i); - - if (bind->is_null != NULL && bind->is_null[i]) { - setNull(data + param->offset, param->type, param->bytes); - continue; - } - - if (!IS_VAR_DATA_TYPE(param->type)) { - memcpy(data + param->offset, (char *)bind->buffer + bind->buffer_length * i, tDataTypes[param->type].bytes); - - if (param->offset == 0) { - if (tsCheckTimestamp(pBlock, data + param->offset) != TSDB_CODE_SUCCESS) { - tscError("invalid timestamp"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - } - } else if (param->type == TSDB_DATA_TYPE_BINARY) { - if (bind->length[i] > (uintptr_t)param->bytes) { - tscError("binary length too long, ignore it, max:%d, actual:%d", param->bytes, (int32_t)bind->length[i]); - return TSDB_CODE_TSC_INVALID_VALUE; - } - int16_t bsize = (short)bind->length[i]; - STR_WITH_SIZE_TO_VARSTR(data + param->offset, (char *)bind->buffer + bind->buffer_length * i, bsize); - } else if (param->type == TSDB_DATA_TYPE_NCHAR) { - if (bind->length[i] > (uintptr_t)param->bytes) { - tscError("nchar string length too long, ignore it, max:%d, actual:%d", param->bytes, (int32_t)bind->length[i]); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - int32_t output = 0; - if (!taosMbsToUcs4((char *)bind->buffer + bind->buffer_length * i, bind->length[i], varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { - tscError("convert nchar string to UCS4_LE failed:%s", (char*)((char *)bind->buffer + bind->buffer_length * i)); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - varDataSetLen(data + param->offset, output); - } - } - - return TSDB_CODE_SUCCESS; -} - -static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { - SSqlCmd* pCmd = &stmt->pSql->cmd; - STscStmt* pStmt = (STscStmt*)stmt; - - STableDataBlocks* pBlock = NULL; - - if (pStmt->multiTbInsert) { - if (pCmd->insertParam.pTableBlockHashList == NULL) { - tscError("0x%"PRIx64" Table block hash list is empty", pStmt->pSql->self); - return TSDB_CODE_TSC_APP_ERROR; - } - - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->insertParam.pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); - if (t1 == NULL) { - tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pStmt->pSql->self, pStmt->mtb.currentUid); - return TSDB_CODE_TSC_APP_ERROR; - } - - pBlock = *t1; - } else { - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pCmd->insertParam.pTableBlockHashList == NULL) { - pCmd->insertParam.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - } - - int32_t ret = - tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), - pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); - if (ret != 0) { - return ret; - } - } - - uint32_t totalDataSize = sizeof(SSubmitBlk) + (pCmd->batchSize + 1) * pBlock->rowSize; - if (totalDataSize > pBlock->nAllocSize) { - const double factor = 1.5; - - void* tmp = realloc(pBlock->pData, (uint32_t)(totalDataSize * factor)); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pBlock->pData = (char*)tmp; - pBlock->nAllocSize = (uint32_t)(totalDataSize * factor); - } - - char* data = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * pCmd->batchSize; - for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { - SParamInfo* param = &pBlock->params[j]; - - int code = doBindParam(pBlock, data, param, &bind[param->idx], 1); - if (code != TSDB_CODE_SUCCESS) { - tscDebug("0x%"PRIx64" bind column %d: type mismatch or invalid", pStmt->pSql->self, param->idx); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "bind column type mismatch or invalid"); - } - } - - return TSDB_CODE_SUCCESS; -} - - -static int insertStmtBindParamBatch(STscStmt* stmt, TAOS_MULTI_BIND* bind, int colIdx) { - SSqlCmd* pCmd = &stmt->pSql->cmd; - STscStmt* pStmt = (STscStmt*)stmt; - int rowNum = bind->num; - - STableDataBlocks* pBlock = NULL; - - if (pStmt->multiTbInsert) { - if (pCmd->insertParam.pTableBlockHashList == NULL) { - tscError("0x%"PRIx64" Table block hash list is empty", pStmt->pSql->self); - return TSDB_CODE_TSC_APP_ERROR; - } - - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->insertParam.pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); - if (t1 == NULL) { - tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pStmt->pSql->self, pStmt->mtb.currentUid); - return TSDB_CODE_TSC_APP_ERROR; - } - - pBlock = *t1; - } else { - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pCmd->insertParam.pTableBlockHashList == NULL) { - pCmd->insertParam.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - } - - int32_t ret = - tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), - pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); - if (ret != 0) { - return ret; - } - } - - if (!(colIdx == -1 || (colIdx >= 0 && colIdx < pBlock->numOfParams))) { - tscError("0x%"PRIx64" invalid colIdx:%d", pStmt->pSql->self, colIdx); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "invalid param colIdx"); - } - - uint32_t totalDataSize = sizeof(SSubmitBlk) + (pCmd->batchSize + rowNum) * pBlock->rowSize; - if (totalDataSize > pBlock->nAllocSize) { - const double factor = 1.5; - - void* tmp = realloc(pBlock->pData, (uint32_t)(totalDataSize * factor)); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pBlock->pData = (char*)tmp; - pBlock->nAllocSize = (uint32_t)(totalDataSize * factor); - } - - if (colIdx == -1) { - for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { - SParamInfo* param = &pBlock->params[j]; - if (bind[param->idx].num != rowNum) { - tscError("0x%"PRIx64" param %d: num[%d:%d] not match", pStmt->pSql->self, param->idx, rowNum, bind[param->idx].num); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "bind row num mismatch"); - } - - int code = doBindBatchParam(pBlock, param, &bind[param->idx], pCmd->batchSize); - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" bind column %d: type mismatch or invalid", pStmt->pSql->self, param->idx); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "bind column type mismatch or invalid"); - } - } - - pCmd->batchSize += rowNum - 1; - } else { - SParamInfo* param = &pBlock->params[colIdx]; - - int code = doBindBatchParam(pBlock, param, bind, pCmd->batchSize); - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" bind column %d: type mismatch or invalid", pStmt->pSql->self, param->idx); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "bind column type mismatch or invalid"); - } - - if (colIdx == (pBlock->numOfParams - 1)) { - pCmd->batchSize += rowNum - 1; - } - } - - return TSDB_CODE_SUCCESS; -} - - -static int insertStmtUpdateBatch(STscStmt* stmt) { - SSqlObj* pSql = stmt->pSql; - SSqlCmd* pCmd = &pSql->cmd; - STableDataBlocks* pBlock = NULL; - - if (pCmd->batchSize > INT16_MAX) { - tscError("too many record:%d", pCmd->batchSize); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "too many records"); - } - - if (taosHashGetSize(pCmd->insertParam.pTableBlockHashList) == 0) { - return TSDB_CODE_SUCCESS; - } - - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pCmd->insertParam.pTableBlockHashList, (const char*)&stmt->mtb.currentUid, sizeof(stmt->mtb.currentUid)); - if (t1 == NULL) { - tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pSql->self, stmt->mtb.currentUid); - return TSDB_CODE_TSC_APP_ERROR; - } - - pBlock = *t1; - - STableMeta* pTableMeta = pBlock->pTableMeta; - - pBlock->size = sizeof(SSubmitBlk) + pCmd->batchSize * pBlock->rowSize; - SSubmitBlk* pBlk = (SSubmitBlk*) pBlock->pData; - pBlk->numOfRows = pCmd->batchSize; - pBlk->dataLen = 0; - pBlk->uid = pTableMeta->id.uid; - pBlk->tid = pTableMeta->id.tid; - - return TSDB_CODE_SUCCESS; -} - -static int insertStmtAddBatch(STscStmt* stmt) { - SSqlCmd* pCmd = &stmt->pSql->cmd; - ++pCmd->batchSize; - - if (stmt->multiTbInsert) { - return insertStmtUpdateBatch(stmt); - } - - return TSDB_CODE_SUCCESS; -} - -static int insertStmtReset(STscStmt* pStmt) { - SSqlCmd* pCmd = &pStmt->pSql->cmd; - if (pCmd->batchSize > 2) { - int32_t alloced = (pCmd->batchSize + 1) / 2; - - size_t size = taosArrayGetSize(pCmd->insertParam.pDataBlocks); - for (int32_t i = 0; i < size; ++i) { - STableDataBlocks* pBlock = taosArrayGetP(pCmd->insertParam.pDataBlocks, i); - - uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk); - pBlock->size = sizeof(SSubmitBlk) + totalDataSize / alloced; - - SSubmitBlk* pSubmit = (SSubmitBlk*)pBlock->pData; - pSubmit->numOfRows = pSubmit->numOfRows / alloced; - } - } - pCmd->batchSize = 0; - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - pTableMetaInfo->vgroupIndex = 0; - return TSDB_CODE_SUCCESS; -} - -static int insertStmtExecute(STscStmt* stmt) { - SSqlCmd* pCmd = &stmt->pSql->cmd; - if (pCmd->batchSize == 0) { - tscError("no records bind"); - return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "no records bind"); - } - - if (taosHashGetSize(pCmd->insertParam.pTableBlockHashList) == 0) { - return TSDB_CODE_SUCCESS; - } - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pCmd->insertParam.pTableBlockHashList == NULL) { - pCmd->insertParam.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - } - - STableDataBlocks* pBlock = NULL; - - int32_t ret = - tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), - pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); - assert(ret == 0); - pBlock->size = sizeof(SSubmitBlk) + pCmd->batchSize * pBlock->rowSize; - SSubmitBlk* pBlk = (SSubmitBlk*) pBlock->pData; - pBlk->numOfRows = pCmd->batchSize; - pBlk->dataLen = 0; - pBlk->uid = pTableMeta->id.uid; - pBlk->tid = pTableMeta->id.tid; - - fillTablesColumnsNull(stmt->pSql); - - int code = tscMergeTableDataBlocks(&stmt->pSql->cmd.insertParam, false); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - STableDataBlocks* pDataBlock = taosArrayGetP(pCmd->insertParam.pDataBlocks, 0); - code = tscCopyDataBlockToPayload(stmt->pSql, pDataBlock); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - SSqlObj* pSql = stmt->pSql; - SSqlRes* pRes = &pSql->res; - pRes->numOfRows = 0; - pRes->numOfTotal = 0; - - tscBuildAndSendRequest(pSql, NULL); - - // wait for the callback function to post the semaphore - tsem_wait(&pSql->rspSem); - - // data block reset - pCmd->batchSize = 0; - for(int32_t i = 0; i < pCmd->insertParam.numOfTables; ++i) { - if (pCmd->insertParam.pTableNameList && pCmd->insertParam.pTableNameList[i]) { - tfree(pCmd->insertParam.pTableNameList[i]); - } - } - - pCmd->insertParam.numOfTables = 0; - tfree(pCmd->insertParam.pTableNameList); - pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); - - return pSql->res.code; -} - -static void insertBatchClean(STscStmt* pStmt) { - SSqlCmd *pCmd = &pStmt->pSql->cmd; - SSqlObj *pSql = pStmt->pSql; - int32_t size = taosHashGetSize(pCmd->insertParam.pTableBlockHashList); - - // data block reset - pCmd->batchSize = 0; - - for(int32_t i = 0; i < size; ++i) { - if (pCmd->insertParam.pTableNameList && pCmd->insertParam.pTableNameList[i]) { - tfree(pCmd->insertParam.pTableNameList[i]); - } - } - - tfree(pCmd->insertParam.pTableNameList); - - pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); - pCmd->insertParam.numOfTables = 0; - - taosHashClear(pCmd->insertParam.pTableBlockHashList); - tscFreeSqlResult(pSql); - tscFreeSubobj(pSql); - tfree(pSql->pSubs); - pSql->subState.numOfSub = 0; -} - -static int insertBatchStmtExecute(STscStmt* pStmt) { - int32_t code = 0; - - if(pStmt->mtb.nameSet == false) { - tscError("0x%"PRIx64" no table name set", pStmt->pSql->self); - return invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "no table name set"); - } - - pStmt->pSql->retry = pStmt->pSql->maxRetry + 1; //no retry - - if (taosHashGetSize(pStmt->pSql->cmd.insertParam.pTableBlockHashList) <= 0) { // merge according to vgId - tscError("0x%"PRIx64" no data block to insert", pStmt->pSql->self); - return TSDB_CODE_TSC_APP_ERROR; - } - - fillTablesColumnsNull(pStmt->pSql); - - if ((code = tscMergeTableDataBlocks(&pStmt->pSql->cmd.insertParam, false)) != TSDB_CODE_SUCCESS) { - return code; - } - - code = tscHandleMultivnodeInsert(pStmt->pSql); - - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // wait for the callback function to post the semaphore - tsem_wait(&pStmt->pSql->rspSem); - - code = pStmt->pSql->res.code; - - insertBatchClean(pStmt); - - return code; -} - -int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) { - SSqlCmd *pCmd = &pSql->cmd; - int32_t ret = TSDB_CODE_SUCCESS; - - if ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS) { - return ret; - } - - int32_t index = 0; - SStrToken sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n == 0) { - tscError("table is is expected, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "table name is expected", pCmd->insertParam.sql); - } - - if (sToken.n == 1 && sToken.type == TK_QUESTION) { - pStmt->multiTbInsert = true; - pStmt->mtb.tbname = sToken; - pStmt->mtb.nameSet = false; - if (pStmt->mtb.pTableHash == NULL) { - pStmt->mtb.pTableHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - } - - if (pStmt->mtb.pTableBlockHashList == NULL) { - pStmt->mtb.pTableBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - } - - pStmt->mtb.tagSet = true; - - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n > 0 && (sToken.type == TK_VALUES || sToken.type == TK_LP)) { - return TSDB_CODE_SUCCESS; - } - - if (sToken.n <= 0 || sToken.type != TK_USING) { - tscError("keywords USING is expected, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "keywords USING is expected", sToken.z ? sToken.z : pCmd->insertParam.sql); - } - - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n <= 0 || ((sToken.type != TK_ID) && (sToken.type != TK_STRING))) { - tscError("invalid token, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "invalid token", sToken.z ? sToken.z : pCmd->insertParam.sql); - } - pStmt->mtb.stbname = sToken; - - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n <= 0 || sToken.type != TK_TAGS) { - tscError("keyword TAGS expected, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "keyword TAGS expected", sToken.z ? sToken.z : pCmd->insertParam.sql); - } - - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n <= 0 || sToken.type != TK_LP) { - tscError("( expected, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "( expected", sToken.z ? sToken.z : pCmd->insertParam.sql); - } - - pStmt->mtb.tags = taosArrayInit(4, sizeof(SStrToken)); - - int32_t loopCont = 1; - - while (loopCont) { - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n <= 0) { - tscError("unexpected sql end, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "unexpected sql end", pCmd->insertParam.sql); - } - - switch (sToken.type) { - case TK_RP: - loopCont = 0; - break; - case TK_VALUES: - tscError("unexpected token values, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "unexpected token", sToken.z); - case TK_QUESTION: - pStmt->mtb.tagSet = false; //continue - default: - taosArrayPush(pStmt->mtb.tags, &sToken); - break; - } - } - - if (taosArrayGetSize(pStmt->mtb.tags) <= 0) { - tscError("no tags, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "no tags", pCmd->insertParam.sql); - } - - sToken = tStrGetToken(pCmd->insertParam.sql, &index, false); - if (sToken.n <= 0 || (sToken.type != TK_VALUES && sToken.type != TK_LP)) { - tscError("sql error, sql:%s", pCmd->insertParam.sql); - return tscSQLSyntaxErrMsg(pCmd->payload, "sql error", sToken.z ? sToken.z : pCmd->insertParam.sql); - } - - pStmt->mtb.values = sToken; - - } - - return TSDB_CODE_SUCCESS; -} - -int stmtGenInsertStatement(SSqlObj* pSql, STscStmt* pStmt, const char* name, TAOS_BIND* tags) { - size_t tagNum = taosArrayGetSize(pStmt->mtb.tags); - size_t size = 1048576; - char *str = calloc(1, size); - size_t len = 0; - int32_t ret = 0; - int32_t j = 0; - - while (1) { - len = (size_t)snprintf(str, size - 1, "insert into %s using %.*s tags(", name, pStmt->mtb.stbname.n, pStmt->mtb.stbname.z); - if (len >= (size -1)) { - size *= 2; - free(str); - str = calloc(1, size); - continue; - } - - j = 0; - - for (size_t i = 0; i < tagNum && len < (size - 1); ++i) { - SStrToken *t = taosArrayGet(pStmt->mtb.tags, i); - if (t->type == TK_QUESTION) { - int32_t l = 0; - if (i > 0) { - str[len++] = ','; - } - - if (tags[j].is_null && (*tags[j].is_null)) { - ret = converToStr(str + len, TSDB_DATA_TYPE_NULL, NULL, -1, &l); - } else { - if (tags[j].buffer == NULL) { - free(str); - tscError("empty tag value in params"); - return invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "empty tag value in params"); - } - - ret = converToStr(str + len, tags[j].buffer_type, tags[j].buffer, tags[j].length ? (int32_t)*tags[j].length : -1, &l); - } - - ++j; - - if (ret) { - free(str); - return ret; - } - - len += l; - } else { - len += (size_t)snprintf(str + len, size - len - 1, i > 0 ? ",%.*s" : "%.*s", t->n, t->z); - } - } - - if (len >= (size - 1)) { - size *= 2; - free(str); - str = calloc(1, size); - continue; - } - - strcat(str, ") "); - len += 2; - - if ((len + strlen(pStmt->mtb.values.z)) >= (size - 1)) { - size *= 2; - free(str); - str = calloc(1, size); - continue; - } - - strcat(str, pStmt->mtb.values.z); - - break; - } - - if (pStmt->mtb.sqlstr == NULL) { - pStmt->mtb.sqlstr = pSql->sqlstr; - } else { - tfree(pSql->sqlstr); - } - - pSql->sqlstr = str; - - return TSDB_CODE_SUCCESS; -} - -//////////////////////////////////////////////////////////////////////////////// -// interface functions - -TAOS_STMT* taos_stmt_init(TAOS* taos) { - STscObj* pObj = (STscObj*)taos; - STscStmt* pStmt = NULL; - - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - tscError("connection disconnected"); - return NULL; - } - - pStmt = calloc(1, sizeof(STscStmt)); - if (pStmt == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("failed to allocate memory for statement"); - return NULL; - } - pStmt->taos = pObj; - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - - if (pSql == NULL) { - free(pStmt); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("failed to allocate memory for statement"); - return NULL; - } - - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { - free(pSql); - free(pStmt); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("failed to malloc payload buffer"); - return NULL; - } - - tsem_init(&pSql->rspSem, 0, 0); - pSql->signature = pSql; - pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_MAX_REPLICA; - pStmt->pSql = pSql; - pStmt->last = STMT_INIT; - - return pStmt; -} - -int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { - STscStmt* pStmt = (STscStmt*)stmt; - - if (stmt == NULL || pStmt->taos == NULL || pStmt->pSql == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (sql == NULL) { - tscError("sql is NULL"); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "sql is NULL")); - } - - if (pStmt->last != STMT_INIT) { - tscError("prepare status error, last:%d", pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "prepare status error")); - } - - pStmt->last = STMT_PREPARE; - - SSqlObj* pSql = pStmt->pSql; - size_t sqlLen = strlen(sql); - - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - pSql->param = (void*) pSql; - pSql->fp = waitForQueryRsp; - pSql->fetchFp = waitForQueryRsp; - - pCmd->insertParam.insertType = TSDB_QUERY_TYPE_STMT_INSERT; - pCmd->insertParam.objectId = pSql->self; - - char* sqlstr = realloc(pSql->sqlstr, sqlLen + 1); - if(sqlstr == NULL && pSql->sqlstr) free(pSql->sqlstr); - pSql->sqlstr = sqlstr; - if (pSql->sqlstr == NULL) { - tscError("%p failed to malloc sql string buffer", pSql); - STMT_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); - } - - pRes->qId = 0; - pRes->numOfRows = 1; - - registerSqlObj(pSql); - - strtolower(pSql->sqlstr, sql); - tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); - - if (tscIsInsertData(pSql->sqlstr)) { - pStmt->isInsert = true; - - pSql->cmd.insertParam.numOfParams = 0; - pSql->cmd.batchSize = 0; - - int32_t ret = stmtParseInsertTbTags(pSql, pStmt); - if (ret != TSDB_CODE_SUCCESS) { - STMT_RET(ret); - } - - if (pStmt->multiTbInsert) { - STMT_RET(TSDB_CODE_SUCCESS); - } - - memset(&pStmt->mtb, 0, sizeof(pStmt->mtb)); - - int32_t code = tsParseSql(pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - // wait for the callback function to post the semaphore - tsem_wait(&pSql->rspSem); - STMT_RET(pSql->res.code); - } - - STMT_RET(code); - } - - pStmt->isInsert = false; - STMT_RET(normalStmtPrepare(pStmt)); -} - -int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags) { - STscStmt* pStmt = (STscStmt*)stmt; - int32_t code = 0; - - if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - SSqlObj* pSql = pStmt->pSql; - SSqlCmd* pCmd = &pSql->cmd; - - if (name == NULL) { - tscError("0x%"PRIx64" name is NULL", pSql->self); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "name is NULL")); - } - - if (pStmt->multiTbInsert == false || !tscIsInsertData(pSql->sqlstr)) { - tscError("0x%"PRIx64" not multiple table insert", pSql->self); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "not multiple table insert")); - } - - if (pStmt->last == STMT_INIT || pStmt->last == STMT_BIND || pStmt->last == STMT_BIND_COL) { - tscError("0x%"PRIx64" set_tbname_tags status error, last:%d", pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "set_tbname_tags status error")); - } - - pStmt->last = STMT_SETTBNAME; - - uint64_t* uid = (uint64_t*)taosHashGet(pStmt->mtb.pTableHash, name, strlen(name)); - if (uid != NULL) { - pStmt->mtb.currentUid = *uid; - - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pStmt->mtb.pTableBlockHashList, (const char*)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid)); - if (t1 == NULL) { - tscError("0x%"PRIx64" no table data block in hash list, uid:%" PRId64 , pSql->self, pStmt->mtb.currentUid); - STMT_RET(TSDB_CODE_TSC_APP_ERROR); - } - - SSubmitBlk* pBlk = (SSubmitBlk*) (*t1)->pData; - pCmd->batchSize = pBlk->numOfRows; - if (pBlk->numOfRows == 0) { - (*t1)->prevTS = INT64_MIN; - } - - tsSetBlockInfo(pBlk, (*t1)->pTableMeta, pBlk->numOfRows); - - taosHashPut(pCmd->insertParam.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)t1, POINTER_BYTES); - - tscDebug("0x%"PRIx64" table:%s is already prepared, uid:%" PRIu64, pSql->self, name, pStmt->mtb.currentUid); - STMT_RET(TSDB_CODE_SUCCESS); - } - - if (pStmt->mtb.subSet && taosHashGetSize(pStmt->mtb.pTableHash) > 0) { - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - char sTableName[TSDB_TABLE_FNAME_LEN] = {0}; - tstrncpy(sTableName, pTableMeta->sTableName, sizeof(sTableName)); - - SStrToken tname = {0}; - tname.type = TK_STRING; - tname.z = (char *)name; - tname.n = (uint32_t)strlen(name); - SName fullname = {0}; - tscSetTableFullName(&fullname, &tname, pSql); - - memcpy(&pTableMetaInfo->name, &fullname, sizeof(fullname)); - - code = tscGetTableMetaEx(pSql, pTableMetaInfo, false, true); - if (code != TSDB_CODE_SUCCESS) { - STMT_RET(code); - } - - pTableMeta = pTableMetaInfo->pTableMeta; - - if (strcmp(sTableName, pTableMeta->sTableName)) { - tscError("0x%"PRIx64" only tables belongs to one stable is allowed", pSql->self); - STMT_RET(TSDB_CODE_TSC_APP_ERROR); - } - - STableDataBlocks* pBlock = NULL; - - insertStmtGenBlock(pStmt, &pBlock, pTableMeta, &pTableMetaInfo->name); - - pCmd->batchSize = 0; - - pStmt->mtb.currentUid = pTableMeta->id.uid; - pStmt->mtb.tbNum++; - - taosHashPut(pCmd->insertParam.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)&pBlock, POINTER_BYTES); - taosHashPut(pStmt->mtb.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)&pBlock, POINTER_BYTES); - taosHashPut(pStmt->mtb.pTableHash, name, strlen(name), (char*) &pTableMeta->id.uid, sizeof(pTableMeta->id.uid)); - - tscDebug("0x%"PRIx64" table:%s is prepared, uid:%" PRIx64, pSql->self, name, pStmt->mtb.currentUid); - - STMT_RET(TSDB_CODE_SUCCESS); - } - - if (pStmt->mtb.tagSet) { - pStmt->mtb.tbname = tscReplaceStrToken(&pSql->sqlstr, &pStmt->mtb.tbname, name); - } else { - if (tags == NULL) { - tscError("No tags set"); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "no tags set")); - } - - int32_t ret = stmtGenInsertStatement(pSql, pStmt, name, tags); - if (ret != TSDB_CODE_SUCCESS) { - STMT_RET(ret); - } - } - - pStmt->mtb.nameSet = true; - - tscDebug("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); - - pSql->cmd.insertParam.numOfParams = 0; - pSql->cmd.batchSize = 0; - - if (taosHashGetSize(pCmd->insertParam.pTableBlockHashList) > 0) { - SHashObj* hashList = pCmd->insertParam.pTableBlockHashList; - pCmd->insertParam.pTableBlockHashList = NULL; - tscResetSqlCmd(pCmd, false, pSql->self); - pCmd->insertParam.pTableBlockHashList = hashList; - } - - code = tsParseSql(pStmt->pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - // wait for the callback function to post the semaphore - tsem_wait(&pStmt->pSql->rspSem); - - code = pStmt->pSql->res.code; - } - - if (code == TSDB_CODE_SUCCESS) { - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - STableDataBlocks* pBlock = NULL; - code = tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), - pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); - if (code != TSDB_CODE_SUCCESS) { - STMT_RET(code); - } - - SSubmitBlk* blk = (SSubmitBlk*)pBlock->pData; - blk->numOfRows = 0; - - pStmt->mtb.currentUid = pTableMeta->id.uid; - pStmt->mtb.tbNum++; - - taosHashPut(pStmt->mtb.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)&pBlock, POINTER_BYTES); - taosHashPut(pStmt->mtb.pTableHash, name, strlen(name), (char*) &pTableMeta->id.uid, sizeof(pTableMeta->id.uid)); - - if (pStmt->mtb.lastBlock == NULL) { - insertStmtGenLastBlock(&pStmt->mtb.lastBlock, pBlock); - } - - tscDebug("0x%"PRIx64" table:%s is prepared, uid:%" PRIx64, pSql->self, name, pStmt->mtb.currentUid); - } - - STMT_RET(code); -} - - -int taos_stmt_set_sub_tbname(TAOS_STMT* stmt, const char* name) { - STscStmt* pStmt = (STscStmt*)stmt; - pStmt->mtb.subSet = true; - return taos_stmt_set_tbname_tags(stmt, name, NULL); -} - - - -int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { - STscStmt* pStmt = (STscStmt*)stmt; - pStmt->mtb.subSet = false; - return taos_stmt_set_tbname_tags(stmt, name, NULL); -} - - -int taos_stmt_close(TAOS_STMT* stmt) { - STscStmt* pStmt = (STscStmt*)stmt; - if (!pStmt->isInsert) { - SNormalStmt* normal = &pStmt->normal; - if (normal->params != NULL) { - for (uint16_t i = 0; i < normal->numParams; i++) { - tVariantDestroy(normal->params + i); - } - free(normal->params); - } - free(normal->parts); - free(normal->sql); - } else { - if (pStmt->multiTbInsert) { - taosHashCleanup(pStmt->mtb.pTableHash); - bool rmMeta = false; - if (pStmt->pSql && pStmt->pSql->res.code != 0) { - rmMeta = true; - } - tscDestroyDataBlock(pStmt->mtb.lastBlock, rmMeta); - pStmt->mtb.pTableBlockHashList = tscDestroyBlockHashTable(pStmt->mtb.pTableBlockHashList, rmMeta); - if (pStmt->pSql){ - taosHashCleanup(pStmt->pSql->cmd.insertParam.pTableBlockHashList); - } - pStmt->pSql->cmd.insertParam.pTableBlockHashList = NULL; - taosArrayDestroy(pStmt->mtb.tags); - tfree(pStmt->mtb.sqlstr); - } - } - - taos_free_result(pStmt->pSql); - tfree(pStmt); - STMT_RET(TSDB_CODE_SUCCESS); -} - -int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { - STscStmt* pStmt = (STscStmt*)stmt; - if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (pStmt->isInsert) { - if (pStmt->multiTbInsert) { - if (pStmt->last != STMT_SETTBNAME && pStmt->last != STMT_ADD_BATCH) { - tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "bind param status error")); - } - } else { - if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_EXECUTE) { - tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "bind param status error")); - } - } - - pStmt->last = STMT_BIND; - - tscDebug("tableId:%" PRIu64 ", try to bind one row", pStmt->mtb.currentUid); - - STMT_RET(insertStmtBindParam(pStmt, bind)); - } else { - STMT_RET(normalStmtBindParam(pStmt, bind)); - } -} - - -int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { - STscStmt* pStmt = (STscStmt*)stmt; - - if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (bind == NULL || bind->num <= 0 || bind->num > INT16_MAX) { - tscError("0x%"PRIx64" invalid parameter", pStmt->pSql->self); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "invalid bind param")); - } - - if (!pStmt->isInsert) { - tscError("0x%"PRIx64" not or invalid batch insert", pStmt->pSql->self); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "not or invalid batch insert")); - } - - if (pStmt->multiTbInsert) { - if (pStmt->last != STMT_SETTBNAME && pStmt->last != STMT_ADD_BATCH) { - tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "bind param status error")); - } - } else { - if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_EXECUTE) { - tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "bind param status error")); - } - } - - pStmt->last = STMT_BIND; - - STMT_RET(insertStmtBindParamBatch(pStmt, bind, -1)); -} - -int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int colIdx) { - STscStmt* pStmt = (STscStmt*)stmt; - if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (bind == NULL || bind->num <= 0 || bind->num > INT16_MAX || colIdx < 0) { - tscError("0x%"PRIx64" invalid parameter", pStmt->pSql->self); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "invalid bind param")); - } - - if (!pStmt->isInsert) { - tscError("0x%"PRIx64" not or invalid batch insert", pStmt->pSql->self); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "not or invalid batch insert")); - } - - if (pStmt->multiTbInsert) { - if (pStmt->last != STMT_SETTBNAME && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_BIND_COL) { - tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "bind param status error")); - } - } else { - if (pStmt->last != STMT_PREPARE && pStmt->last != STMT_ADD_BATCH && pStmt->last != STMT_BIND_COL && pStmt->last != STMT_EXECUTE) { - tscError("0x%"PRIx64" bind param status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "bind param status error")); - } - } - - pStmt->last = STMT_BIND_COL; - - STMT_RET(insertStmtBindParamBatch(pStmt, bind, colIdx)); -} - - - -int taos_stmt_add_batch(TAOS_STMT* stmt) { - STscStmt* pStmt = (STscStmt*)stmt; - if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (pStmt->isInsert) { - if (pStmt->last != STMT_BIND && pStmt->last != STMT_BIND_COL) { - tscError("0x%"PRIx64" add batch status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "add batch status error")); - } - - pStmt->last = STMT_ADD_BATCH; - - STMT_RET(insertStmtAddBatch(pStmt)); - } - - STMT_RET(TSDB_CODE_COM_OPS_NOT_SUPPORT); -} - -int taos_stmt_reset(TAOS_STMT* stmt) { - STscStmt* pStmt = (STscStmt*)stmt; - if (pStmt->isInsert) { - STMT_RET(insertStmtReset(pStmt)); - } - STMT_RET(TSDB_CODE_SUCCESS); -} - -int taos_stmt_execute(TAOS_STMT* stmt) { - int ret = 0; - STscStmt* pStmt = (STscStmt*)stmt; - if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (pStmt->isInsert) { - if (pStmt->last != STMT_ADD_BATCH) { - tscError("0x%"PRIx64" exec status error, last:%d", pStmt->pSql->self, pStmt->last); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "exec status error")); - } - - pStmt->last = STMT_EXECUTE; - - pStmt->pSql->cmd.insertParam.payloadType = PAYLOAD_TYPE_RAW; - if (pStmt->multiTbInsert) { - ret = insertBatchStmtExecute(pStmt); - } else { - ret = insertStmtExecute(pStmt); - } - } else { // normal stmt query - char* sql = normalStmtBuildSql(pStmt); - if (sql == NULL) { - ret = TSDB_CODE_TSC_OUT_OF_MEMORY; - } else { - if (pStmt->pSql != NULL) { - tscFreeSqlObj(pStmt->pSql); - pStmt->pSql = NULL; - } - - pStmt->pSql = taos_query((TAOS*)pStmt->taos, sql); - ret = taos_errno(pStmt->pSql); - free(sql); - } - } - - STMT_RET(ret); -} - -TAOS_RES *taos_stmt_use_result(TAOS_STMT* stmt) { - if (stmt == NULL) { - tscError("statement is invalid."); - return NULL; - } - - STscStmt* pStmt = (STscStmt*)stmt; - if (pStmt->pSql == NULL) { - tscError("result has been used already."); - return NULL; - } - - TAOS_RES* result = pStmt->pSql; - pStmt->pSql = NULL; - return result; -} - -int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert) { - STscStmt* pStmt = (STscStmt*)stmt; - - if (stmt == NULL || pStmt->taos == NULL || pStmt->pSql == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (insert) *insert = pStmt->isInsert; - - STMT_RET(TSDB_CODE_SUCCESS); -} - -int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) { - STscStmt* pStmt = (STscStmt*)stmt; - - if (stmt == NULL || pStmt->taos == NULL || pStmt->pSql == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (pStmt->isInsert) { - SSqlObj* pSql = pStmt->pSql; - SSqlCmd *pCmd = &pSql->cmd; - *nums = pCmd->insertParam.numOfParams; - STMT_RET(TSDB_CODE_SUCCESS); - } else { - SNormalStmt* normal = &pStmt->normal; - *nums = normal->numParams; - STMT_RET(TSDB_CODE_SUCCESS); - } -} - -int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes) { - STscStmt* pStmt = (STscStmt*)stmt; - - if (stmt == NULL || pStmt->taos == NULL || pStmt->pSql == NULL) { - STMT_RET(TSDB_CODE_TSC_DISCONNECTED); - } - - if (pStmt->isInsert) { - SSqlCmd* pCmd = &pStmt->pSql->cmd; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pCmd->insertParam.pTableBlockHashList == NULL) { - pCmd->insertParam.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - } - - STableDataBlocks* pBlock = NULL; - - int32_t ret = - tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), - pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); - if (ret != 0) { - STMT_RET(ret); - } - - if (idx<0 || idx>=pBlock->numOfParams) { - tscError("0x%"PRIx64" param %d: out of range", pStmt->pSql->self, idx); - STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "idx out of range")); - } - - SParamInfo* param = &pBlock->params[idx]; - if (type) *type = param->type; - if (bytes) *bytes = param->bytes; - - STMT_RET(TSDB_CODE_SUCCESS); - } else { - STMT_RET(TSDB_CODE_COM_OPS_NOT_SUPPORT); - } -} - - -char *taos_stmt_errstr(TAOS_STMT *stmt) { - STscStmt* pStmt = (STscStmt*)stmt; - - if (stmt == NULL) { - return (char*) tstrerror(terrno); - } - - return taos_errstr(pStmt->pSql); -} - - - -const char *taos_data_type(int type) { - switch (type) { - case TSDB_DATA_TYPE_NULL: return "TSDB_DATA_TYPE_NULL"; - case TSDB_DATA_TYPE_BOOL: return "TSDB_DATA_TYPE_BOOL"; - case TSDB_DATA_TYPE_TINYINT: return "TSDB_DATA_TYPE_TINYINT"; - case TSDB_DATA_TYPE_SMALLINT: return "TSDB_DATA_TYPE_SMALLINT"; - case TSDB_DATA_TYPE_INT: return "TSDB_DATA_TYPE_INT"; - case TSDB_DATA_TYPE_BIGINT: return "TSDB_DATA_TYPE_BIGINT"; - case TSDB_DATA_TYPE_FLOAT: return "TSDB_DATA_TYPE_FLOAT"; - case TSDB_DATA_TYPE_DOUBLE: return "TSDB_DATA_TYPE_DOUBLE"; - case TSDB_DATA_TYPE_BINARY: return "TSDB_DATA_TYPE_BINARY"; - case TSDB_DATA_TYPE_TIMESTAMP: return "TSDB_DATA_TYPE_TIMESTAMP"; - case TSDB_DATA_TYPE_NCHAR: return "TSDB_DATA_TYPE_NCHAR"; - default: return "UNKNOWN"; - } -} - diff --git a/2.0/src/client/src/tscProfile.c b/2.0/src/client/src/tscProfile.c deleted file mode 100644 index 43e7365a5911d8436ea953eb3eb23d563948308a..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscProfile.c +++ /dev/null @@ -1,368 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tscLog.h" -#include "tsclient.h" -#include "tsocket.h" -#include "ttimer.h" -#include "tmsg.h" -#include "tcq.h" - -#include "../../../include/client/taos.h" -#include "tscUtil.h" - -void tscSaveSlowQueryFp(void *handle, void *tmrId); -TAOS *tscSlowQueryConn = NULL; -bool tscSlowQueryConnInitialized = false; - -void tscInitConnCb(void *param, TAOS_RES *result, int code) { - char *sql = param; - if (code < 0) { - tscError("taos:%p, slow query connect failed, code:%d", tscSlowQueryConn, code); - taos_close(tscSlowQueryConn); - tscSlowQueryConn = NULL; - tscSlowQueryConnInitialized = false; - free(sql); - } else { - tscDebug("taos:%p, slow query connect success, code:%d", tscSlowQueryConn, code); - tscSlowQueryConnInitialized = true; - tscSaveSlowQueryFp(sql, NULL); - } - taos_free_result(result); -} - -void tscAddIntoSqlList(SSqlObj *pSql) { - static uint32_t queryId = 1; - - STscObj *pObj = pSql->pTscObj; - if (pSql->listed) return; - - pthread_mutex_lock(&pObj->mutex); - - assert(pSql != pObj->sqlList); - pSql->next = pObj->sqlList; - if (pObj->sqlList) pObj->sqlList->prev = pSql; - pObj->sqlList = pSql; - pSql->queryId = atomic_fetch_add_32(&queryId, 1); - - pthread_mutex_unlock(&pObj->mutex); - - pSql->stime = taosGetTimestampMs(); - pSql->listed = 1; - - tscDebug("0x%"PRIx64" added into sqlList, queryId:%u", pSql->self, pSql->queryId); -} - -void tscSaveSlowQueryFpCb(void *param, TAOS_RES *result, int code) { - if (code < 0) { - tscError("failed to save slow query, code:%d", code); - } else { - tscDebug("success to save slow query, code:%d", code); - } - taos_free_result(result); -} - -void tscSaveSlowQueryFp(void *handle, void *tmrId) { - char *sql = handle; - - if (!tscSlowQueryConnInitialized) { - if (tscSlowQueryConn == NULL) { - tscDebug("start to init slow query connect"); - taos_connect_a(NULL, "monitor", tsInternalPass, "", 0, tscInitConnCb, sql, &tscSlowQueryConn); - } else { - tscError("taos:%p, slow query connect is already initialized", tscSlowQueryConn); - free(sql); - } - } else { - tscDebug("taos:%p, save slow query:%s", tscSlowQueryConn, sql); - taos_query_a(tscSlowQueryConn, sql, tscSaveSlowQueryFpCb, NULL); - free(sql); - } -} - -void tscSaveSlowQuery(SSqlObj *pSql) { - const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable - size_t size = 200; // other part of sql string, expect the main sql str - - if (pSql->res.useconds < SLOW_QUERY_INTERVAL) { - return; - } - - tscDebug("0x%"PRIx64" query time:%" PRId64 " sql:%s", pSql->self, pSql->res.useconds, pSql->sqlstr); - int32_t sqlSize = (int32_t)(TSDB_SLOW_QUERY_SQL_LEN + size); - - char *sql = malloc(sqlSize); - if (sql == NULL) { - tscError("0x%"PRIx64" failed to allocate memory to sent slow query to dnode", pSql->self); - return; - } - - int len = snprintf(sql, size, "insert into %s.slowquery values(now, '%s', %" PRId64 ", %" PRId64 ", '", tsMonitorDbName, - pSql->pTscObj->user, pSql->stime, pSql->res.useconds); - int sqlLen = snprintf(sql + len, TSDB_SLOW_QUERY_SQL_LEN, "%s", pSql->sqlstr); - if (sqlLen > TSDB_SLOW_QUERY_SQL_LEN - 1) { - sqlLen = len + TSDB_SLOW_QUERY_SQL_LEN - 1; - } else { - sqlLen += len; - } - - strcpy(sql + sqlLen, "')"); - taosTmrStart(tscSaveSlowQueryFp, 200, sql, tscTmr); -} - -void tscRemoveFromSqlList(SSqlObj *pSql) { - STscObj *pObj = pSql->pTscObj; - if (pSql->listed == 0) return; - - pthread_mutex_lock(&pObj->mutex); - - if (pSql->prev) - pSql->prev->next = pSql->next; - else - pObj->sqlList = pSql->next; - - if (pSql->next) pSql->next->prev = pSql->prev; - - pthread_mutex_unlock(&pObj->mutex); - - pSql->next = NULL; - pSql->prev = NULL; - pSql->listed = 0; - - tscSaveSlowQuery(pSql); - tscDebug("0x%"PRIx64" removed from sqlList", pSql->self); -} - -void tscKillQuery(STscObj *pObj, uint32_t killId) { - pthread_mutex_lock(&pObj->mutex); - - SSqlObj *pSql = pObj->sqlList; - while (pSql) { - if (pSql->queryId == killId) break; - pSql = pSql->next; - } - - pthread_mutex_unlock(&pObj->mutex); - - if (pSql == NULL) { - tscError("failed to kill query, id:%d, it may have completed/terminated", killId); - } else { - tscDebug("0x%"PRIx64" query is killed, queryId:%d", pSql->self, killId); - taos_stop_query(pSql); - } -} - -void tscAddIntoStreamList(SSqlStream *pStream) { - static uint32_t streamId = 1; - STscObj * pObj = pStream->pSql->pTscObj; - - pthread_mutex_lock(&pObj->mutex); - - pStream->next = pObj->streamList; - if (pObj->streamList) pObj->streamList->prev = pStream; - pObj->streamList = pStream; - pStream->streamId = streamId++; - - pthread_mutex_unlock(&pObj->mutex); - - pStream->listed = 1; -} - -void tscRemoveFromStreamList(SSqlStream *pStream, SSqlObj *pSqlObj) { - if (pStream->listed == 0) return; - - STscObj *pObj = pSqlObj->pTscObj; - - pthread_mutex_lock(&pObj->mutex); - - if (pStream->prev) - pStream->prev->next = pStream->next; - else - pObj->streamList = pStream->next; - - if (pStream->next) pStream->next->prev = pStream->prev; - - pthread_mutex_unlock(&pObj->mutex); - - pStream->next = NULL; - pStream->prev = NULL; - - pStream->listed = 0; -} - -void tscKillStream(STscObj *pObj, uint32_t killId) { - pthread_mutex_lock(&pObj->mutex); - - SSqlStream *pStream = pObj->streamList; - while (pStream) { - if (pStream->streamId == killId) break; - pStream = pStream->next; - } - - pthread_mutex_unlock(&pObj->mutex); - - if (pStream) { - tscDebug("0x%"PRIx64" stream:%p is killed, streamId:%d", pStream->pSql->self, pStream, killId); - if (pStream->callback) { - pStream->callback(pStream->param); - } - taos_close_stream(pStream); - } else { - tscError("failed to kill stream, streamId:%d not exist", killId); - } -} - -int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { - SHeartBeatReq *pHeartbeat = pMsg; - - int allocedQueriesNum = pHeartbeat->numOfQueries; - int allocedStreamsNum = pHeartbeat->numOfStreams; - - pHeartbeat->numOfQueries = 0; - SQueryDesc *pQdesc = (SQueryDesc *)pHeartbeat->pData; - - int64_t now = taosGetTimestampMs(); - SSqlObj *pSql = pObj->sqlList; - - while (pSql) { - /* - * avoid sqlobj may not be correctly removed from sql list - * e.g., forgetting to free the sql result may cause the sql object still in sql list - */ - if (pSql->sqlstr == NULL) { - pSql = pSql->next; - continue; - } - - tstrncpy(pQdesc->sql, pSql->sqlstr, sizeof(pQdesc->sql)); - pQdesc->stime = htobe64(pSql->stime); - pQdesc->queryId = htonl(pSql->queryId); - pQdesc->useconds = htobe64(now - pSql->stime); - pQdesc->qId = htobe64(pSql->res.qId); - pQdesc->sqlObjId = htobe64(pSql->self); - pQdesc->pid = pHeartbeat->pid; - pQdesc->numOfSub = pSql->subState.numOfSub; - - // todo race condition - pQdesc->stableQuery = 0; - - char *p = pQdesc->subSqlInfo; - int32_t remainLen = sizeof(pQdesc->subSqlInfo); - if (pQdesc->numOfSub == 0) { - snprintf(p, remainLen, "N/A"); - } else { -// SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); -// if (pQueryInfo != NULL) { -// pQdesc->stableQuery = (pQueryInfo->stableQuery)?1:0; -// } else { -// pQdesc->stableQuery = 0; -// } - - if (pSql->pSubs != NULL && pSql->subState.states != NULL) { - for (int32_t i = 0; i < pQdesc->numOfSub; ++i) { - SSqlObj *psub = pSql->pSubs[i]; - int64_t self = (psub != NULL)? psub->self : 0; - - int32_t len = snprintf(p, remainLen, "[%d]0x%" PRIx64 "(%c) ", i, self, pSql->subState.states[i] ? 'C' : 'I'); - if (len > remainLen) { - break; - } - - remainLen -= len; - p += len; - } - } - } - - pQdesc->numOfSub = htonl(pQdesc->numOfSub); - taosGetFqdn(pQdesc->fqdn); - - pHeartbeat->numOfQueries++; - pQdesc++; - - pSql = pSql->next; - if (pHeartbeat->numOfQueries >= allocedQueriesNum) { - break; - } - } - - pHeartbeat->numOfStreams = 0; - SStreamDesc *pSdesc = (SStreamDesc *)pQdesc; - - SSqlStream *pStream = pObj->streamList; - while (pStream) { - tstrncpy(pSdesc->sql, pStream->pSql->sqlstr, sizeof(pSdesc->sql)); - if (pStream->dstTable == NULL) { - pSdesc->dstTable[0] = 0; - } else { - tstrncpy(pSdesc->dstTable, pStream->dstTable, sizeof(pSdesc->dstTable)); - } - pSdesc->streamId = htonl(pStream->streamId); - pSdesc->num = htobe64(pStream->num); - - pSdesc->useconds = htobe64(pStream->useconds); - pSdesc->stime = (pStream->stime == INT64_MIN) ? htobe64(pStream->stime) : htobe64(pStream->stime - pStream->interval.interval); - pSdesc->ctime = htobe64(pStream->ctime); - - pSdesc->slidingTime = htobe64(pStream->interval.sliding); - pSdesc->interval = htobe64(pStream->interval.interval); - - pHeartbeat->numOfStreams++; - pSdesc++; - pStream = pStream->next; - if (pHeartbeat->numOfStreams >= allocedStreamsNum) break; - } - - int32_t msgLen = pHeartbeat->numOfQueries * sizeof(SQueryDesc) + pHeartbeat->numOfStreams * sizeof(SStreamDesc) + - sizeof(SHeartBeatReq); - pHeartbeat->connId = htonl(pObj->connId); - pHeartbeat->numOfQueries = htonl(pHeartbeat->numOfQueries); - pHeartbeat->numOfStreams = htonl(pHeartbeat->numOfStreams); - - return msgLen; -} - -// cqContext->dbconn is killed then call this callback -void cqConnKilledNotify(void* handle, void* conn) { - if (handle == NULL || conn == NULL){ - return ; - } - - SCqContext* pContext = (SCqContext*) handle; - if (pContext->dbConn == conn){ - atomic_store_ptr(&(pContext->dbConn), NULL); - } -} - -void tscKillConnection(STscObj *pObj) { - // get stream header by locked - pthread_mutex_lock(&pObj->mutex); - SSqlStream *pStream = pObj->streamList; - pthread_mutex_unlock(&pObj->mutex); - - while (pStream) { - SSqlStream *tmp = pStream->next; - // set associate variant to NULL - cqConnKilledNotify(pStream->cqhandle, pObj); - // taos_close_stream function call pObj->mutet lock , careful death-lock - taos_close_stream(pStream); - pStream = tmp; - } - - tscDebug("connection:%p is killed", pObj); - taos_close(pObj); -} - diff --git a/2.0/src/client/src/tscSQLParser.c b/2.0/src/client/src/tscSQLParser.c deleted file mode 100644 index 1c50a2823e0becba0ed35f021bfe9f46f1e7de2a..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscSQLParser.c +++ /dev/null @@ -1,9411 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef __APPLE__ -#define _BSD_SOURCE -#define _XOPEN_SOURCE 500 -#define _DEFAULT_SOURCE -#define _GNU_SOURCE -#endif // __APPLE__ - -#include -#include "../../../include/client/taos.h" -#include "os.h" -#include "qPlan.h" -#include "qSqlparser.h" -#include "qTableMeta.h" -#include "qUtil.h" -#include "regex.h" -#include "tmsg.h" -#include "tcompare.h" -#include "texpr.h" -#include "tfilter.h" -#include "tname.h" -#include "tscLog.h" -#include "tscUtil.h" -#include "tsclient.h" -#include "tscript.h" -#include "tstrbuild.h" -#include "ttoken.h" -#include "ttokendef.h" -#include "ttype.h" - -#define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" - -#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) - -// -1 is tbname column index, so here use the -2 as the initial value -#define COLUMN_INDEX_INITIAL_VAL (-2) -#define COLUMN_INDEX_INITIALIZER \ - { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL } -#define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX)) -#define TBNAME_LIST_SEP "," - -typedef struct SColumnList { // todo refactor - int32_t num; - SColumnIndex ids[TSDB_MAX_COLUMNS]; -} SColumnList; - -typedef struct SConvertFunc { - int32_t originFuncId; - int32_t execFuncId; -} SConvertFunc; - -static SExprInfo* doAddOneProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tableIndex, int32_t colId); - -static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); -static char* getAccountId(SSqlObj* pSql); - -static int convertTimestampStrToInt64(tVariant *pVar, int32_t precision); -static bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, uint8_t precision); - -static bool has(SArray* pFieldList, int32_t startIdx, const char* name); -static int32_t getDelimiterIndex(SStrToken* pTableName); -static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd); -static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd); - -static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len); -static void getColumnName(tSqlExprItem* pItem, char* resultFieldName, char* rawName, int32_t nameLength); - -static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, - bool finalResult, SUdfInfo* pUdfInfo); -static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pColList, int16_t bytes, - int8_t type, char* fieldName, SExprInfo* pSqlExpr); - -static uint8_t convertRelationalOperator(SStrToken *pToken); - -static int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelNodeList, bool isSTable, bool joinQuery, bool timeWindowQuery); - -static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool twQuery); - -static int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd); - -static int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode); -static int32_t parseIntervalOffset(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* offsetToken); -static int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* pSliding); -static int32_t validateStateWindowNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, bool isStable); - -static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem, bool outerQuery); - -static int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql); -static int32_t validateFillNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode); -static int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSchema* pSchema); - -static int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); -static int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type); -static int32_t validateEp(char* ep); -static int32_t validateDNodeConfig(SMiscInfo* pOptions); -static int32_t validateLocalConfig(SMiscInfo* pOptions); -static int32_t validateColumnName(char* name); -static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType); -static int32_t setCompactVnodeInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); - -static int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField); -static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); -static bool hasNormalColumnFilter(SQueryInfo* pQueryInfo); - -static int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSqlObj* pSql); -static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDbInfo* pCreateDbSql); -static int32_t getColumnIndexByName(const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex, char* msg); -static int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); - -static int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); -static int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* msg); -static int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode); -static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate); - -static SColumnList createColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex); - -static int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* pInfo); -static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); -static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); -static int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInfo); - -static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, uint64_t *uid); -static bool validateDebugFlag(int32_t v); -static int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo); -static tSqlExpr* extractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, int32_t tableIndex); - -static bool isTimeWindowQuery(SQueryInfo* pQueryInfo) { - return pQueryInfo->interval.interval > 0 || pQueryInfo->sessionWindow.gap > 0; -} - - -int16_t getNewResColId(SSqlCmd* pCmd) { - return pCmd->resColumnId--; -} - -// serialize expr in exprlist to binary -// format "type | size | value" -bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, uint8_t precision) { - bool ret = false; - if (!pList || pList->size <= 0 || colType < 0) { - return ret; - } - - tSqlExpr* item = ((tSqlExprItem*)(taosArrayGet(pList, 0)))->pNode; - int32_t firstVarType = item->value.nType; - - SBufferWriter bw = tbufInitWriter( NULL, false); - tbufEnsureCapacity(&bw, 512); - if (colType == TSDB_DATA_TYPE_TIMESTAMP) { - tbufWriteUint32(&bw, TSDB_DATA_TYPE_BIGINT); - } else { - tbufWriteUint32(&bw, colType); - } - - tbufWriteInt32(&bw, (int32_t)(pList->size)); - - for (int32_t i = 0; i < (int32_t)pList->size; i++) { - tSqlExpr* pSub = ((tSqlExprItem*)(taosArrayGet(pList, i)))->pNode; - tVariant* var = &pSub->value; - - // check all the exprToken type in expr list same or not - if (firstVarType != var->nType) { - break; - } - if ((colType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(colType))) { - if (var->nType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(var->nType)) { - break; - } - if (colType == TSDB_DATA_TYPE_BOOL && (var->i64 > 1 ||var->i64 < 0)) { - break; - } - tbufWriteInt64(&bw, var->i64); - } else if (IS_UNSIGNED_NUMERIC_TYPE(colType)) { - if (IS_SIGNED_NUMERIC_TYPE(var->nType) || IS_UNSIGNED_NUMERIC_TYPE(var->nType)) { - tbufWriteUint64(&bw, var->u64); - } else { - break; - } - } else if (colType == TSDB_DATA_TYPE_DOUBLE || colType == TSDB_DATA_TYPE_FLOAT) { - if (IS_SIGNED_NUMERIC_TYPE(var->nType) || IS_UNSIGNED_NUMERIC_TYPE(var->nType)) { - tbufWriteDouble(&bw, (double)(var->i64)); - } else if (var->nType == TSDB_DATA_TYPE_DOUBLE || var->nType == TSDB_DATA_TYPE_FLOAT){ - tbufWriteDouble(&bw, var->dKey); - } else { - break; - } - } else if (colType == TSDB_DATA_TYPE_BINARY) { - if (var->nType != TSDB_DATA_TYPE_BINARY) { - break; - } - tbufWriteBinary(&bw, var->pz, var->nLen); - } else if (colType == TSDB_DATA_TYPE_NCHAR) { - if (var->nType != TSDB_DATA_TYPE_BINARY) { - break; - } - char *buf = (char *)calloc(1, (var->nLen + 1)*TSDB_NCHAR_SIZE); - if (tVariantDump(var, buf, colType, false) != TSDB_CODE_SUCCESS) { - free(buf); - break; - } - tbufWriteBinary(&bw, buf, twcslen((wchar_t *)buf) * TSDB_NCHAR_SIZE); - free(buf); - } else if (colType == TSDB_DATA_TYPE_TIMESTAMP) { - if (var->nType == TSDB_DATA_TYPE_BINARY) { - if (convertTimestampStrToInt64(var, precision) < 0) { - break; - } - tbufWriteInt64(&bw, var->i64); - } else if (var->nType == TSDB_DATA_TYPE_BIGINT) { - tbufWriteInt64(&bw, var->i64); - } else { - break; - } - } else { - break; - } - if (i == (int32_t)(pList->size - 1)) { ret = true;} - } - if (ret == true) { - if ((*dst = calloc(1, sizeof(tVariant))) != NULL) { - tVariantCreateFromBinary(*dst, tbufGetData(&bw, false), tbufTell(&bw), TSDB_DATA_TYPE_BINARY); - } else { - ret = false; - } - } - tbufCloseWriter(&bw); - return ret; -} - - -static uint8_t convertRelationalOperator(SStrToken *pToken) { - switch (pToken->type) { - case TK_LT: - return TSDB_RELATION_LESS; - case TK_LE: - return TSDB_RELATION_LESS_EQUAL; - case TK_GT: - return TSDB_RELATION_GREATER; - case TK_GE: - return TSDB_RELATION_GREATER_EQUAL; - case TK_NE: - return TSDB_RELATION_NOT_EQUAL; - case TK_AND: - return TSDB_RELATION_AND; - case TK_OR: - return TSDB_RELATION_OR; - case TK_EQ: - return TSDB_RELATION_EQUAL; - case TK_PLUS: - return TSDB_BINARY_OP_ADD; - - case TK_MINUS: - return TSDB_BINARY_OP_SUBTRACT; - case TK_STAR: - return TSDB_BINARY_OP_MULTIPLY; - case TK_SLASH: - case TK_DIVIDE: - return TSDB_BINARY_OP_DIVIDE; - case TK_REM: - return TSDB_BINARY_OP_REMAINDER; - case TK_LIKE: - return TSDB_RELATION_LIKE; - case TK_MATCH: - return TSDB_RELATION_MATCH; - case TK_NMATCH: - return TSDB_RELATION_NMATCH; - case TK_ISNULL: - return TSDB_RELATION_ISNULL; - case TK_NOTNULL: - return TSDB_RELATION_NOTNULL; - case TK_IN: - return TSDB_RELATION_IN; - default: { return 0; } - } -} - -static bool validateDebugFlag(int32_t v) { - const static int validFlag[] = {131, 135, 143}; - - for (int i = 0; i < tListLen(validFlag); i++) { - if (v == validFlag[i]) { - return true; - } - } - return false; -} -/* - * Used during parsing query sql. Since the query sql usually small in length, error position - * is not needed in the final error message. - */ -static int32_t invalidOperationMsg(char* dstBuffer, const char* errMsg) { - return tscInvalidOperationMsg(dstBuffer, errMsg, NULL); -} - -static int convertTimestampStrToInt64(tVariant *pVar, int32_t precision) { - int64_t time = 0; - strdequote(pVar->pz); - - char* seg = strnchr(pVar->pz, '-', pVar->nLen, false); - if (seg != NULL) { - if (taosParseTime(pVar->pz, &time, pVar->nLen, precision, tsDaylight) != TSDB_CODE_SUCCESS) { - return -1; - } - } else { - if (tVariantDump(pVar, (char*)&time, TSDB_DATA_TYPE_BIGINT, true)) { - return -1; - } - } - tVariantDestroy(pVar); - tVariantCreateFromBinary(pVar, (char*)&time, 0, TSDB_DATA_TYPE_BIGINT); - return 0; -} -static int setColumnFilterInfoForTimestamp(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tVariant* pVar) { - const char* msg = "invalid timestamp"; - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - if (convertTimestampStrToInt64(pVar, tinfo.precision) < -1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - return TSDB_CODE_SUCCESS; -} - -static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) { - const char* msg1 = "password can not be empty"; - const char* msg2 = "name or password too long"; - const char* msg3 = "password needs single quote marks enclosed"; - - if (pPwd->type != TK_STRING) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - strdequote(pPwd->z); - pPwd->n = (uint32_t)strtrim(pPwd->z); // trim space before and after passwords - - if (pPwd->n <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pPwd->n >= TSDB_KEY_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - return TSDB_CODE_SUCCESS; -} - -// validate the out put field type for "UNION ALL" subclause -static int32_t normalizeVarDataTypeLength(SSqlCmd* pCmd) { - const char* msg1 = "columns in select clause not identical"; - - int32_t diffSize = 0; - - // if there is only one element, the limit of clause is the limit of global result. - SQueryInfo* pQueryInfo1 = pCmd->pQueryInfo; - SQueryInfo* pSibling = pQueryInfo1->sibling; - - while(pSibling != NULL) { - int32_t ret = tscFieldInfoCompare(&pQueryInfo1->fieldsInfo, &pSibling->fieldsInfo, &diffSize); - if (ret != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - pSibling = pSibling->sibling; - } - - if (diffSize) { - pQueryInfo1 = pCmd->pQueryInfo; - pSibling = pQueryInfo1->sibling; - - while(pSibling->sibling != NULL) { - tscFieldInfoSetSize(&pQueryInfo1->fieldsInfo, &pSibling->fieldsInfo); - pSibling = pSibling->sibling; - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t readFromFile(char *name, uint32_t *len, void **buf) { - struct stat fileStat; - if (stat(name, &fileStat) < 0) { - tscError("stat file %s failed, error:%s", name, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - *len = fileStat.st_size; - - if (*len <= 0) { - tscError("file %s is empty", name); - return TSDB_CODE_TSC_FILE_EMPTY; - } - - *buf = calloc(1, *len); - if (*buf == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int fd = open(name, O_RDONLY); - if (fd < 0) { - tscError("open file %s failed, error:%s", name, strerror(errno)); - tfree(*buf); - return TAOS_SYSTEM_ERROR(errno); - } - - int64_t s = taosRead(fd, *buf, *len); - if (s != *len) { - tscError("read file %s failed, error:%s", name, strerror(errno)); - close(fd); - tfree(*buf); - return TSDB_CODE_TSC_APP_ERROR; - } - close(fd); - return TSDB_CODE_SUCCESS; -} - - -int32_t handleUserDefinedFunc(SSqlObj* pSql, struct SSqlInfo* pInfo) { - const char *msg1 = "invalidate function name"; - const char *msg2 = "path is too long"; - const char *msg3 = "invalid outputtype"; - const char *msg4 = "invalid script"; - SSqlCmd *pCmd = &pSql->cmd; - - switch (pInfo->type) { - case TSDB_SQL_CREATE_FUNCTION: { - SCreateFuncInfo *createInfo = &pInfo->pMiscInfo->funcOpt; - uint32_t len = 0; - void *buf = NULL; - - if (createInfo->output.type == (uint8_t)-1 || createInfo->output.bytes < 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - createInfo->name.z[createInfo->name.n] = 0; - // funcname's naming rule is same to column - if (validateColumnName(createInfo->name.z) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - strdequote(createInfo->name.z); - - if (strlen(createInfo->name.z) >= TSDB_FUNC_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - createInfo->path.z[createInfo->path.n] = 0; - - strdequote(createInfo->path.z); - - if (strlen(createInfo->path.z) >= PATH_MAX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - int32_t ret = readFromFile(createInfo->path.z, &len, &buf); - if (ret) { - return ret; - } - //distinguish *.lua and *.so - int32_t pathLen = (int32_t)strlen(createInfo->path.z); - if ((pathLen > 3) && (0 == strncmp(createInfo->path.z + pathLen - 3, "lua", 3)) && !isValidScript(buf, len)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - //TODO CHECK CODE - if (len + sizeof(SCreateFuncMsg) > pSql->cmd.allocSize) { - ret = tscAllocPayload(&pSql->cmd, len + sizeof(SCreateFuncMsg)); - if (ret) { - tfree(buf); - return ret; - } - } - - SCreateFuncMsg *pMsg = (SCreateFuncMsg *)pSql->cmd.payload; - - strcpy(pMsg->name, createInfo->name.z); - strcpy(pMsg->path, createInfo->path.z); - - pMsg->funcType = htonl(createInfo->type); - pMsg->bufSize = htonl(createInfo->bufSize); - - pMsg->outputType = createInfo->output.type; - pMsg->outputLen = htons(createInfo->output.bytes); - - pMsg->codeLen = htonl(len); - memcpy(pMsg->code, buf, len); - tfree(buf); - - break; - } - case TSDB_SQL_DROP_FUNCTION: { - SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); - - SDropFuncMsg *pMsg = (SDropFuncMsg *)pSql->cmd.payload; - - t0->z[t0->n] = 0; - - strdequote(t0->z); - - if (strlen(t0->z) >= TSDB_FUNC_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - strcpy(pMsg->name, t0->z); - - break; - } - default: - return TSDB_CODE_TSC_APP_ERROR; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { - if (pInfo == NULL || pSql == NULL) { - return TSDB_CODE_TSC_APP_ERROR; - } - - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - int32_t code = TSDB_CODE_SUCCESS; - if (!pInfo->valid || terrno == TSDB_CODE_TSC_SQL_SYNTAX_ERROR) { - terrno = TSDB_CODE_SUCCESS; // clear the error number - return tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), NULL, pInfo->msg); - } - - SQueryInfo* pQueryInfo = tscGetQueryInfoS(pCmd); - if (pQueryInfo == NULL) { - pRes->code = terrno; - return pRes->code; - } - - STableMetaInfo* pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo) : pQueryInfo->pTableMetaInfo[0]; - if (pTableMetaInfo == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; - } - - pCmd->command = pInfo->type; - - switch (pInfo->type) { - case TSDB_SQL_DROP_TABLE: - case TSDB_SQL_DROP_USER: - case TSDB_SQL_DROP_ACCT: - case TSDB_SQL_DROP_DNODE: - case TSDB_SQL_DROP_DB: { - const char* msg2 = "invalid name"; - const char* msg3 = "param name too long"; - - SStrToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); - if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (tscValidateName(pzName) != TSDB_CODE_SUCCESS)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - if (pInfo->type == TSDB_SQL_DROP_DB) { - assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); - code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); - if (code != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - } else if (pInfo->type == TSDB_SQL_DROP_TABLE) { - assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); - - code = tscSetTableFullName(&pTableMetaInfo->name, pzName, pSql); - if(code != TSDB_CODE_SUCCESS) { - return code; - } - } else if (pInfo->type == TSDB_SQL_DROP_DNODE) { - if (pzName->type == TK_STRING) { - pzName->n = strdequote(pzName->z); - } - strncpy(pCmd->payload, pzName->z, pzName->n); - } else { // drop user/account - if (pzName->n >= TSDB_USER_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - strncpy(pCmd->payload, pzName->z, pzName->n); - } - - break; - } - - case TSDB_SQL_USE_DB: { - const char* msg = "invalid db name"; - SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); - - if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); - if (ret != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - break; - } - - case TSDB_SQL_RESET_CACHE: { - return TSDB_CODE_SUCCESS; - } - - case TSDB_SQL_SHOW: { - if (setShowInfo(pSql, pInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - break; - } - - case TSDB_SQL_CREATE_FUNCTION: - case TSDB_SQL_DROP_FUNCTION: { - code = handleUserDefinedFunc(pSql, pInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - break; - } - - case TSDB_SQL_ALTER_DB: - case TSDB_SQL_CREATE_DB: { - const char* msg1 = "invalid db name"; - const char* msg2 = "name too long"; - - SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt); - if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - char buf[TSDB_DB_NAME_LEN] = {0}; - SStrToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf)); - - if (tscValidateName(&token) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), &token); - if (ret != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - if (parseCreateDBOptions(pCmd, pCreateDB) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - break; - } - - case TSDB_SQL_CREATE_DNODE: { - const char* msg = "invalid host name (ip address)"; - - if (taosArrayGetSize(pInfo->pMiscInfo->a) > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - SStrToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0); - if (id->type == TK_STRING) { - id->n = strdequote(id->z); - } - break; - } - - case TSDB_SQL_CREATE_ACCT: - case TSDB_SQL_ALTER_ACCT: { - const char* msg1 = "invalid state option, available options[no, r, w, all]"; - const char* msg2 = "invalid user/account name"; - const char* msg3 = "name too long"; - - SStrToken* pName = &pInfo->pMiscInfo->user.user; - SStrToken* pPwd = &pInfo->pMiscInfo->user.passwd; - - if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pName->n >= TSDB_USER_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; - if (pAcctOpt->stat.n > 0) { - if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { - } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { - } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { - } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - break; - } - - case TSDB_SQL_DESCRIBE_TABLE: { - const char* msg1 = "invalid table name"; - - SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); - if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - // additional msg has been attached already - code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - return tscGetTableMeta(pSql, pTableMetaInfo); - } - case TSDB_SQL_SHOW_CREATE_STABLE: - case TSDB_SQL_SHOW_CREATE_TABLE: { - const char* msg1 = "invalid table name"; - - SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); - if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - return tscGetTableMeta(pSql, pTableMetaInfo); - } - case TSDB_SQL_SHOW_CREATE_DATABASE: { - const char* msg1 = "invalid database name"; - - SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); - if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pToken->n > TSDB_DB_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - return tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); - } - case TSDB_SQL_CFG_DNODE: { - const char* msg2 = "invalid configure options or values, such as resetlog / debugFlag 135 / balance 'vnode:2-dnode:2' / monitor 1 "; - const char* msg3 = "invalid dnode ep"; - - /* validate the ip address */ - SMiscInfo* pMiscInfo = pInfo->pMiscInfo; - - /* validate the parameter names and options */ - if (validateDNodeConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - char* pMsg = pCmd->payload; - - SMCfgDnodeReq* pCfg = (SMCfgDnodeReq*)pMsg; - - SStrToken* t0 = taosArrayGet(pMiscInfo->a, 0); - SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1); - - t0->n = strdequote(t0->z); - strncpy(pCfg->ep, t0->z, t0->n); - - if (validateEp(pCfg->ep) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - strncpy(pCfg->config, t1->z, t1->n); - - if (taosArrayGetSize(pMiscInfo->a) == 3) { - SStrToken* t2 = taosArrayGet(pMiscInfo->a, 2); - - pCfg->config[t1->n] = ' '; // add sep - strncpy(&pCfg->config[t1->n + 1], t2->z, t2->n); - } - - break; - } - - case TSDB_SQL_CREATE_USER: - case TSDB_SQL_ALTER_USER: { - const char* msg2 = "invalid user/account name"; - const char* msg3 = "name too long"; - const char* msg5 = "invalid user rights"; - const char* msg7 = "not support options"; - - pCmd->command = pInfo->type; - - SUserInfo* pUser = &pInfo->pMiscInfo->user; - SStrToken* pName = &pUser->user; - SStrToken* pPwd = &pUser->passwd; - - if (pName->n >= TSDB_USER_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - if (pCmd->command == TSDB_SQL_CREATE_USER) { - if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else { - if (pUser->type == TSDB_ALTER_USER_PASSWD) { - if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - assert(pPwd->type == TSDB_DATA_TYPE_NULL); - - SStrToken* pPrivilege = &pUser->privilege; - - if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) { - pCmd->count = 1; - } else if (strncasecmp(pPrivilege->z, "read", 4) == 0 && pPrivilege->n == 4) { - pCmd->count = 2; - } else if (strncasecmp(pPrivilege->z, "write", 5) == 0 && pPrivilege->n == 5) { - pCmd->count = 3; - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - } - - break; - } - - case TSDB_SQL_CFG_LOCAL: { - SMiscInfo *pMiscInfo = pInfo->pMiscInfo; - const char *msg = "invalid configure options or values"; - - // validate the parameter names and options - if (validateLocalConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - int32_t numOfToken = (int32_t) taosArrayGetSize(pMiscInfo->a); - assert(numOfToken >= 1 && numOfToken <= 2); - - SStrToken* t = taosArrayGet(pMiscInfo->a, 0); - strncpy(pCmd->payload, t->z, t->n); - if (numOfToken == 2) { - SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1); - pCmd->payload[t->n] = ' '; // add sep - strncpy(&pCmd->payload[t->n + 1], t1->z, t1->n); - } - return TSDB_CODE_SUCCESS; - } - - case TSDB_SQL_CREATE_TABLE: { - SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; - - if (pCreateTable->type == TSQL_CREATE_TABLE || pCreateTable->type == TSQL_CREATE_STABLE) { - if ((code = doCheckForCreateTable(pSql, 0, pInfo)) != TSDB_CODE_SUCCESS) { - return code; - } - - } else if (pCreateTable->type == TSQL_CREATE_TABLE_FROM_STABLE) { - assert(pCmd->numOfCols == 0); - if ((code = doCheckForCreateFromStable(pSql, pInfo)) != TSDB_CODE_SUCCESS) { - return code; - } - - } else if (pCreateTable->type == TSQL_CREATE_STREAM) { - if ((code = doCheckForStream(pSql, pInfo)) != TSDB_CODE_SUCCESS) { - return code; - } - } - - break; - } - - case TSDB_SQL_SELECT: { - const char * msg1 = "no nested query supported in union clause"; - code = loadAllTableMeta(pSql, pInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - pQueryInfo = tscGetQueryInfo(pCmd); - - size_t size = taosArrayGetSize(pInfo->list); - for (int32_t i = 0; i < size; ++i) { - SSqlNode* pSqlNode = taosArrayGetP(pInfo->list, i); - - tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size); - - if (size > 1 && pSqlNode->from && pSqlNode->from->type == SQL_FROM_NODE_SUBQUERY) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - -// normalizeSqlNode(pSqlNode); // normalize the column name in each function - if ((code = validateSqlNode(pSql, pSqlNode, pQueryInfo)) != TSDB_CODE_SUCCESS) { - return code; - } - - tscPrintSelNodeList(pSql, i); - - if ((i + 1) < size && pQueryInfo->sibling == NULL) { - if ((code = tscAddQueryInfo(pCmd)) != TSDB_CODE_SUCCESS) { - return code; - } - - SArray *pUdfInfo = NULL; - if (pQueryInfo->pUdfInfo) { - pUdfInfo = taosArrayDup(pQueryInfo->pUdfInfo); - } - - pQueryInfo = pCmd->active; - pQueryInfo->pUdfInfo = pUdfInfo; - pQueryInfo->udfCopy = true; - } - } - - if ((code = normalizeVarDataTypeLength(pCmd)) != TSDB_CODE_SUCCESS) { - return code; - } - - // set the command/global limit parameters from the first subclause to the sqlcmd object - pCmd->active = pCmd->pQueryInfo; - pCmd->command = pCmd->pQueryInfo->command; - - STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pCmd->active, 0); - if (pTableMetaInfo1->pTableMeta != NULL) { - pSql->res.precision = tscGetTableInfo(pTableMetaInfo1->pTableMeta).precision; - } - - return TSDB_CODE_SUCCESS; // do not build query message here - } - - case TSDB_SQL_ALTER_TABLE: { - if ((code = setAlterTableInfo(pSql, pInfo)) != TSDB_CODE_SUCCESS) { - return code; - } - - break; - } - - case TSDB_SQL_KILL_QUERY: - case TSDB_SQL_KILL_STREAM: - case TSDB_SQL_KILL_CONNECTION: { - if ((code = setKillInfo(pSql, pInfo, pInfo->type)) != TSDB_CODE_SUCCESS) { - return code; - } - break; - } - - case TSDB_SQL_SYNC_DB_REPLICA: { - const char* msg1 = "invalid db name"; - SStrToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); - - assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); - code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); - if (code != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - break; - } - case TSDB_SQL_COMPACT_VNODE:{ - const char* msg = "invalid compact"; - if (setCompactVnodeInfo(pSql, pInfo) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - break; - } - default: - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support sql expression"); - } - - if (tscBuildMsg[pCmd->command] != NULL) { - return tscBuildMsg[pCmd->command](pSql, pInfo); - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support sql expression"); - } -} - -/* - * if the top/bottom exists, only tags columns, tbname column, and primary timestamp column - * are available. - */ -static bool isTopBottomQuery(SQueryInfo* pQueryInfo) { - size_t size = tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < size; ++i) { - int32_t functionId = tscExprGet(pQueryInfo, i)->base.functionId; - - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - return true; - } - } - - return false; -} - -// need to add timestamp column in result set, if it is a time window query -static int32_t addPrimaryTsColumnForTimeWindowQuery(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) { - uint64_t uid = tscExprGet(pQueryInfo, 0)->base.uid; - - int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - if (pTableMetaInfo->pTableMeta->id.uid == uid) { - tableIndex = i; - break; - } - } - - if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - SSchema s = {.bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tstrncpy(s.name, aAggs[TSDB_FUNC_TS].name, sizeof(s.name)); - - SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL, getNewResColId(pCmd)); - return TSDB_CODE_SUCCESS; -} - -static int32_t checkInvalidExprForTimeWindow(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char* msg1 = "invalid query expression"; - const char* msg2 = "top/bottom query does not support order by value in time window query"; - - // for top/bottom + interval query, we do not add additional timestamp column in the front - if (isTopBottomQuery(pQueryInfo)) { - - // invalid sql: - // top(col, k) from table_name [interval(1d)|session(ts, 1d)] order by k asc - // order by normal column is not supported - int32_t colId = pQueryInfo->order.orderColId; - if (isTimeWindowQuery(pQueryInfo) && colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - return TSDB_CODE_SUCCESS; - } - - /* - * invalid sql: - * select count(tbname)/count(tag1)/count(tag2) from super_table_name [interval(1d)|session(ts, 1d)]; - */ - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_COUNT && TSDB_COL_IS_TAG(pExpr->base.colInfo.flag)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - /* - * invalid sql: - * select tbname, tags_fields from super_table_name [interval(1s)|session(ts,1s)] - */ - if (tscQueryTags(pQueryInfo) && isTimeWindowQuery(pQueryInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - return addPrimaryTsColumnForTimeWindowQuery(pQueryInfo, pCmd); -} - -int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode) { - const char* msg1 = "sliding cannot be used without interval"; - const char* msg2 = "interval cannot be less than 1 us"; - const char* msg3 = "interval value is too small"; - const char* msg4 = "only point interpolation query requires keyword EVERY"; - - SSqlCmd* pCmd = &pSql->cmd; - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - if (!TPARSER_HAS_TOKEN(pSqlNode->interval.interval)) { - if (TPARSER_HAS_TOKEN(pSqlNode->sliding)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - return TSDB_CODE_SUCCESS; - } - - // orderby column not set yet, set it to be the primary timestamp column - if (pQueryInfo->order.orderColId == INT32_MIN) { - pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - } - - // interval is not null - SStrToken *t = &pSqlNode->interval.interval; - if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.interval, - &pQueryInfo->interval.intervalUnit, tinfo.precision) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pQueryInfo->interval.interval <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { - // interval cannot be less than 10 milliseconds - if (convertTimePrecision(pQueryInfo->interval.interval, tinfo.precision, TSDB_TIME_PRECISION_MICRO) < tsMinIntervalTime) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } - - if (parseIntervalOffset(pCmd, pQueryInfo, &pSqlNode->interval.offset) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (parseSlidingClause(pCmd, pQueryInfo, &pSqlNode->sliding) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - bool interpQuery = tscIsPointInterpQuery(pQueryInfo); - if ((pSqlNode->interval.token == TK_EVERY && (!interpQuery)) || (pSqlNode->interval.token == TK_INTERVAL && interpQuery)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - // The following part is used to check for the invalid query expression. - return checkInvalidExprForTimeWindow(pCmd, pQueryInfo); -} - -static int32_t validateStateWindowNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, bool isStable) { - - const char* msg1 = "invalid column name"; - const char* msg2 = "invalid column type"; - const char* msg3 = "not support state_window with group by "; - const char* msg4 = "function not support for super table query"; - const char* msg5 = "not support state_window on tag column"; - - SStrToken *col = &(pSqlNode->windowstateVal.col) ; - if (col->z == NULL || col->n <= 0) { - return TSDB_CODE_SUCCESS; - } - - if (pQueryInfo->colList == NULL) { - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - } - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - pQueryInfo->groupbyExpr.numOfGroupCols = 1; - - //TODO(dengyihao): check tag column - if (isStable) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(col, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } else if (index.columnIndex >= numOfCols) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - SGroupbyExpr* pGroupExpr = &pQueryInfo->groupbyExpr; - if (pGroupExpr->columnInfo == NULL) { - pGroupExpr->columnInfo = taosArrayInit(4, sizeof(SColIndex)); - } - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); - if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT - || pSchema->type == TSDB_DATA_TYPE_DOUBLE || pSchema->type == TSDB_DATA_TYPE_NCHAR - || pSchema->type == TSDB_DATA_TYPE_BINARY) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - tscColumnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->id.uid, pSchema); - SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId }; - taosArrayPush(pGroupExpr->columnInfo, &colIndex); - pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; - pQueryInfo->stateWindow = true; - return TSDB_CODE_SUCCESS; -} - -int32_t validateSessionNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode * pSqlNode) { - const char* msg1 = "gap should be fixed time window"; - const char* msg2 = "only one type time window allowed"; - const char* msg3 = "invalid column name"; - const char* msg4 = "invalid time window"; - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - // no session window - if (!TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap)) { - return TSDB_CODE_SUCCESS; - } - - SStrToken* col = &pSqlNode->sessionVal.col; - SStrToken* gap = &pSqlNode->sessionVal.gap; - - char timeUnit = 0; - if (parseNatualDuration(gap->z, gap->n, &pQueryInfo->sessionWindow.gap, &timeUnit, tinfo.precision) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - if (timeUnit == 'y' || timeUnit == 'n') { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pQueryInfo->sessionWindow.gap != 0 && pQueryInfo->interval.interval != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - if (pQueryInfo->sessionWindow.gap == 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(col, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - pQueryInfo->sessionWindow.primaryColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - - // The following part is used to check for the invalid query expression. - return checkInvalidExprForTimeWindow(pCmd, pQueryInfo); -} - -int32_t parseIntervalOffset(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* offsetToken) { - const char* msg1 = "interval offset cannot be negative"; - const char* msg2 = "interval offset should be shorter than interval"; - const char* msg3 = "cannot use 'year' as offset when interval is 'month'"; - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - SStrToken* t = offsetToken; - if (t->n == 0) { - pQueryInfo->interval.offsetUnit = pQueryInfo->interval.intervalUnit; - pQueryInfo->interval.offset = 0; - return TSDB_CODE_SUCCESS; - } - - if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.offset, - &pQueryInfo->interval.offsetUnit, tinfo.precision) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pQueryInfo->interval.offset < 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pQueryInfo->interval.offsetUnit != 'n' && pQueryInfo->interval.offsetUnit != 'y') { - if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { - if (pQueryInfo->interval.offset >= pQueryInfo->interval.interval) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } - } else if (pQueryInfo->interval.offsetUnit == pQueryInfo->interval.intervalUnit) { - if (pQueryInfo->interval.offset >= pQueryInfo->interval.interval) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } else if (pQueryInfo->interval.intervalUnit == 'n' && pQueryInfo->interval.offsetUnit == 'y') { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } else if (pQueryInfo->interval.intervalUnit == 'y' && pQueryInfo->interval.offsetUnit == 'n') { - if (pQueryInfo->interval.interval * 12 <= pQueryInfo->interval.offset) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } else { - // TODO: offset should be shorter than interval, but how to check - // conflicts like 30days offset and 1 month interval - } - - return TSDB_CODE_SUCCESS; -} - -int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* pSliding) { - const char* msg0 = "sliding value too small"; - const char* msg1 = "sliding value no larger than the interval value"; - const char* msg2 = "sliding value can not less than 1% of interval value"; - const char* msg3 = "does not support sliding when interval is natural month/year"; - - const static int32_t INTERVAL_SLIDING_FACTOR = 100; - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - SInterval* pInterval = &pQueryInfo->interval; - if (pSliding->n == 0) { - pInterval->slidingUnit = pInterval->intervalUnit; - pInterval->sliding = pInterval->interval; - return TSDB_CODE_SUCCESS; - } - - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - parseAbsoluteDuration(pSliding->z, pSliding->n, &pInterval->sliding, &pInterval->slidingUnit, tinfo.precision); - - if (pInterval->sliding < convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, tinfo.precision)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); - } - - if (pInterval->sliding > pInterval->interval) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if ((pInterval->interval != 0) && (pInterval->interval/pInterval->sliding > INTERVAL_SLIDING_FACTOR)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tscSetTableFullName(SName* pName, SStrToken* pTableName, SSqlObj* pSql) { - const char* msg1 = "name too long"; - const char* msg2 = "acctId too long"; - const char* msg3 = "no acctId"; - const char* msg4 = "db name too long"; - const char* msg5 = "table name too long"; - - SSqlCmd* pCmd = &pSql->cmd; - int32_t code = TSDB_CODE_SUCCESS; - int32_t idx = getDelimiterIndex(pTableName); - if (idx != -1) { // db has been specified in sql string so we ignore current db path - char* acctId = getAccountId(pSql); - if (acctId == NULL || strlen(acctId) <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - code = tNameSetAcctId(pName, acctId); - if (code != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - if (idx >= TSDB_DB_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - if (pTableName->n - 1 - idx >= TSDB_TABLE_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - strncpy(name, pTableName->z, pTableName->n); - - code = tNameFromString(pName, name, T_NAME_DB|T_NAME_TABLE); - if (code != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } else { // get current DB name first, and then set it into path - char* t = cloneCurrentDBName(pSql); - if (strlen(t) == 0) { - tfree(t); - return TSDB_CODE_TSC_DB_NOT_SELECTED; - } - - code = tNameFromString(pName, t, T_NAME_ACCT | T_NAME_DB); - if (code != 0) { - tfree(t); - return TSDB_CODE_TSC_DB_NOT_SELECTED; - } - - tfree(t); - - if (pTableName->n >= TSDB_TABLE_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - strncpy(name, pTableName->z, pTableName->n); - - code = tNameFromString(pName, name, T_NAME_TABLE); - if (code != 0) { - code = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - return code; -} - -static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { - assert(pFieldList != NULL); - - const char* msg = "illegal number of columns"; - const char* msg1 = "first column must be timestamp"; - const char* msg2 = "row length exceeds max length"; - const char* msg3 = "duplicated column names"; - const char* msg4 = "invalid data type"; - const char* msg5 = "invalid binary/nchar column length"; - const char* msg6 = "invalid column name"; - const char* msg7 = "too many columns"; - - // number of fields no less than 2 - size_t numOfCols = taosArrayGetSize(pFieldList); - if (numOfCols <= 1 ) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - return false; - } else if (numOfCols > TSDB_MAX_COLUMNS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - return false; - } - - // first column must be timestamp - TAOS_FIELD* pField = taosArrayGet(pFieldList, 0); - if (pField->type != TSDB_DATA_TYPE_TIMESTAMP) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - return false; - } - - int32_t nLen = 0; - for (int32_t i = 0; i < numOfCols; ++i) { - pField = taosArrayGet(pFieldList, i); - if (!isValidDataType(pField->type)) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - return false; - } - - if (pField->bytes == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; - } - - if ((pField->type == TSDB_DATA_TYPE_BINARY && (pField->bytes <= 0 || pField->bytes > TSDB_MAX_BINARY_LEN)) || - (pField->type == TSDB_DATA_TYPE_NCHAR && (pField->bytes <= 0 || pField->bytes > TSDB_MAX_NCHAR_LEN))) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; - } - - if (validateColumnName(pField->name) != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - return false; - } - - // field name must be unique - if (has(pFieldList, i + 1, pField->name) == true) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; - } - - nLen += pField->bytes; - } - - // max row length must be less than TSDB_MAX_BYTES_PER_ROW - if (nLen > TSDB_MAX_BYTES_PER_ROW) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - return false; - } - - return true; -} - - -static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd) { - assert(pTagsList != NULL); - - const char* msg1 = "invalid number of tag columns"; - const char* msg2 = "tag length too long"; - const char* msg3 = "duplicated column names"; - //const char* msg4 = "timestamp not allowed in tags"; - const char* msg5 = "invalid data type in tags"; - const char* msg6 = "invalid tag name"; - const char* msg7 = "invalid binary/nchar tag length"; - - // number of fields at least 1 - size_t numOfTags = taosArrayGetSize(pTagsList); - if (numOfTags < 1 || numOfTags > TSDB_MAX_TAGS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - return false; - } - - for (int32_t i = 0; i < numOfTags; ++i) { - TAOS_FIELD* p = taosArrayGet(pTagsList, i); - if (!isValidDataType(p->type)) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; - } - - if ((p->type == TSDB_DATA_TYPE_BINARY && p->bytes <= 0) || - (p->type == TSDB_DATA_TYPE_NCHAR && p->bytes <= 0)) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - return false; - } - - if (validateColumnName(p->name) != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - return false; - } - - if (has(pTagsList, i + 1, p->name) == true) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; - } - } - - int32_t nLen = 0; - for (int32_t i = 0; i < numOfTags; ++i) { - TAOS_FIELD* p = taosArrayGet(pTagsList, i); - if (p->bytes == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - return false; - } - - nLen += p->bytes; - } - - // max tag row length must be less than TSDB_MAX_TAGS_LEN - if (nLen > TSDB_MAX_TAGS_LEN) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - return false; - } - - // field name must be unique - for (int32_t i = 0; i < numOfTags; ++i) { - TAOS_FIELD* p = taosArrayGet(pTagsList, i); - - if (has(pFieldList, 0, p->name) == true) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; - } - } - - return true; -} - -/* - * tags name /column name is truncated in sql.y - */ -int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { - const char* msg3 = "tag length too long"; - const char* msg4 = "invalid tag name"; - const char* msg5 = "invalid binary/nchar tag length"; - const char* msg6 = "invalid data type in tags"; - const char* msg7 = "too many columns"; - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - int32_t numOfTags = tscGetNumOfTags(pTableMeta); - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - - // no more max columns - if (numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - - // no more than 6 tags - if (numOfTags == TSDB_MAX_TAGS) { - char msg[128] = {0}; - sprintf(msg, "tags no more than %d", TSDB_MAX_TAGS); - - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - // no timestamp allowable - //if (pTagField->type == TSDB_DATA_TYPE_TIMESTAMP) { - // invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - // return false; - //} - - if ((pTagField->type < TSDB_DATA_TYPE_BOOL) || (pTagField->type > TSDB_DATA_TYPE_UBIGINT)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - int32_t nLen = 0; - - for (int32_t i = 0; i < numOfTags; ++i) { - nLen += pTagSchema[i].bytes; - } - - // length less than TSDB_MAX_TASG_LEN - if (nLen + pTagField->bytes > TSDB_MAX_TAGS_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - // tags name can not be a keyword - if (validateColumnName(pTagField->name) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - // binary(val), val can not be equalled to or less than 0 - if ((pTagField->type == TSDB_DATA_TYPE_BINARY || pTagField->type == TSDB_DATA_TYPE_NCHAR) && pTagField->bytes <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - // field name must be unique - SSchema* pSchema = tscGetTableSchema(pTableMeta); - - for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { - if (strncasecmp(pTagField->name, pSchema[i].name, sizeof(pTagField->name) - 1) == 0) { - //return tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pTagField->name, NULL); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "duplicated column names"); - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { - const char* msg1 = "too many columns"; - const char* msg3 = "column length too long"; - const char* msg4 = "invalid data type"; - const char* msg5 = "invalid column name"; - const char* msg6 = "invalid column length"; - -// assert(pCmd->numOfClause == 1); - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - int32_t numOfTags = tscGetNumOfTags(pTableMeta); - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - - // no more max columns - if (numOfCols >= TSDB_MAX_COLUMNS || numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pColField->type < TSDB_DATA_TYPE_BOOL || pColField->type > TSDB_DATA_TYPE_UBIGINT) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - if (validateColumnName(pColField->name) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - SSchema* pSchema = tscGetTableSchema(pTableMeta); - int32_t nLen = 0; - - for (int32_t i = 0; i < numOfCols; ++i) { - nLen += pSchema[i].bytes; - } - - if (pColField->bytes <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - // length less than TSDB_MAX_BYTES_PER_ROW - if (nLen + pColField->bytes > TSDB_MAX_BYTES_PER_ROW) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - // field name must be unique - for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { - if (strncasecmp(pColField->name, pSchema[i].name, sizeof(pColField->name) - 1) == 0) { - //return tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pColField->name, NULL); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "duplicated column names"); - } - } - - return TSDB_CODE_SUCCESS; -} - -/* is contained in pFieldList or not */ -static bool has(SArray* pFieldList, int32_t startIdx, const char* name) { - size_t numOfCols = taosArrayGetSize(pFieldList); - for (int32_t j = startIdx; j < numOfCols; ++j) { - TAOS_FIELD* field = taosArrayGet(pFieldList, j); - if (strncasecmp(name, field->name, sizeof(field->name) - 1) == 0) return true; - } - - return false; -} - -static char* getAccountId(SSqlObj* pSql) { return pSql->pTscObj->acctId; } - -/* length limitation, strstr cannot be applied */ -static int32_t getDelimiterIndex(SStrToken* pTableName) { - for (uint32_t i = 0; i < pTableName->n; ++i) { - if (pTableName->z[i] == TS_PATH_DELIMITER[0]) { - return i; - } - } - return -1; -} - -int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* xlen) { - int32_t totalLen = 0; - - if (account != NULL) { - int32_t len = (int32_t)strlen(account); - strcpy(fullName, account); - fullName[len] = TS_PATH_DELIMITER[0]; - totalLen += (len + 1); - } - - /* db name is not specified, the tableName dose not include db name */ - if (pDB != NULL) { - if (pDB->n >= TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN || pDB->n == 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - memcpy(&fullName[totalLen], pDB->z, pDB->n); - totalLen += pDB->n; - } - - if (tableName != NULL) { - if (pDB != NULL) { - fullName[totalLen] = TS_PATH_DELIMITER[0]; - totalLen += 1; - - /* here we only check the table name length limitation */ - if (!tscValidateTableNameLength(tableName->n)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else { // pDB == NULL, the db prefix name is specified in tableName - /* the length limitation includes tablename + dbname + sep */ - if (tableName->n >= TSDB_TABLE_NAME_LEN + TSDB_DB_NAME_LEN) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - memcpy(&fullName[totalLen], tableName->z, tableName->n); - totalLen += tableName->n; - } - - if (xlen != NULL) { - *xlen = totalLen; - } - - if (totalLen < TSDB_TABLE_FNAME_LEN) { - fullName[totalLen] = 0; - } - - return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_OPERATION; -} - -void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, uint64_t tableUid) { - SSchema s = {.type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = TSDB_KEYSIZE, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnListInsert(pQueryInfo->colList, PRIMARYKEY_TIMESTAMP_COL_INDEX, tableUid, &s); -} - -static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t exprIndex, tSqlExprItem* pItem) { - const char* msg1 = "invalid column name, illegal column type, or columns in arithmetic expression from two tables"; - const char* msg2 = "invalid arithmetic expression in select clause"; - const char* msg3 = "tag columns can not be used in arithmetic expression"; - const char* msg4 = "columns from different table mixed up in arithmetic expression"; - - SColumnList columnList = {0}; - int32_t arithmeticType = NON_ARITHMEIC_EXPR; - - if (validateArithmeticSQLExpr(pCmd, pItem->pNode, pQueryInfo, &columnList, &arithmeticType) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - int32_t tableIndex = columnList.ids[0].tableIndex; - if (arithmeticType == NORMAL_ARITHMETIC) { - pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; - - // all columns in arithmetic expression must belong to the same table - for (int32_t f = 1; f < columnList.num; ++f) { - if (columnList.ids[f].tableIndex != tableIndex) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - } - - // expr string is set as the parameter of function - SColumnIndex index = {.tableIndex = tableIndex}; - - SExprInfo* pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), - getNewResColId(pCmd), sizeof(double), false); - - char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->exprToken.z; - size_t len = TMIN(sizeof(pExpr->base.aliasName), pItem->pNode->exprToken.n + 1); - tstrncpy(pExpr->base.aliasName, name, len); - - tExprNode* pNode = NULL; - SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - - int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo, colList, NULL); - if (ret != TSDB_CODE_SUCCESS) { - taosArrayDestroy(colList); - tExprTreeDestroy(pNode, NULL); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - // check for if there is a tag in the arithmetic express - size_t numOfNode = taosArrayGetSize(colList); - for(int32_t k = 0; k < numOfNode; ++k) { - SColIndex* pIndex = taosArrayGet(colList, k); - if (TSDB_COL_IS_TAG(pIndex->flag)) { - tExprTreeDestroy(pNode, NULL); - taosArrayDestroy(colList); - - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - - SBufferWriter bw = tbufInitWriter(NULL, false); - - TRY(0) { - exprTreeToBinary(&bw, pNode); - } CATCH(code) { - tbufCloseWriter(&bw); - UNUSED(code); - // TODO: other error handling - } END_TRY - - len = tbufTell(&bw); - char* c = tbufGetData(&bw, false); - - // set the serialized binary string as the parameter of arithmetic expression - tscExprAddParams(&pExpr->base, c, TSDB_DATA_TYPE_BINARY, (int32_t)len); - insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->base.aliasName, pExpr); - - // add ts column - tscInsertPrimaryTsSourceColumn(pQueryInfo, pExpr->base.uid); - - tbufCloseWriter(&bw); - taosArrayDestroy(colList); - tExprTreeDestroy(pNode, NULL); - } else { - columnList.num = 0; - columnList.ids[0] = (SColumnIndex) {0, 0}; - - char rawName[TSDB_COL_NAME_LEN] = {0}; - char aliasName[TSDB_COL_NAME_LEN] = {0}; - getColumnName(pItem, aliasName, rawName, TSDB_COL_NAME_LEN); - - insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, aliasName, NULL); - - int32_t slot = tscNumOfFields(pQueryInfo) - 1; - SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, slot); - assert(pInfo->pExpr == NULL); - - SExprInfo* pExprInfo = calloc(1, sizeof(SExprInfo)); - - // arithmetic expression always return result in the format of double float - pExprInfo->base.resBytes = sizeof(double); - pExprInfo->base.interBytes = 0; - pExprInfo->base.resType = TSDB_DATA_TYPE_DOUBLE; - - pExprInfo->base.functionId = TSDB_FUNC_ARITHM; - pExprInfo->base.numOfParams = 1; - pExprInfo->base.resColId = getNewResColId(pCmd); - strncpy(pExprInfo->base.aliasName, aliasName, tListLen(pExprInfo->base.aliasName)); - strncpy(pExprInfo->base.token, rawName, tListLen(pExprInfo->base.token)); - - int32_t ret = exprTreeFromSqlExpr(pCmd, &pExprInfo->pExpr, pItem->pNode, pQueryInfo, NULL, &(pExprInfo->base.uid)); - if (ret != TSDB_CODE_SUCCESS) { - tExprTreeDestroy(pExprInfo->pExpr, NULL); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); - } - - pInfo->pExpr = pExprInfo; - - SBufferWriter bw = tbufInitWriter(NULL, false); - - TRY(0) { - exprTreeToBinary(&bw, pInfo->pExpr->pExpr); - } CATCH(code) { - tbufCloseWriter(&bw); - UNUSED(code); - // TODO: other error handling - } END_TRY - - SSqlExpr* pSqlExpr = &pInfo->pExpr->base; - pSqlExpr->param[0].nLen = (int16_t) tbufTell(&bw); - pSqlExpr->param[0].pz = tbufGetData(&bw, true); - pSqlExpr->param[0].nType = TSDB_DATA_TYPE_BINARY; - -// tbufCloseWriter(&bw); // TODO there is a memory leak - } - - return TSDB_CODE_SUCCESS; -} - -static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSqlExprItem* pItem, int32_t colId) { - SExprInfo* pExpr = doAddOneProjectCol(pQueryInfo, pIndex->columnIndex, pIndex->tableIndex, colId); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); - - char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; - tstrncpy(pExpr->base.aliasName, colName, sizeof(pExpr->base.aliasName)); - - SColumnList ids = {0}; - ids.num = 1; - ids.ids[0] = *pIndex; - - if (pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX || pIndex->columnIndex == TSDB_UD_COLUMN_INDEX || - pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta)) { - ids.num = 0; - } - - insertResultField(pQueryInfo, startPos, &ids, pExpr->base.resBytes, (int8_t)pExpr->base.resType, pExpr->base.aliasName, pExpr); -} - -static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) { - // primary timestamp column has been added already - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return; - } - } - - - // set the constant column value always attached to first table. - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, PRIMARYKEY_TIMESTAMP_COL_INDEX); - - // add the timestamp column into the output columns - SColumnIndex index = {0}; // primary timestamp column info - int32_t numOfCols = (int32_t)tscNumOfExprs(pQueryInfo); - tscAddFuncInSelectClause(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL, getNewResColId(pCmd)); - - SInternalField* pSupInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfCols); - pSupInfo->visible = false; - - pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; -} - -static bool hasNoneUserDefineExpr(SQueryInfo* pQueryInfo) { - size_t numOfExprs = taosArrayGetSize(pQueryInfo->exprList); - for (int32_t i = 0; i < numOfExprs; ++i) { - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, i); - - if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) { - continue; - } - - return true; - } - - return false; -} - -void genUdfList(SArray* pUdfInfo, tSqlExpr *pNode) { - if (pNode == NULL) { - return; - } - - if (pNode->type == SQL_NODE_EXPR) { - genUdfList(pUdfInfo, pNode->pLeft); - genUdfList(pUdfInfo, pNode->pRight); - return; - } - - if (pNode->type == SQL_NODE_SQLFUNCTION) { - pNode->functionId = isValidFunction(pNode->Expr.operand.z, pNode->Expr.operand.n); - if (pNode->functionId < 0) { // extract all possible user defined function - struct SUdfInfo info = {0}; - info.name = strndup(pNode->Expr.operand.z, pNode->Expr.operand.n); - int32_t functionId = (int32_t)taosArrayGetSize(pUdfInfo) * (-1) - 1; - info.functionId = functionId; - - taosArrayPush(pUdfInfo, &info); - } - } -} - -/* -static int32_t checkForUdf(SSqlObj* pSql, SQueryInfo* pQueryInfo, SArray* pSelection) { - if (pQueryInfo->pUdfInfo != NULL) { - return TSDB_CODE_SUCCESS; - } - - pQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(struct SUdfInfo)); - - size_t nExpr = taosArrayGetSize(pSelection); - - for (int32_t i = 0; i < nExpr; ++i) { - tSqlExprItem* pItem = taosArrayGet(pSelection, i); - - int32_t type = pItem->pNode->type; - if (type == SQL_NODE_EXPR || type == SQL_NODE_SQLFUNCTION) { - genUdfList(pQueryInfo->pUdfInfo, pItem->pNode); - } - } - - if (taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { - return tscGetUdfFromNode(pSql, pQueryInfo); - } else { - return TSDB_CODE_SUCCESS; - } -} -*/ - -static SUdfInfo* isValidUdf(SArray* pUdfInfo, const char* name, int32_t len) { - if(pUdfInfo == NULL){ - tscError("udfinfo is null"); - return NULL; - } - size_t t = taosArrayGetSize(pUdfInfo); - for(int32_t i = 0; i < t; ++i) { - SUdfInfo* pUdf = taosArrayGet(pUdfInfo, i); - if (strlen(pUdf->name) == len && strncasecmp(pUdf->name, name, len) == 0) { - return pUdf; - } - } - - return NULL; -} - -int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelNodeList, bool joinQuery, - bool timeWindowQuery, bool outerQuery) { - assert(pSelNodeList != NULL && pCmd != NULL); - - const char* msg1 = "too many items in selection clause"; - const char* msg2 = "functions or others can not be mixed up"; - const char* msg3 = "not support query expression"; - const char* msg4 = "not support distinct mixed with proj/agg func"; - const char* msg5 = "invalid function name"; - const char* msg6 = "not support distinct mixed with join"; - const char* msg7 = "not support distinct mixed with groupby"; - const char* msg8 = "not support distinct in nest query"; - const char* msg9 = "_block_dist not support subquery, only support stable/table"; - - // too many result columns not support order by in query - if (taosArrayGetSize(pSelNodeList) > TSDB_MAX_COLUMNS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pQueryInfo->colList == NULL) { - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - } - - - bool hasDistinct = false; - bool hasAgg = false; - size_t numOfExpr = taosArrayGetSize(pSelNodeList); - int32_t distIdx = -1; - for (int32_t i = 0; i < numOfExpr; ++i) { - int32_t outputIndex = (int32_t)tscNumOfExprs(pQueryInfo); - tSqlExprItem* pItem = taosArrayGet(pSelNodeList, i); - if (hasDistinct == false) { - hasDistinct = (pItem->distinct == true); - distIdx = hasDistinct ? i : -1; - } - - int32_t type = pItem->pNode->type; - if (type == SQL_NODE_SQLFUNCTION) { - hasAgg = true; - if (hasDistinct) break; - - pItem->pNode->functionId = isValidFunction(pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n); - - if (pItem->pNode->functionId == TSDB_FUNC_BLKINFO && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - - SUdfInfo* pUdfInfo = NULL; - if (pItem->pNode->functionId < 0) { - pUdfInfo = isValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n); - if (pUdfInfo == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - pItem->pNode->functionId = pUdfInfo->functionId; - } - - // sql function in selection clause, append sql function info in pSqlCmd structure sequentially - if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, pItem, true, pUdfInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else if (type == SQL_NODE_TABLE_COLUMN || type == SQL_NODE_VALUE) { - // use the dynamic array list to decide if the function is valid or not - // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 - if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem, outerQuery) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else if (type == SQL_NODE_EXPR) { - int32_t code = handleArithmeticExpr(pCmd, pQueryInfo, i, pItem); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (pQueryInfo->fieldsInfo.numOfOutput > TSDB_MAX_COLUMNS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - //TODO(dengyihao), refactor as function - //handle distinct func mixed with other func - if (hasDistinct == true) { - if (distIdx != 0 || hasAgg) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - if (joinQuery) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - if (pQueryInfo->groupbyExpr.numOfGroupCols != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - if (pQueryInfo->pDownstream != NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); - } - - pQueryInfo->distinct = true; - } - - - // there is only one user-defined column in the final result field, add the timestamp column. - size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); - if ((numOfSrcCols <= 0 || !hasNoneUserDefineExpr(pQueryInfo)) && !tscQueryTags(pQueryInfo) && !tscQueryBlockInfo(pQueryInfo)) { - addPrimaryTsColIntoResult(pQueryInfo, pCmd); - } - - if (!functionCompatibleCheck(pQueryInfo, joinQuery, timeWindowQuery)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pColList, int16_t bytes, - int8_t type, char* fieldName, SExprInfo* pSqlExpr) { - for (int32_t i = 0; i < pColList->num; ++i) { - int32_t tableIndex = pColList->ids[i].tableIndex; - STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[tableIndex]->pTableMeta; - - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - if (pColList->ids[i].columnIndex >= numOfCols) { - continue; - } - - uint64_t uid = pTableMeta->id.uid; - SSchema* pSchema = tscGetTableSchema(pTableMeta); - tscColumnListInsert(pQueryInfo->colList, pColList->ids[i].columnIndex, uid, &pSchema[pColList->ids[i].columnIndex]); - } - - TAOS_FIELD f = tscCreateField(type, fieldName, bytes); - SInternalField* pInfo = tscFieldInfoInsert(&pQueryInfo->fieldsInfo, outputIndex, &f); - pInfo->pExpr = pSqlExpr; - - return TSDB_CODE_SUCCESS; -} - -SExprInfo* doAddOneProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tableIndex, int32_t colId) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, colIndex); - - int16_t functionId = (int16_t)((colIndex >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); - SColumnIndex index = {.tableIndex = tableIndex,}; - - if (functionId == TSDB_FUNC_TAGPRJ) { - index.columnIndex = colIndex - tscGetNumOfColumns(pTableMeta); - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pSchema); - } else { - index.columnIndex = colIndex; - } - - return tscExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, colId, 0, - (functionId == TSDB_FUNC_TAGPRJ)); -} - -SExprInfo* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, - SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag, int16_t colId) { - SExprInfo* pExpr = tscExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, - pColSchema->bytes, colId, 0, TSDB_COL_IS_TAG(flag)); - tstrncpy(pExpr->base.aliasName, pColSchema->name, sizeof(pExpr->base.aliasName)); - tstrncpy(pExpr->base.token, pColSchema->name, sizeof(pExpr->base.token)); - - SColumnList ids = createColumnList(1, pIndex->tableIndex, pIndex->columnIndex); - if (TSDB_COL_IS_TAG(flag)) { - ids.num = 0; - } - - insertResultField(pQueryInfo, outputColIndex, &ids, pColSchema->bytes, pColSchema->type, pColSchema->name, pExpr); - - pExpr->base.colInfo.flag = flag; - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); - - if (TSDB_COL_IS_TAG(flag)) { - tscColumnListInsert(pTableMetaInfo->tagColList, pIndex->columnIndex, pTableMetaInfo->pTableMeta->id.uid, pColSchema); - } - - return pExpr; -} - -static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos, SSqlCmd* pCmd) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); - - int32_t numOfTotalColumns = 0; - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - SSchema* pSchema = tscGetTableSchema(pTableMeta); - - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - numOfTotalColumns = tinfo.numOfColumns + tinfo.numOfTags; - } else { - numOfTotalColumns = tinfo.numOfColumns; - } - - for (int32_t j = 0; j < numOfTotalColumns; ++j) { - SExprInfo* pExpr = doAddOneProjectCol(pQueryInfo, j, pIndex->tableIndex, getNewResColId(pCmd)); - tstrncpy(pExpr->base.aliasName, pSchema[j].name, sizeof(pExpr->base.aliasName)); - - pIndex->columnIndex = j; - SColumnList ids = {0}; - ids.ids[0] = *pIndex; - ids.num = 1; - - insertResultField(pQueryInfo, startPos + j, &ids, pSchema[j].bytes, pSchema[j].type, pSchema[j].name, pExpr); - } - - return numOfTotalColumns; -} - -SSchema tGetUserSpecifiedColumnSchema(SVariant* pVal, SStrToken* exprStr, const char* name) { - SSchema s = {0}; - - s.type = pVal->nType; - if (s.type == TSDB_DATA_TYPE_BINARY || s.type == TSDB_DATA_TYPE_NCHAR) { - s.bytes = (int16_t)(pVal->nLen + VARSTR_HEADER_SIZE); - } else { - s.bytes = tDataTypes[pVal->nType].bytes; - } - - s.colId = TSDB_UD_COLUMN_INDEX; - if (name != NULL) { - tstrncpy(s.name, name, sizeof(s.name)); - } else { - size_t tlen = TMIN(sizeof(s.name), exprStr->n + 1); - tstrncpy(s.name, exprStr->z, tlen); - strdequote(s.name); - } - - return s; -} - -int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem, bool outerQuery) { - const char* msg1 = "tag for normal table query is not allowed"; - const char* msg2 = "invalid column name"; - const char* msg3 = "tbname not allowed in outer query"; - - int32_t startPos = (int32_t)tscNumOfExprs(pQueryInfo); - int32_t tokenId = pItem->pNode->tokenId; - - if (tokenId == TK_ALL) { // project on all fields - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getTableIndexByName(&pItem->pNode->columnName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - // all meters columns are required - if (index.tableIndex == COLUMN_INDEX_INITIAL_VAL) { // all table columns are required. - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - index.tableIndex = i; - int32_t inc = doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos, pCmd); - startPos += inc; - } - } else { - doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos, pCmd); - } - - // add the primary timestamp column even though it is not required by user - STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[index.tableIndex]->pTableMeta; - if (pTableMeta->tableType != TSDB_TEMP_TABLE) { - tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMeta->id.uid); - } - } else if (tokenId == TK_STRING || tokenId == TK_INTEGER || tokenId == TK_FLOAT) { // simple column projection query - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - - // user-specified constant value as a new result column - index.columnIndex = (pQueryInfo->udColumnId--); - index.tableIndex = 0; - - SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->value, &pItem->pNode->exprToken, pItem->aliasName); - SExprInfo* pExpr = tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC, - getNewResColId(pCmd)); - - // NOTE: the first parameter is reserved for the tag column id during join query process. - pExpr->base.numOfParams = 2; - tVariantAssign(&pExpr->base.param[1], &pItem->pNode->value); - } else if (tokenId == TK_ID) { - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - - if (getColumnIndexByName(&pItem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - if (outerQuery) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - bool existed = false; - SSchema* pSchema = pTableMetaInfo->pTableMeta->schema; - for (int32_t i = 0; i < numOfCols; ++i) { - if (strncasecmp(pSchema[i].name, TSQL_TBNAME_L, tListLen(pSchema[i].name)) == 0) { - existed = true; - index.columnIndex = i; - break; - } - } - - if (!existed) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - SSchema colSchema = pSchema[index.columnIndex]; - char name[TSDB_COL_NAME_LEN] = {0}; - getColumnName(pItem, name, colSchema.name, sizeof(colSchema.name) - 1); - - tstrncpy(colSchema.name, name, TSDB_COL_NAME_LEN); - /*SExprInfo* pExpr = */ tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, - TSDB_COL_NORMAL, getNewResColId(pCmd)); - } else { - SSchema colSchema = *tGetTbnameColumnSchema(); - char name[TSDB_COL_NAME_LEN] = {0}; - getColumnName(pItem, name, colSchema.name, sizeof(colSchema.name) - 1); - - tstrncpy(colSchema.name, name, TSDB_COL_NAME_LEN); - /*SExprInfo* pExpr = */ tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, - TSDB_COL_TAG, getNewResColId(pCmd)); - } - } else { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - addProjectQueryCol(pQueryInfo, startPos, &index, pItem, getNewResColId(pCmd)); - pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; - } - - // add the primary timestamp column even though it is not required by user - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - if (!UTIL_TABLE_IS_TMP_TABLE(pTableMetaInfo)) { - tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); - } - } else { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc, - const char* name, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult, - SUdfInfo* pUdfInfo) { - const char* msg1 = "not support column types"; - - int32_t f = cvtFunc.execFuncId; - if (f == TSDB_FUNC_SPREAD) { - int32_t t1 = pSchema->type; - if (IS_VAR_DATA_TYPE(t1) || t1 == TSDB_DATA_TYPE_BOOL) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - return -1; - } - } - - int16_t resType = 0; - int16_t resBytes = 0; - int32_t interBufSize = 0; - - getResultDataInfo(pSchema->type, pSchema->bytes, f, 0, &resType, &resBytes, &interBufSize, 0, false, pUdfInfo); - SExprInfo* pExpr = tscExprAppend(pQueryInfo, f, pColIndex, resType, resBytes, getNewResColId(pCmd), interBufSize, false); - tstrncpy(pExpr->base.aliasName, name, tListLen(pExpr->base.aliasName)); - - if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != f) { - pExpr->base.colInfo.flag |= TSDB_COL_NULL; - } - - // set reverse order scan data blocks for last query - if (f == TSDB_FUNC_LAST) { - pExpr->base.numOfParams = 1; - pExpr->base.param[0].i64 = TSDB_ORDER_DESC; - pExpr->base.param[0].nType = TSDB_DATA_TYPE_INT; - } - - // for all queries, the timestamp column needs to be loaded - SSchema s = {.colId = PRIMARYKEY_TIMESTAMP_COL_INDEX, .bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP,}; - tscColumnListInsert(pQueryInfo->colList, PRIMARYKEY_TIMESTAMP_COL_INDEX, pExpr->base.uid, &s); - - // if it is not in the final result, do not add it - SColumnList ids = createColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); - if (finalResult) { - insertResultField(pQueryInfo, resColIdx, &ids, resBytes, (int8_t)resType, pExpr->base.aliasName, pExpr); - } else { - tscColumnListInsert(pQueryInfo->colList, ids.ids[0].columnIndex, pExpr->base.uid, pSchema); - } - - return TSDB_CODE_SUCCESS; -} - -void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrToken* pToken, bool multiCols) { - if (pItem->aliasName != NULL) { - tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); - } else if (multiCols) { - char uname[TSDB_COL_NAME_LEN] = {0}; - int32_t len = TMIN(pToken->n + 1, TSDB_COL_NAME_LEN); - tstrncpy(uname, pToken->z, len); - - if (tsKeepOriginalColumnName) { // keep the original column name - tstrncpy(name, uname, TSDB_COL_NAME_LEN); - } else { - int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].name) + 2 + 1; - char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].name) + 2 + 1] = {0}; - snprintf(tmp, size, "%s(%s)", aAggs[functionId].name, uname); - - tstrncpy(name, tmp, TSDB_COL_NAME_LEN); - } - } else { // use the user-input result column name - int32_t len = TMIN(pItem->pNode->exprToken.n + 1, TSDB_COL_NAME_LEN); - tstrncpy(name, pItem->pNode->exprToken.z, len); - } -} - -static void updateLastScanOrderIfNeeded(SQueryInfo* pQueryInfo) { - if (pQueryInfo->sessionWindow.gap > 0 || - pQueryInfo->stateWindow || - taosArrayGetSize(pQueryInfo->pUpstream) > 0 || - tscGroupbyColumn(pQueryInfo)) { - size_t numOfExpr = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExpr; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId != TSDB_FUNC_LAST && pExpr->base.functionId != TSDB_FUNC_LAST_DST) { - continue; - } - - pExpr->base.numOfParams = 1; - pExpr->base.param->i64 = TSDB_ORDER_ASC; - pExpr->base.param->nType = TSDB_DATA_TYPE_INT; - } - } -} - -static UNUSED_FUNC void updateFunctionInterBuf(SQueryInfo* pQueryInfo, bool superTable, SUdfInfo* pUdfInfo) { - size_t numOfExpr = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExpr; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - - int32_t param = (int32_t)pExpr->base.param[0].i64; - getResultDataInfo(pExpr->base.colType, pExpr->base.colBytes, pExpr->base.functionId, param, &pExpr->base.resType, &pExpr->base.resBytes, - &pExpr->base.interBytes, 0, superTable, pUdfInfo); - } -} - -int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult, - SUdfInfo* pUdfInfo) { - STableMetaInfo* pTableMetaInfo = NULL; - int32_t functionId = pItem->pNode->functionId; - - const char* msg1 = "not support column types"; - const char* msg2 = "invalid parameters"; - const char* msg3 = "illegal column name"; - const char* msg4 = "invalid table name"; - const char* msg5 = "parameter is out of range [0, 100]"; - const char* msg6 = "functions applied to tags are not allowed"; - const char* msg7 = "normal table can not apply this function"; - const char* msg8 = "multi-columns selection does not support alias column name"; - const char* msg9 = "diff/derivative can no be applied to unsigned numeric type"; - const char* msg10 = "derivative duration should be greater than 1 Second"; - const char* msg11 = "third parameter in derivative should be 0 or 1"; - const char* msg12 = "parameter is out of range [1, 100]"; - - switch (functionId) { - case TSDB_FUNC_COUNT: { - /* more than one parameter for count() function */ - if (pItem->pNode->Expr.paramList != NULL && taosArrayGetSize(pItem->pNode->Expr.paramList) != 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - SExprInfo* pExpr = NULL; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - - if (pItem->pNode->Expr.paramList != NULL) { - tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->Expr.paramList, 0); - SStrToken* pToken = &pParamElem->pNode->columnName; - int16_t tokenId = pParamElem->pNode->tokenId; - if ((pToken->z == NULL || pToken->n == 0) && (TK_INTEGER != tokenId)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - // select count(table.*), select count(1), count(2) - if (tokenId == TK_ALL || tokenId == TK_INTEGER) { - // check if the table name is valid or not - SStrToken tmpToken = pParamElem->pNode->columnName; - - if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, false); - } else { - // count the number of table created according to the super table - if (getColumnIndexByName(pToken, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - - // count tag is equalled to count(tbname) - bool isTag = false; - if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta) || index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - index.columnIndex = TSDB_TBNAME_COLUMN_INDEX; - isTag = true; - } - - int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, isTag); - } - } else { // count(*) is equalled to count(primary_timestamp_key) - index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pCmd), size, false); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - - memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); - getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token,sizeof(pExpr->base.aliasName) - 1); - - SColumnList list = createColumnList(1, index.tableIndex, index.columnIndex); - if (finalResult) { - int32_t numOfOutput = tscNumOfFields(pQueryInfo); - insertResultField(pQueryInfo, numOfOutput, &list, sizeof(int64_t), TSDB_DATA_TYPE_BIGINT, pExpr->base.aliasName, pExpr); - } else { - for (int32_t i = 0; i < list.num; ++i) { - SSchema* ps = tscGetTableSchema(pTableMetaInfo->pTableMeta); - tscColumnListInsert(pQueryInfo->colList, list.ids[i].columnIndex, pTableMetaInfo->pTableMeta->id.uid, - &ps[list.ids[i].columnIndex]); - } - } - - // the time stamp may be always needed - if (index.tableIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); - } - - return TSDB_CODE_SUCCESS; - } - - case TSDB_FUNC_SUM: - case TSDB_FUNC_AVG: - case TSDB_FUNC_RATE: - case TSDB_FUNC_IRATE: - case TSDB_FUNC_TWA: - case TSDB_FUNC_MIN: - case TSDB_FUNC_MAX: - case TSDB_FUNC_DIFF: - case TSDB_FUNC_DERIVATIVE: - case TSDB_FUNC_STDDEV: - case TSDB_FUNC_LEASTSQR: { - // 1. valid the number of parameters - int32_t numOfParams = (pItem->pNode->Expr.paramList == NULL)? 0: (int32_t) taosArrayGetSize(pItem->pNode->Expr.paramList); - - // no parameters or more than one parameter for function - if (pItem->pNode->Expr.paramList == NULL || - (functionId != TSDB_FUNC_LEASTSQR && functionId != TSDB_FUNC_DERIVATIVE && numOfParams != 1) || - ((functionId == TSDB_FUNC_LEASTSQR || functionId == TSDB_FUNC_DERIVATIVE) && numOfParams != 3)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->Expr.paramList, 0); - if (pParamElem->pNode->tokenId != TK_ALL && pParamElem->pNode->tokenId != TK_ID) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if ((getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableComInfo info = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - // functions can not be applied to tags - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta))) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - // 2. check if sql function can be applied on this column data type - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - - if (!IS_NUMERIC_TYPE(pSchema->type)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } else if (IS_UNSIGNED_NUMERIC_TYPE(pSchema->type) && (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - - int16_t resultType = 0; - int16_t resultSize = 0; - int32_t intermediateResSize = 0; - - if (getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &resultType, &resultSize, - &intermediateResSize, 0, false, NULL) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // set the first column ts for diff query - if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; - SExprInfo* pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, - TSDB_KEYSIZE, getNewResColId(pCmd), TSDB_KEYSIZE, false); - - SColumnList ids = createColumnList(1, 0, 0); - insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr); - } - - SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), intermediateResSize, false); - - if (functionId == TSDB_FUNC_LEASTSQR) { // set the leastsquares parameters - char val[8] = {0}; - if (tVariantDump(&pParamElem[1].pNode->value, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, DOUBLE_BYTES); - - memset(val, 0, tListLen(val)); - if (tVariantDump(&pParamElem[2].pNode->value, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, DOUBLE_BYTES); - } else if (functionId == TSDB_FUNC_IRATE) { - int64_t prec = info.precision; - tscExprAddParams(&pExpr->base, (char*)&prec, TSDB_DATA_TYPE_BIGINT, LONG_BYTES); - } else if (functionId == TSDB_FUNC_DERIVATIVE) { - char val[8] = {0}; - - int64_t tickPerSec = 0; - if (tVariantDump(&pParamElem[1].pNode->value, (char*) &tickPerSec, TSDB_DATA_TYPE_BIGINT, true) < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (info.precision == TSDB_TIME_PRECISION_MILLI) { - tickPerSec /= TSDB_TICK_PER_SECOND(TSDB_TIME_PRECISION_MICRO); - } else if (info.precision == TSDB_TIME_PRECISION_MICRO) { - tickPerSec /= TSDB_TICK_PER_SECOND(TSDB_TIME_PRECISION_MILLI); - } - - if (tickPerSec <= 0 || tickPerSec < TSDB_TICK_PER_SECOND(info.precision)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg10); - } - - tscExprAddParams(&pExpr->base, (char*) &tickPerSec, TSDB_DATA_TYPE_BIGINT, LONG_BYTES); - memset(val, 0, tListLen(val)); - - if (tVariantDump(&pParamElem[2].pNode->value, val, TSDB_DATA_TYPE_BIGINT, true) < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int64_t v = *(int64_t*) val; - if (v != 0 && v != 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg11); - } - - tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_BIGINT, LONG_BYTES); - } - - SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); - - memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); - getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token,sizeof(pExpr->base.aliasName) - 1); - - if (finalResult) { - int32_t numOfOutput = tscNumOfFields(pQueryInfo); - insertResultField(pQueryInfo, numOfOutput, &ids, pExpr->base.resBytes, (int32_t)pExpr->base.resType, - pExpr->base.aliasName, pExpr); - } else { - assert(ids.num == 1); - tscColumnListInsert(pQueryInfo->colList, ids.ids[0].columnIndex, pExpr->base.uid, pSchema); - } - tscInsertPrimaryTsSourceColumn(pQueryInfo, pExpr->base.uid); - - return TSDB_CODE_SUCCESS; - } - - case TSDB_FUNC_FIRST: - case TSDB_FUNC_LAST: - case TSDB_FUNC_SPREAD: - case TSDB_FUNC_LAST_ROW: - case TSDB_FUNC_INTERP: { - bool requireAllFields = (pItem->pNode->Expr.paramList == NULL); - - // NOTE: has time range condition or normal column filter condition, the last_row query will be transferred to last query - SConvertFunc cvtFunc = {.originFuncId = functionId, .execFuncId = functionId}; - if (functionId == TSDB_FUNC_LAST_ROW && ((!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER)) || - (hasNormalColumnFilter(pQueryInfo)) || - taosArrayGetSize(pQueryInfo->pUpstream)>0)) { - cvtFunc.execFuncId = TSDB_FUNC_LAST; - } - - if (!requireAllFields) { - if (taosArrayGetSize(pItem->pNode->Expr.paramList) < 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (taosArrayGetSize(pItem->pNode->Expr.paramList) > 1 && (pItem->aliasName != NULL && strlen(pItem->aliasName) > 0)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); - } - - /* in first/last function, multiple columns can be add to resultset */ - for (int32_t i = 0; i < taosArrayGetSize(pItem->pNode->Expr.paramList); ++i) { - tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->Expr.paramList, i); - if (pParamElem->pNode->tokenId != TK_ALL && pParamElem->pNode->tokenId != TK_ID) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - - if (pParamElem->pNode->tokenId == TK_ALL) { // select table.* - SStrToken tmpToken = pParamElem->pNode->columnName; - - if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - char name[TSDB_COL_NAME_LEN] = {0}; - for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { - index.columnIndex = j; - SStrToken t = {.z = pSchema[j].name, .n = (uint32_t)strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)}; - setResultColName(name, pItem, cvtFunc.originFuncId, &t, true); - - if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, - finalResult, pUdfInfo) != 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - } else { - if (getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - - // functions can not be applied to tags - if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - char name[TSDB_COL_NAME_LEN] = {0}; - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - - bool multiColOutput = taosArrayGetSize(pItem->pNode->Expr.paramList) > 1; - setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->columnName, multiColOutput); - - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex++, &index, finalResult, pUdfInfo) != 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - } - - } else { // select * from xxx - int32_t numOfFields = 0; - - // multicolumn selection does not support alias name - if (pItem->aliasName != NULL && strlen(pItem->aliasName) > 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); - } - - for (int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, j); - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { - SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - - char name[TSDB_COL_NAME_LEN] = {0}; - SStrToken t = {.z = pSchema[i].name, .n = (uint32_t)strnlen(pSchema[i].name, TSDB_COL_NAME_LEN)}; - setResultColName(name, pItem, cvtFunc.originFuncId, &t, true); - - if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, - finalResult, pUdfInfo) != 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - colIndex++; - } - - numOfFields += tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - } - } - return TSDB_CODE_SUCCESS; - } - - case TSDB_FUNC_TOP: - case TSDB_FUNC_BOTTOM: - case TSDB_FUNC_PERCT: - case TSDB_FUNC_APERCT: { - // 1. valid the number of parameters - if (pItem->pNode->Expr.paramList == NULL || taosArrayGetSize(pItem->pNode->Expr.paramList) != 2) { - /* no parameters or more than one parameter for function */ - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->Expr.paramList, 0); - if (pParamElem->pNode->tokenId != TK_ID) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - - // functions can not be applied to tags - if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - // 2. valid the column type - if (!IS_NUMERIC_TYPE(pSchema->type)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - // 3. valid the parameters - if (pParamElem[1].pNode->tokenId == TK_ID) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - tVariant* pVariant = &pParamElem[1].pNode->value; - - int16_t resultType = pSchema->type; - int16_t resultSize = pSchema->bytes; - int32_t interResult = 0; - - char val[8] = {0}; - - SExprInfo* pExpr = NULL; - if (functionId == TSDB_FUNC_PERCT || functionId == TSDB_FUNC_APERCT) { - tVariantDump(pVariant, val, TSDB_DATA_TYPE_DOUBLE, true); - - double dp = GET_DOUBLE_VAL(val); - if (dp < 0 || dp > TOP_BOTTOM_QUERY_LIMIT) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &resultType, &resultSize, &interResult, 0, false, - pUdfInfo); - - /* - * sql function transformation - * for dp = 0, it is actually min, - * for dp = 100, it is max, - */ - tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); - colIndex += 1; // the first column is ts - - pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), interResult, false); - tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - } else { - tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT, true); - - int64_t nTop = GET_INT32_VAL(val); - if (nTop <= 0 || nTop > 100) { // todo use macro - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg12); - } - - // todo REFACTOR - // set the first column ts for top/bottom query - SColumnIndex index1 = {index.tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pCmd), - 0, false); - tstrncpy(pExpr->base.aliasName, aAggs[TSDB_FUNC_TS].name, sizeof(pExpr->base.aliasName)); - - const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX; - SColumnList ids = createColumnList(1, index.tableIndex, TS_COLUMN_INDEX); - insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, - aAggs[TSDB_FUNC_TS].name, pExpr); - - colIndex += 1; // the first column is ts - - pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), resultSize, false); - tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t)); - } - - memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); - getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token,sizeof(pExpr->base.aliasName) - 1); - - // todo refactor: tscColumnListInsert part - SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); - - if (finalResult) { - insertResultField(pQueryInfo, colIndex, &ids, resultSize, (int8_t)resultType, pExpr->base.aliasName, pExpr); - } else { - assert(ids.num == 1); - tscColumnListInsert(pQueryInfo->colList, ids.ids[0].columnIndex, pExpr->base.uid, pSchema); - } - - return TSDB_CODE_SUCCESS; - } - - case TSDB_FUNC_TID_TAG: { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - - // no parameters or more than one parameter for function - if (pItem->pNode->Expr.paramList == NULL || taosArrayGetSize(pItem->pNode->Expr.paramList) != 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - tSqlExprItem* pParamItem = taosArrayGet(pItem->pNode->Expr.paramList, 0); - tSqlExpr* pParam = pParamItem->pNode; - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pParam->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - - // functions can not be applied to normal columns - int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - if (index.columnIndex < numOfCols && index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - if (index.columnIndex > 0) { - index.columnIndex -= numOfCols; - } - - // 2. valid the column type - int16_t colType = 0; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - colType = TSDB_DATA_TYPE_BINARY; - } else { - colType = pSchema[index.columnIndex].type; - } - - if (colType == TSDB_DATA_TYPE_BOOL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMetaInfo->pTableMeta->id.uid, - &pSchema[index.columnIndex]); - SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - - SSchema s = {0}; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - s = *tGetTbnameColumnSchema(); - } else { - s = pTagSchema[index.columnIndex]; - } - - int16_t bytes = 0; - int16_t type = 0; - int32_t inter = 0; - - int32_t ret = getResultDataInfo(s.type, s.bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0, NULL); - assert(ret == TSDB_CODE_SUCCESS); - - s.type = (uint8_t)type; - s.bytes = bytes; - - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); - tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s, TSDB_COL_TAG, getNewResColId(pCmd)); - - return TSDB_CODE_SUCCESS; - } - - case TSDB_FUNC_BLKINFO: { - // no parameters or more than one parameter for function - if (pItem->pNode->Expr.paramList != NULL && taosArrayGetSize(pItem->pNode->Expr.paramList) != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - SColumnIndex index = {.tableIndex = 0, .columnIndex = 0,}; - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - - int32_t inter = 0; - int16_t resType = 0; - int16_t bytes = 0; - - getResultDataInfo(TSDB_DATA_TYPE_INT, 4, TSDB_FUNC_BLKINFO, 0, &resType, &bytes, &inter, 0, 0, NULL); - - SSchema s = {.name = "block_dist", .type = TSDB_DATA_TYPE_BINARY, .bytes = bytes}; - - SExprInfo* pExpr = - tscExprInsert(pQueryInfo, 0, TSDB_FUNC_BLKINFO, &index, resType, bytes, getNewResColId(pCmd), bytes, 0); - tstrncpy(pExpr->base.aliasName, s.name, sizeof(pExpr->base.aliasName)); - - SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); - insertResultField(pQueryInfo, 0, &ids, bytes, s.type, s.name, pExpr); - - pExpr->base.numOfParams = 1; - pExpr->base.param[0].i64 = pTableMetaInfo->pTableMeta->tableInfo.rowSize; - pExpr->base.param[0].nType = TSDB_DATA_TYPE_BIGINT; - - return TSDB_CODE_SUCCESS; - } - - default: { - pUdfInfo = isValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n); - if (pUdfInfo == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - - tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->Expr.paramList, 0);; - if (pParamElem->pNode->tokenId != TK_ID) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - - // functions can not be applied to tags - if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - int32_t inter = 0; - int16_t resType = 0; - int16_t bytes = 0; - getResultDataInfo(TSDB_DATA_TYPE_INT, 4, functionId, 0, &resType, &bytes, &inter, 0, false, pUdfInfo); - - SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resType, bytes, getNewResColId(pCmd), inter, false); - - memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); - getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token, sizeof(pExpr->base.aliasName) - 1); - - SSchema s = {0}; - s.type = (uint8_t)resType; - s.bytes = bytes; - s.colId = pExpr->base.colInfo.colId; - - uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; - SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); - if (finalResult) { - insertResultField(pQueryInfo, colIndex, &ids, pUdfInfo->resBytes, pUdfInfo->resType, pExpr->base.aliasName, pExpr); - } else { - for (int32_t i = 0; i < ids.num; ++i) { - tscColumnListInsert(pQueryInfo->colList, index.columnIndex, uid, &s); - } - } - tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); - return TSDB_CODE_SUCCESS; - } - } - - return TSDB_CODE_TSC_INVALID_OPERATION; -} - -// todo refactor -static SColumnList createColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex) { - assert(num == 1 && tableIndex >= 0); - - SColumnList columnList = {0}; - columnList.num = num; - - int32_t index = num - 1; - columnList.ids[index].tableIndex = tableIndex; - columnList.ids[index].columnIndex = columnIndex; - - return columnList; -} - -void getColumnName(tSqlExprItem* pItem, char* resultFieldName, char* rawName, int32_t nameLength) { - int32_t len = ((int32_t)pItem->pNode->exprToken.n < nameLength) ? (int32_t)pItem->pNode->exprToken.n : nameLength; - strncpy(rawName, pItem->pNode->exprToken.z, len); - - if (pItem->aliasName != NULL) { - int32_t aliasNameLen = (int32_t) strlen(pItem->aliasName); - len = (aliasNameLen < nameLength)? aliasNameLen:nameLength; - strncpy(resultFieldName, pItem->aliasName, len); - } else { - strncpy(resultFieldName, rawName, len); - } -} - -static bool isTablenameToken(SStrToken* token) { - SStrToken tmpToken = *token; - SStrToken tableToken = {0}; - - extractTableNameFromToken(&tmpToken, &tableToken); - return (tmpToken.n == strlen(TSQL_TBNAME_L) && strncasecmp(TSQL_TBNAME_L, tmpToken.z, tmpToken.n) == 0); -} - -static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken* pToken) { - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index)->pTableMeta; - - int32_t numOfCols = tscGetNumOfColumns(pTableMeta) + tscGetNumOfTags(pTableMeta); - SSchema* pSchema = tscGetTableSchema(pTableMeta); - - int16_t columnIndex = COLUMN_INDEX_INITIAL_VAL; - - for (int16_t i = 0; i < numOfCols; ++i) { - if (pToken->n != strlen(pSchema[i].name)) { - continue; - } - - if (strncasecmp(pSchema[i].name, pToken->z, pToken->n) == 0) { - columnIndex = i; - break; - } - } - - return columnIndex; -} - -int32_t doGetColumnIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex, char* msg) { - const char* msg0 = "ambiguous column name"; - const char* msg1 = "invalid column name"; - - if (isTablenameToken(pToken)) { - pIndex->columnIndex = TSDB_TBNAME_COLUMN_INDEX; - } else if (strlen(DEFAULT_PRIMARY_TIMESTAMP_COL_NAME) == pToken->n && - strncasecmp(pToken->z, DEFAULT_PRIMARY_TIMESTAMP_COL_NAME, pToken->n) == 0) { - pIndex->columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX; // just make runtime happy, need fix java test case InsertSpecialCharacterJniTest - } else if (pToken->n == 0) { - pIndex->columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX; // just make runtime happy, need fix java test case InsertSpecialCharacterJniTest - } else { - // not specify the table name, try to locate the table index by column name - if (pIndex->tableIndex == COLUMN_INDEX_INITIAL_VAL) { - for (int16_t i = 0; i < pQueryInfo->numOfTables; ++i) { - int16_t colIndex = doGetColumnIndex(pQueryInfo, i, pToken); - - if (colIndex != COLUMN_INDEX_INITIAL_VAL) { - if (pIndex->columnIndex != COLUMN_INDEX_INITIAL_VAL) { - return invalidOperationMsg(msg, msg0); - } else { - pIndex->tableIndex = i; - pIndex->columnIndex = colIndex; - } - } - } - } else { // table index is valid, get the column index - int16_t colIndex = doGetColumnIndex(pQueryInfo, pIndex->tableIndex, pToken); - if (colIndex != COLUMN_INDEX_INITIAL_VAL) { - pIndex->columnIndex = colIndex; - } - } - - if (pIndex->columnIndex == COLUMN_INDEX_INITIAL_VAL) { - return invalidOperationMsg(msg, msg1); - } - } - - if (COLUMN_INDEX_VALIDE(*pIndex)) { - return TSDB_CODE_SUCCESS; - } else { - return TSDB_CODE_TSC_INVALID_OPERATION; - } -} - -int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - if (pTableToken->n == 0) { // only one table and no table name prefix in column name - if (pQueryInfo->numOfTables == 1) { - pIndex->tableIndex = 0; - } else { - pIndex->tableIndex = COLUMN_INDEX_INITIAL_VAL; - } - - return TSDB_CODE_SUCCESS; - } - - pIndex->tableIndex = COLUMN_INDEX_INITIAL_VAL; - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - char* name = pTableMetaInfo->aliasName; - if (strncasecmp(name, pTableToken->z, pTableToken->n) == 0 && strlen(name) == pTableToken->n) { - pIndex->tableIndex = i; - break; - } - } - - if (pIndex->tableIndex < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - SStrToken tableToken = {0}; - extractTableNameFromToken(pToken, &tableToken); - - if (getTableIndexImpl(&tableToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t getColumnIndexByName(const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex, char* msg) { - if (pQueryInfo->pTableMetaInfo == NULL || pQueryInfo->numOfTables == 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - SStrToken tmpToken = *pToken; - - if (getTableIndexByName(&tmpToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return doGetColumnIndexByName(&tmpToken, pQueryInfo, pIndex, msg); -} - -int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { - SSqlCmd* pCmd = &pSql->cmd; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - pCmd->command = TSDB_SQL_SHOW; - - const char* msg1 = "invalid name"; - const char* msg2 = "wildcard string should be less than %d characters"; - const char* msg3 = "database name too long"; - const char* msg5 = "database name is empty"; - const char* msg6 = "pattern string is empty"; - - /* - * database prefix in pInfo->pMiscInfo->a[0] - * wildcard in like clause in pInfo->pMiscInfo->a[1] - */ - SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; - int16_t showType = pShowInfo->showType; - if (showType == TSDB_MGMT_TABLE_TABLE || showType == TSDB_MGMT_TABLE_METRIC || showType == TSDB_MGMT_TABLE_VGROUP) { - // db prefix in tagCond, show table conds in payload - SStrToken* pDbPrefixToken = &pShowInfo->prefix; - if (pDbPrefixToken->type != 0) { - - if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (pDbPrefixToken->n <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - if (tscValidateName(pDbPrefixToken) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pDbPrefixToken); - if (ret != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - // show table/stable like 'xxxx', set the like pattern for show tables - SStrToken* pPattern = &pShowInfo->pattern; - if (pPattern->type != 0) { - pPattern->n = strdequote(pPattern->z); - - if (pPattern->n <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - if (pPattern->n > tsMaxWildCardsLen){ - char tmp[64] = {0}; - sprintf(tmp, msg2, tsMaxWildCardsLen); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), tmp); - } - } - } else if (showType == TSDB_MGMT_TABLE_VNODES) { - if (pShowInfo->prefix.type == 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "No specified dnode ep"); - } - - if (pShowInfo->prefix.type == TK_STRING) { - pShowInfo->prefix.n = strdequote(pShowInfo->prefix.z); - } - } - return TSDB_CODE_SUCCESS; -} - -int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType) { - const char* msg1 = "invalid connection ID"; - const char* msg2 = "invalid query ID"; - const char* msg3 = "invalid stream ID"; - - SSqlCmd* pCmd = &pSql->cmd; - pCmd->command = pInfo->type; - - SStrToken* idStr = &(pInfo->pMiscInfo->id); - if (idStr->n > TSDB_KILL_MSG_LEN) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - strncpy(pCmd->payload, idStr->z, idStr->n); - - const char delim = ':'; - char* connIdStr = strtok(idStr->z, &delim); - char* queryIdStr = strtok(NULL, &delim); - - int32_t connId = (int32_t)strtol(connIdStr, NULL, 10); - if (connId <= 0) { - memset(pCmd->payload, 0, strlen(pCmd->payload)); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (killType == TSDB_SQL_KILL_CONNECTION) { - return TSDB_CODE_SUCCESS; - } - - int32_t queryId = (int32_t)strtol(queryIdStr, NULL, 10); - if (queryId <= 0) { - memset(pCmd->payload, 0, strlen(pCmd->payload)); - if (killType == TSDB_SQL_KILL_QUERY) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - - return TSDB_CODE_SUCCESS; -} -static int32_t setCompactVnodeInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { - SSqlCmd* pCmd = &pSql->cmd; - pCmd->command = pInfo->type; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscTansformFuncForSTableQuery(SQueryInfo* pQueryInfo) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - assert(tscGetNumOfTags(pTableMetaInfo->pTableMeta) >= 0); - - int16_t bytes = 0; - int16_t type = 0; - int32_t interBytes = 0; - - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t k = 0; k < size; ++k) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, k); - int16_t functionId = aAggs[pExpr->base.functionId].stableFuncId; - - int32_t colIndex = pExpr->base.colInfo.colIndex; - SSchema* pSrcSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, colIndex); - - if ((functionId >= TSDB_FUNC_SUM && functionId <= TSDB_FUNC_TWA) || - (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_STDDEV_DST) || - (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_IRATE)) { - if (getResultDataInfo(pSrcSchema->type, pSrcSchema->bytes, functionId, (int32_t)pExpr->base.param[0].i64, &type, &bytes, - &interBytes, 0, true, NULL) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - tscExprUpdate(pQueryInfo, k, functionId, pExpr->base.colInfo.colIndex, TSDB_DATA_TYPE_BINARY, bytes); - // todo refactor - pExpr->base.interBytes = interBytes; - } - } - - tscFieldInfoUpdateOffset(pQueryInfo); - return TSDB_CODE_SUCCESS; -} - -/* transfer the field-info back to original input format */ -void tscRestoreFuncForSTableQuery(SQueryInfo* pQueryInfo) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return; - } - - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->base.colInfo.colIndex); - - // the final result size and type in the same as query on single table. - // so here, set the flag to be false; - int32_t inter = 0; - - int32_t functionId = pExpr->base.functionId; - if (functionId < 0) { - continue; - } - - if (functionId >= TSDB_FUNC_TS && functionId <= TSDB_FUNC_DIFF) { - continue; - } - - if (functionId == TSDB_FUNC_FIRST_DST) { - functionId = TSDB_FUNC_FIRST; - } else if (functionId == TSDB_FUNC_LAST_DST) { - functionId = TSDB_FUNC_LAST; - } else if (functionId == TSDB_FUNC_STDDEV_DST) { - functionId = TSDB_FUNC_STDDEV; - } - - getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &pExpr->base.resType, &pExpr->base.resBytes, &inter, - 0, false, NULL); - } -} - -bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char* msg1 = "TWA/Diff/Derivative/Irate are not allowed to apply to super table directly"; - const char* msg2 = "TWA/Diff/Derivative/Irate only support group by tbname for super table query"; - const char* msg3 = "functions not support for super table query"; - - // filter sql function not supported by metric query yet. - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - int32_t functionId = tscExprGet(pQueryInfo, i)->base.functionId; - if (functionId < 0) { - continue; - } - - if ((aAggs[functionId].status & TSDB_FUNCSTATE_STABLE) == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return true; - } - } - - if (tscIsTWAQuery(pQueryInfo) || tscIsDiffDerivQuery(pQueryInfo) || tscIsIrateQuery(pQueryInfo)) { - if (pQueryInfo->groupbyExpr.numOfGroupCols == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - return true; - } - - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); - if (pColIndex->colIndex != TSDB_TBNAME_COLUMN_INDEX) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - return true; - } - } else if (tscIsSessionWindowQuery(pQueryInfo)) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return true; - } - - return false; -} - -static bool groupbyTagsOrNull(SQueryInfo* pQueryInfo) { - if (pQueryInfo->groupbyExpr.columnInfo == NULL || - taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo) == 0) { - return true; - } - - size_t s = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); - for (int32_t i = 0; i < s; i++) { - SColIndex* colIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i); - if (colIndex->flag != TSDB_COL_TAG) { - return false; - } - } - - return true; -} - -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool twQuery) { - int32_t startIdx = 0; - int32_t aggUdf = 0; - int32_t scalarUdf = 0; - int32_t prjNum = 0; - int32_t aggNum = 0; - - size_t numOfExpr = tscNumOfExprs(pQueryInfo); - assert(numOfExpr > 0); - - int32_t factor = INT32_MAX; - - // diff function cannot be executed with other function - // arithmetic function can be executed with other arithmetic functions - size_t size = tscNumOfExprs(pQueryInfo); - - for (int32_t i = startIdx; i < size; ++i) { - SExprInfo* pExpr1 = tscExprGet(pQueryInfo, i); - - int16_t functionId = pExpr1->base.functionId; - if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); - pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE ? ++aggUdf : ++scalarUdf; - - continue; - } - - if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY) { - ++prjNum; - - continue; - } - - if (functionId == TSDB_FUNC_PRJ) { - ++prjNum; - } - - if (functionId == TSDB_FUNC_PRJ && (pExpr1->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX || TSDB_COL_IS_UD_COL(pExpr1->base.colInfo.flag))) { - continue; - } - - if (factor == INT32_MAX) { - factor = functionCompatList[functionId]; - } else { - if (functionCompatList[functionId] != factor) { - return false; - } else { - if (factor == -1) { // two functions with the same -1 flag - return false; - } - } - } - - if (functionId == TSDB_FUNC_LAST_ROW && (joinQuery || twQuery || !groupbyTagsOrNull(pQueryInfo))) { - return false; - } - } - - aggNum = (int32_t)size - prjNum - aggUdf - scalarUdf; - - assert(aggNum >= 0); - - if (aggUdf > 0 && (prjNum > 0 || aggNum > 0 || scalarUdf > 0)) { - return false; - } - - if (scalarUdf > 0 && aggNum > 0) { - return false; - } - - return true; -} - -int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd) { - const char* msg1 = "too many columns in group by clause"; - const char* msg2 = "invalid column name in group by clause"; - const char* msg3 = "columns from one table allowed as group by columns"; - const char* msg4 = "join query does not support group by"; - const char* msg5 = "not allowed column type for group by"; - const char* msg6 = "tags not allowed for table query"; - const char* msg7 = "not support group by expression"; - const char* msg8 = "normal column can only locate at the end of group by clause"; - - // todo : handle two tables situation - STableMetaInfo* pTableMetaInfo = NULL; - if (pList == NULL) { - return TSDB_CODE_SUCCESS; - } - - if (pQueryInfo->numOfTables > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - SGroupbyExpr* pGroupExpr = &pQueryInfo->groupbyExpr; - if (pGroupExpr->columnInfo == NULL) { - pGroupExpr->columnInfo = taosArrayInit(4, sizeof(SColIndex)); - } - - if (pQueryInfo->colList == NULL) { - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - } - - if (pGroupExpr->columnInfo == NULL || pQueryInfo->colList == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pGroupExpr->numOfGroupCols = (int16_t)taosArrayGetSize(pList); - if (pGroupExpr->numOfGroupCols > TSDB_MAX_TAGS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - SSchema *pSchema = NULL; - int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; - int32_t numOfGroupCols = 0; - - size_t num = taosArrayGetSize(pList); - for (int32_t i = 0; i < num; ++i) { - tVariantListItem * pItem = taosArrayGet(pList, i); - tVariant* pVar = &pItem->pVar; - - SStrToken token = {pVar->nLen, pVar->nType, pVar->pz}; - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&token, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { - tableIndex = index.tableIndex; - } else if (tableIndex != index.tableIndex) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - pSchema = tGetTbnameColumnSchema(); - } else { - pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); - } - - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - bool groupTag = (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= numOfCols); - - if (groupTag) { - if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - int32_t relIndex = index.columnIndex; - if (index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { - relIndex -= numOfCols; - } - - SColIndex colIndex = { .colIndex = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId, }; - strncpy(colIndex.name, pSchema->name, tListLen(colIndex.name)); - taosArrayPush(pGroupExpr->columnInfo, &colIndex); - - index.columnIndex = relIndex; - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pSchema); - } else { - // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by - if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - tscColumnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->id.uid, pSchema); - - SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId }; - strncpy(colIndex.name, pSchema->name, tListLen(colIndex.name)); - - taosArrayPush(pGroupExpr->columnInfo, &colIndex); - pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; - numOfGroupCols++; - } - } - - // 1. only one normal column allowed in the group by clause - // 2. the normal column in the group by clause can only located in the end position - if (numOfGroupCols > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - - for(int32_t i = 0; i < num; ++i) { - SColIndex* pIndex = taosArrayGet(pGroupExpr->columnInfo, i); - if (TSDB_COL_IS_NORMAL_COL(pIndex->flag) && i != num - 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); - } - } - - pQueryInfo->groupbyExpr.tableIndex = tableIndex; - return TSDB_CODE_SUCCESS; -} - - -static SColumnFilterInfo* addColumnFilterInfo(SColumnFilterList* filterList) { - int32_t size = (filterList->numOfFilters) + 1; - - char* tmp = (char*) realloc((void*)(filterList->filterInfo), sizeof(SColumnFilterInfo) * (size)); - if (tmp != NULL) { - filterList->filterInfo = (SColumnFilterInfo*)tmp; - } else { - return NULL; - } - - filterList->numOfFilters = size; - - SColumnFilterInfo* pColFilterInfo = &(filterList->filterInfo[size - 1]); - memset(pColFilterInfo, 0, sizeof(SColumnFilterInfo)); - - return pColFilterInfo; -} - -static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t timePrecision, SColumnFilterInfo* pColumnFilter, - int16_t colType, tSqlExpr* pExpr) { - const char* msg = "not supported filter condition"; - - tSqlExpr *pRight = pExpr->pRight; - - if (colType >= TSDB_DATA_TYPE_TINYINT && colType <= TSDB_DATA_TYPE_BIGINT) { - colType = TSDB_DATA_TYPE_BIGINT; - } else if (colType == TSDB_DATA_TYPE_FLOAT || colType == TSDB_DATA_TYPE_DOUBLE) { - colType = TSDB_DATA_TYPE_DOUBLE; - } else if ((colType == TSDB_DATA_TYPE_TIMESTAMP) && (TSDB_DATA_TYPE_BINARY == pRight->value.nType)) { - int retVal = setColumnFilterInfoForTimestamp(pCmd, pQueryInfo, &pRight->value); - if (TSDB_CODE_SUCCESS != retVal) { - return retVal; - } - } else if ((colType == TSDB_DATA_TYPE_TIMESTAMP) && (TSDB_DATA_TYPE_BIGINT == pRight->value.nType)) { - if (pRight->flags & (1 << EXPR_FLAG_NS_TIMESTAMP)) { - pRight->value.i64 = - convertTimePrecision(pRight->value.i64, TSDB_TIME_PRECISION_NANO, timePrecision); - pRight->flags &= ~(1 << EXPR_FLAG_NS_TIMESTAMP); - } - } - - int32_t retVal = TSDB_CODE_SUCCESS; - - int32_t bufLen = 0; - if (IS_NUMERIC_TYPE(pRight->value.nType)) { - bufLen = 60; - } else { - /* - * make memory sanitizer happy; - */ - if (pRight->value.nLen == 0) { - bufLen = pRight->value.nLen + 2; - } else { - bufLen = pRight->value.nLen + 1; - } - } - - if (pExpr->tokenId == TK_LE || pExpr->tokenId == TK_LT) { - retVal = tVariantDump(&pRight->value, (char*)&pColumnFilter->upperBndd, colType, false); - - // TK_GT,TK_GE,TK_EQ,TK_NE are based on the pColumn->lowerBndd - } else if (pExpr->tokenId == TK_IN) { - tVariant *pVal; - if (pRight->tokenId != TK_SET || !serializeExprListToVariant(pRight->Expr.paramList, &pVal, colType, timePrecision)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - pColumnFilter->pz = (int64_t)calloc(1, pVal->nLen); - pColumnFilter->len = pVal->nLen; - pColumnFilter->filterstr = 1; - memcpy((char *)(pColumnFilter->pz), (char *)(pVal->pz), pVal->nLen); - - tVariantDestroy(pVal); - free(pVal); - - } else if (colType == TSDB_DATA_TYPE_BINARY) { - pColumnFilter->pz = (int64_t)calloc(1, bufLen * TSDB_NCHAR_SIZE); - pColumnFilter->len = pRight->value.nLen; - retVal = tVariantDump(&pRight->value, (char*)pColumnFilter->pz, colType, false); - - } else if (colType == TSDB_DATA_TYPE_NCHAR) { - // bufLen + 1 is larger than the actual nchar string length - pColumnFilter->pz = (int64_t)calloc(1, (bufLen + 1) * TSDB_NCHAR_SIZE); - retVal = tVariantDump(&pRight->value, (char*)pColumnFilter->pz, colType, false); - size_t len = twcslen((wchar_t*)pColumnFilter->pz); - pColumnFilter->len = len * TSDB_NCHAR_SIZE; - - } else { - retVal = tVariantDump(&pRight->value, (char*)&pColumnFilter->lowerBndd, colType, false); - } - - if (retVal != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - switch (pExpr->tokenId) { - case TK_LE: - pColumnFilter->upperRelOptr = TSDB_RELATION_LESS_EQUAL; - break; - case TK_LT: - pColumnFilter->upperRelOptr = TSDB_RELATION_LESS; - break; - case TK_GT: - pColumnFilter->lowerRelOptr = TSDB_RELATION_GREATER; - break; - case TK_GE: - pColumnFilter->lowerRelOptr = TSDB_RELATION_GREATER_EQUAL; - break; - case TK_EQ: - pColumnFilter->lowerRelOptr = TSDB_RELATION_EQUAL; - break; - case TK_NE: - pColumnFilter->lowerRelOptr = TSDB_RELATION_NOT_EQUAL; - break; - case TK_LIKE: - pColumnFilter->lowerRelOptr = TSDB_RELATION_LIKE; - break; - case TK_MATCH: - pColumnFilter->lowerRelOptr = TSDB_RELATION_MATCH; - break; - case TK_NMATCH: - pColumnFilter->lowerRelOptr = TSDB_RELATION_NMATCH; - break; - case TK_ISNULL: - pColumnFilter->lowerRelOptr = TSDB_RELATION_ISNULL; - break; - case TK_NOTNULL: - pColumnFilter->lowerRelOptr = TSDB_RELATION_NOTNULL; - break; - case TK_IN: - pColumnFilter->lowerRelOptr = TSDB_RELATION_IN; - break; - default: - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - return TSDB_CODE_SUCCESS; -} - -typedef struct SCondExpr { - tSqlExpr* pTagCond; - tSqlExpr* pTimewindow; - - tSqlExpr* pColumnCond; - - tSqlExpr* pTableCond; - int16_t relType; // relation between table name in expression and other tag - // filter condition expression, TK_AND or TK_OR - int16_t tableCondIndex; - - tSqlExpr* pJoinExpr; // join condition - bool tsJoin; -} SCondExpr; - -static int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t timePrecision); - -static int32_t tablenameListToString(tSqlExpr* pExpr, SStringBuilder* sb) { - SArray* pList = pExpr->Expr.paramList; - - int32_t size = (int32_t) taosArrayGetSize(pList); - if (size <= 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (size > 0) { - taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); - } - - for (int32_t i = 0; i < size; ++i) { - tSqlExprItem* pSub = taosArrayGet(pList, i); - tVariant* pVar = &pSub->pNode->value; - - taosStringBuilderAppendStringLen(sb, pVar->pz, pVar->nLen); - - if (i < size - 1) { - taosStringBuilderAppendString(sb, TBNAME_LIST_SEP); - } - - if (pVar->nLen <= 0 || !tscValidateTableNameLength(pVar->nLen)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t tablenameCondToString(tSqlExpr* pExpr, uint32_t opToken, SStringBuilder* sb) { - assert(opToken == TK_LIKE || opToken == TK_MATCH || opToken == TK_NMATCH); - if (opToken == TK_LIKE) { - taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN); - taosStringBuilderAppendString(sb, pExpr->value.pz); - } else if (opToken == TK_MATCH) { - taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN); - taosStringBuilderAppendString(sb, pExpr->value.pz); - } else if (opToken == TK_NMATCH) { - taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_NMATCH, QUERY_COND_REL_PREFIX_NMATCH_LEN); - taosStringBuilderAppendString(sb, pExpr->value.pz); - } - return TSDB_CODE_SUCCESS; -} - -enum { - TSQL_EXPR_TS = 1, - TSQL_EXPR_TAG = 2, - TSQL_EXPR_COLUMN = 4, - TSQL_EXPR_TBNAME = 8, - TSQL_EXPR_JOIN = 16, -}; - -#define GET_MIXED_TYPE(t) (((t) >= TSQL_EXPR_JOIN) || ((t) > TSQL_EXPR_COLUMN && (t) < TSQL_EXPR_TBNAME) || ((t) == (TSQL_EXPR_TS|TSQL_EXPR_TAG))) - -static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SColumnIndex* pIndex, tSqlExpr* pExpr, int32_t sqlOptr) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); - int32_t ret = 0; - const char* msg1 = "non binary column not support like/match operator"; - const char* msg2 = "binary column not support this operator"; - const char* msg3 = "bool column not support this operator"; - const char* msg4 = "primary key not support this operator"; - - SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex->columnIndex, pTableMeta->id.uid, pSchema); - - pColumn->info.flist.numOfFilters++; - - /* - * in case of TK_AND filter condition, we first find the corresponding column and build the query condition together - * the already existed condition. - */ - if (sqlOptr != TK_AND && sqlOptr != TK_OR) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - SColumnFilterInfo* pColFilter = calloc(1, sizeof(SColumnFilterInfo)); - - pColFilter->filterstr = - ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0); - - if (pColFilter->filterstr) { - if (pExpr->tokenId != TK_EQ - && pExpr->tokenId != TK_NE - && pExpr->tokenId != TK_ISNULL - && pExpr->tokenId != TK_NOTNULL - && pExpr->tokenId != TK_LIKE - && pExpr->tokenId != TK_MATCH - && pExpr->tokenId != TK_NMATCH - && pExpr->tokenId != TK_IN) { - ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - goto _err_ret; - } - } else { - if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_NMATCH) { - ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - goto _err_ret; - } - - if (pSchema->type == TSDB_DATA_TYPE_BOOL) { - int32_t t = pExpr->tokenId; - if (t != TK_EQ && t != TK_NE && t != TK_NOTNULL && t != TK_ISNULL && t != TK_IN) { - ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - goto _err_ret; - } - } - } - - pColumn->columnIndex = pIndex->columnIndex; - pColumn->tableUid = pTableMeta->id.uid; - if (pColumn->columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && pExpr->tokenId == TK_IN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - ret = doExtractColumnFilterInfo(pCmd, pQueryInfo, tinfo.precision, pColFilter, pSchema->type, pExpr); - -_err_ret: - freeColumnFilterInfo(pColFilter, 1); - - return ret; -} - -static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pTableCond, SStringBuilder* sb) { - const char* msg0 = "invalid table name list"; - const char* msg1 = "not string following like"; - - if (pTableCond == NULL) { - return TSDB_CODE_SUCCESS; - } - - tSqlExpr* pLeft = pTableCond->pLeft; - tSqlExpr* pRight = pTableCond->pRight; - - if (!isTablenameToken(&pLeft->columnName)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int32_t ret = TSDB_CODE_SUCCESS; - - if (pTableCond->tokenId == TK_IN) { - ret = tablenameListToString(pRight, sb); - } else if (pTableCond->tokenId == TK_LIKE || pTableCond->tokenId == TK_MATCH || pTableCond->tokenId == TK_NMATCH) { - if (pRight->tokenId != TK_STRING) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - ret = tablenameCondToString(pRight, pTableCond->tokenId, sb); - } - - if (ret != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); - } - - return ret; -} - -static int32_t getColQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr) { - int32_t ret = TSDB_CODE_SUCCESS; - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - tSqlExpr* p1 = extractExprForSTable(pCmd, pExpr, pQueryInfo, i); - if (p1 == NULL) { // no query condition on this table - continue; - } - - tExprNode* p = NULL; - - SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL); - taosArrayDestroy(colList); - - SBufferWriter bw = tbufInitWriter(NULL, false); - - TRY(0) { - exprTreeToBinary(&bw, p); - } CATCH(code) { - tbufCloseWriter(&bw); - UNUSED(code); - // TODO: more error handling - } END_TRY - - // add to required table column list - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - int64_t uid = pTableMetaInfo->pTableMeta->id.uid; - - STblCond cond = { - .uid = uid, - .idx = i, - .len = (int32_t)(tbufTell(&bw)), - .cond = tbufGetData(&bw, true) - }; - - if (pQueryInfo->colCond == NULL) { - pQueryInfo->colCond = taosArrayInit(2, sizeof(SCond)); - } - - taosArrayPush(pQueryInfo->colCond, &cond); - - tSqlExprDestroy(p1); - tExprTreeDestroy(p, NULL); - - if (ret) { - break; - } - } - - return ret; -} - - -static int32_t checkColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr, int32_t relOptr) { - if (pExpr == NULL) { - pQueryInfo->onlyHasTagCond &= true; - return TSDB_CODE_SUCCESS; - } - pQueryInfo->onlyHasTagCond &= false; - - if (!tSqlExprIsParentOfLeaf(pExpr)) { // internal node - int32_t ret = checkColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->tokenId); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - return checkColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pRight, pExpr->tokenId); - } else { // handle leaf node - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return checkColumnFilterInfo(pCmd, pQueryInfo, &index, pExpr, relOptr); - } -} - -static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) { - int32_t code = 0; - const char* msg1 = "timestamp required for join tables"; - const char* msg2 = "only support one join tag for each table"; - const char* msg3 = "type of join columns must be identical"; - const char* msg4 = "invalid column name in join condition"; - - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - if (!tSqlExprIsParentOfLeaf(pExpr)) { - code = checkAndSetJoinCondInfo(pCmd, pQueryInfo, pExpr->pLeft); - if (code) { - return code; - } - - return checkAndSetJoinCondInfo(pCmd, pQueryInfo, pExpr->pRight); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pTagSchema1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - - assert(index.tableIndex >= 0 && index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); - - SJoinNode **leftNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; - if (*leftNode == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - (*leftNode)->uid = pTableMetaInfo->pTableMeta->id.uid; - (*leftNode)->tagColId = pTagSchema1->colId; - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - if (tscColumnExists(pTableMetaInfo->tagColList, pTagSchema1->colId, pTableMetaInfo->pTableMeta->id.uid) < 0) { - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pTagSchema1); - - if (taosArrayGetSize(pTableMetaInfo->tagColList) > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } - } - - int16_t leftIdx = index.tableIndex; - - index = (SColumnIndex)COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->pRight->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pTagSchema2 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - - assert(index.tableIndex >= 0 && index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); - - SJoinNode **rightNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; - if (*rightNode == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - (*rightNode)->uid = pTableMetaInfo->pTableMeta->id.uid; - (*rightNode)->tagColId = pTagSchema2->colId; - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMeta); - if (tscColumnExists(pTableMetaInfo->tagColList, pTagSchema2->colId, pTableMeta->id.uid) < 0) { - - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pTagSchema2); - if (taosArrayGetSize(pTableMetaInfo->tagColList) > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } - } - - int16_t rightIdx = index.tableIndex; - - if (pTagSchema1->type != pTagSchema2->type) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if ((*leftNode)->tagJoin == NULL) { - (*leftNode)->tagJoin = taosArrayInit(2, sizeof(int16_t)); - } - - if ((*rightNode)->tagJoin == NULL) { - (*rightNode)->tagJoin = taosArrayInit(2, sizeof(int16_t)); - } - - taosArrayPush((*leftNode)->tagJoin, &rightIdx); - taosArrayPush((*rightNode)->tagJoin, &leftIdx); - - pQueryInfo->tagCond.joinInfo.hasJoin = true; - - return TSDB_CODE_SUCCESS; - -} - -static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) { - if (pExpr == NULL) { - pQueryInfo->onlyHasTagCond &= true; - return TSDB_CODE_SUCCESS; - } - - return checkAndSetJoinCondInfo(pCmd, pQueryInfo, pExpr); -} - -static int32_t validateSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, - int32_t* type, uint64_t* uid) { - if (pExpr->type == SQL_NODE_TABLE_COLUMN) { - if (*type == NON_ARITHMEIC_EXPR) { - *type = NORMAL_ARITHMETIC; - } else if (*type == AGG_ARIGHTMEIC) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // if column is timestamp, bool, binary, nchar, not support arithmetic, so return invalid sql - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index.tableIndex)->pTableMeta; - SSchema* pSchema = tscGetTableSchema(pTableMeta) + index.columnIndex; - - if ((pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) || (pSchema->type == TSDB_DATA_TYPE_BOOL) || - (pSchema->type == TSDB_DATA_TYPE_BINARY) || (pSchema->type == TSDB_DATA_TYPE_NCHAR)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - pList->ids[pList->num++] = index; - } else if ((pExpr->tokenId == TK_FLOAT && (isnan(pExpr->value.dKey) || isinf(pExpr->value.dKey))) || - pExpr->tokenId == TK_NULL) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } else if (pExpr->type == SQL_NODE_SQLFUNCTION) { - if (*type == NON_ARITHMEIC_EXPR) { - *type = AGG_ARIGHTMEIC; - } else if (*type == NORMAL_ARITHMETIC) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int32_t outputIndex = (int32_t)tscNumOfExprs(pQueryInfo); - - tSqlExprItem item = {.pNode = pExpr, .aliasName = NULL}; - - // sql function list in selection clause. - // Append the sqlExpr into exprList of pQueryInfo structure sequentially - pExpr->functionId = isValidFunction(pExpr->Expr.operand.z, pExpr->Expr.operand.n); - if (pExpr->functionId < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, false, NULL) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // It is invalid in case of more than one sqlExpr, such as first(ts, k) - last(ts, k) - int32_t inc = (int32_t) tscNumOfExprs(pQueryInfo) - outputIndex; - if (inc > 1) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // Not supported data type in arithmetic expression - uint64_t id = -1; - for(int32_t i = 0; i < inc; ++i) { - SExprInfo* p1 = tscExprGet(pQueryInfo, i + outputIndex); - - int16_t t = p1->base.resType; - if (t == TSDB_DATA_TYPE_BINARY || t == TSDB_DATA_TYPE_NCHAR || t == TSDB_DATA_TYPE_BOOL || t == TSDB_DATA_TYPE_TIMESTAMP) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (i == 0) { - id = p1->base.uid; - continue; - } - - if (id != p1->base.uid) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - *uid = id; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) { - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - tSqlExpr* pLeft = pExpr->pLeft; - uint64_t uidLeft = 0; - uint64_t uidRight = 0; - - if (pLeft->type == SQL_NODE_EXPR) { - int32_t ret = validateArithmeticSQLExpr(pCmd, pLeft, pQueryInfo, pList, type); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - } else { - int32_t ret = validateSQLExpr(pCmd, pLeft, pQueryInfo, pList, type, &uidLeft); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - } - - tSqlExpr* pRight = pExpr->pRight; - if (pRight->type == SQL_NODE_EXPR) { - int32_t ret = validateArithmeticSQLExpr(pCmd, pRight, pQueryInfo, pList, type); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - } else { - int32_t ret = validateSQLExpr(pCmd, pRight, pQueryInfo, pList, type, &uidRight); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - // the expression not from the same table, return error - if (uidLeft != uidRight && uidLeft != 0 && uidRight != 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - return TSDB_CODE_SUCCESS; -} - -static bool isValidExpr(tSqlExpr* pLeft, tSqlExpr* pRight, int32_t optr) { - if (pLeft == NULL || (pRight == NULL && optr != TK_IN)) { - return false; - } - - /* - * filter illegal expression in where clause: - * 1. count(*) > 12 - * 2. sum(columnA) > sum(columnB) - * 3. 4 < 5, 'ABC'>'abc' - * - * However, columnA < 4+12 is valid - */ - if (pLeft->type == SQL_NODE_SQLFUNCTION) { - return false; - } - - if (pRight == NULL) { - return true; - } - - if (pLeft->tokenId >= TK_BOOL && pLeft->tokenId <= TK_BINARY && pRight->tokenId >= TK_BOOL && pRight->tokenId <= TK_BINARY) { - return false; - } - - if (pLeft->tokenId >= TK_BOOL && pLeft->tokenId <= TK_BINARY && (optr == TK_NOTNULL || optr == TK_ISNULL)) { - return false; - } - - return true; -} - -static void exchangeExpr(tSqlExpr* pExpr) { - tSqlExpr* pLeft = pExpr->pLeft; - tSqlExpr* pRight = pExpr->pRight; - - if (pRight->tokenId == TK_ID && (pLeft->tokenId == TK_INTEGER || pLeft->tokenId == TK_FLOAT || - pLeft->tokenId == TK_STRING || pLeft->tokenId == TK_BOOL)) { - /* - * exchange value of the left handside and the value of the right-handside - * to make sure that the value of filter expression always locates in - * right-handside and - * the column-id is at the left handside. - */ - uint32_t optr = 0; - switch (pExpr->tokenId) { - case TK_LE: - optr = TK_GE; - break; - case TK_LT: - optr = TK_GT; - break; - case TK_GT: - optr = TK_LT; - break; - case TK_GE: - optr = TK_LE; - break; - default: - optr = pExpr->tokenId; - } - - pExpr->tokenId = optr; - TSWAP(pExpr->pLeft, pExpr->pRight, void*); - } -} - -static bool validateJoinExprNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr, SColumnIndex* pLeftIndex) { - const char* msg1 = "illegal column name"; - const char* msg2 = "= is expected in join expression"; - const char* msg3 = "join column must have same type"; - const char* msg4 = "self join is not allowed"; - const char* msg5 = "join table must be the same type(table to table, super table to super table)"; - - tSqlExpr* pRight = pExpr->pRight; - - if (pRight->tokenId != TK_ID) { - return true; - } - - if (pExpr->tokenId != TK_EQ) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - return false; - } - - SColumnIndex rightIndex = COLUMN_INDEX_INITIALIZER; - - if (getColumnIndexByName(&pRight->columnName, pQueryInfo, &rightIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - return false; - } - - // todo extract function - STableMetaInfo* pLeftMeterMeta = tscGetMetaInfo(pQueryInfo, pLeftIndex->tableIndex); - SSchema* pLeftSchema = tscGetTableSchema(pLeftMeterMeta->pTableMeta); - int16_t leftType = pLeftSchema[pLeftIndex->columnIndex].type; - - tscColumnListInsert(pQueryInfo->colList, pLeftIndex->columnIndex, pLeftMeterMeta->pTableMeta->id.uid, &pLeftSchema[pLeftIndex->columnIndex]); - - STableMetaInfo* pRightMeterMeta = tscGetMetaInfo(pQueryInfo, rightIndex.tableIndex); - SSchema* pRightSchema = tscGetTableSchema(pRightMeterMeta->pTableMeta); - int16_t rightType = pRightSchema[rightIndex.columnIndex].type; - - tscColumnListInsert(pQueryInfo->colList, rightIndex.columnIndex, pRightMeterMeta->pTableMeta->id.uid, &pRightSchema[rightIndex.columnIndex]); - - if (leftType != rightType) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; - } else if (pLeftIndex->tableIndex == rightIndex.tableIndex) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - return false; - } - - // table to table/ super table to super table are allowed - if (UTIL_TABLE_IS_SUPER_TABLE(pLeftMeterMeta) != UTIL_TABLE_IS_SUPER_TABLE(pRightMeterMeta)) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; - } - - return true; -} - -static bool validTableNameOptr(tSqlExpr* pExpr) { - const char nameFilterOptr[] = {TK_IN, TK_LIKE, TK_MATCH, TK_NMATCH}; - - for (int32_t i = 0; i < tListLen(nameFilterOptr); ++i) { - if (pExpr->tokenId == nameFilterOptr[i]) { - return true; - } - } - - return false; -} - -static int32_t setExprToCond(tSqlExpr** parent, tSqlExpr* pExpr, const char* msg, int32_t parentOptr, char* msgBuf) { - if (*parent != NULL) { - if (parentOptr == TK_OR && msg != NULL) { - return invalidOperationMsg(msgBuf, msg); - } - - *parent = tSqlExprCreate((*parent), pExpr, parentOptr); - } else { - *parent = pExpr; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t setNormalExprToCond(tSqlExpr** parent, tSqlExpr* pExpr, int32_t parentOptr) { - if (*parent != NULL) { - *parent = tSqlExprCreate((*parent), pExpr, parentOptr); - } else { - *parent = pExpr; - } - - return TSDB_CODE_SUCCESS; -} - - -static int32_t validateNullExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) { - const char* msg = "only support is [not] null"; - - tSqlExpr* pRight = pExpr->pRight; - if (pRight->tokenId == TK_NULL && (!(pExpr->tokenId == TK_ISNULL || pExpr->tokenId == TK_NOTNULL))) { - return invalidOperationMsg(msgBuf, msg); - } - - if (pRight->tokenId == TK_STRING) { - SSchema* pSchema = tscGetTableSchema(pTableMeta); - if (IS_VAR_DATA_TYPE(pSchema[index].type)) { - return TSDB_CODE_SUCCESS; - } - - char *v = strndup(pRight->exprToken.z, pRight->exprToken.n); - int32_t len = strRmquote(v, pRight->exprToken.n); - if (len > 0) { - uint32_t type = 0; - tGetToken(v, &type); - - if (type == TK_NULL) { - free(v); - return invalidOperationMsg(msgBuf, msg); - } - } - - free(v); - } - - return TSDB_CODE_SUCCESS; -} - -// check for like expression -static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) { - const char* msg1 = "wildcard string should be less than %d characters"; - const char* msg2 = "illegal column type for like"; - - tSqlExpr* pLeft = pExpr->pLeft; - tSqlExpr* pRight = pExpr->pRight; - - if (pExpr->tokenId == TK_LIKE) { - if (pRight->value.nLen > tsMaxWildCardsLen) { - char tmp[64] = {0}; - sprintf(tmp, msg1, tsMaxWildCardsLen); - return invalidOperationMsg(msgBuf, tmp); - } - - SSchema* pSchema = tscGetTableSchema(pTableMeta); - if ((!isTablenameToken(&pLeft->columnName)) && !IS_VAR_DATA_TYPE(pSchema[index].type)) { - return invalidOperationMsg(msgBuf, msg2); - } - } - - return TSDB_CODE_SUCCESS; -} - -// check for match expression -static int32_t validateMatchExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) { - const char* msg1 = "regular expression string should be less than %d characters"; - const char* msg2 = "illegal column type for match/nmatch"; - const char* msg3 = "invalid regular expression"; - - tSqlExpr* pLeft = pExpr->pLeft; - tSqlExpr* pRight = pExpr->pRight; - - if (pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_NMATCH) { - if (pRight->value.nLen > tsMaxRegexStringLen) { - char tmp[64] = {0}; - sprintf(tmp, msg1, tsMaxRegexStringLen); - return invalidOperationMsg(msgBuf, tmp); - } - - SSchema* pSchema = tscGetTableSchema(pTableMeta); - if ((!isTablenameToken(&pLeft->columnName)) &&(pSchema[index].type != TSDB_DATA_TYPE_BINARY)) { - return invalidOperationMsg(msgBuf, msg2); - } - - if (!(pRight->type == SQL_NODE_VALUE && pRight->value.nType == TSDB_DATA_TYPE_BINARY)) { - return invalidOperationMsg(msgBuf, msg3); - } - - int errCode = 0; - regex_t regex; - char regErrBuf[256] = {0}; - - const char* pattern = pRight->value.pz; - int cflags = REG_EXTENDED; - if ((errCode = regcomp(®ex, pattern, cflags)) != 0) { - regerror(errCode, ®ex, regErrBuf, sizeof(regErrBuf)); - tscError("Failed to compile regex pattern %s. reason %s", pattern, regErrBuf); - return invalidOperationMsg(msgBuf, msg3); - } - regfree(®ex); - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t handleNeOptr(tSqlExpr** rexpr, tSqlExpr* expr) { - tSqlExpr* left = tSqlExprClone(expr); - tSqlExpr* right = expr; - - left->tokenId = TK_LT; - right->tokenId = TK_GT; - - *rexpr = tSqlExprCreate(left, right, TK_OR); - - return TSDB_CODE_SUCCESS; -} - - -static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr, - int32_t* type, int32_t* tbIdx, int32_t parentOptr, tSqlExpr** columnExpr, tSqlExpr** tsExpr) { - const char* msg1 = "table query cannot use tags filter"; - const char* msg2 = "illegal column name"; - const char* msg4 = "too many join tables"; - const char* msg5 = "not support ordinary column join"; - const char* msg6 = "only one query condition on tbname allowed"; - const char* msg7 = "only in/like allowed in filter table name"; - - tSqlExpr* pLeft = (*pExpr)->pLeft; - tSqlExpr* pRight = (*pExpr)->pRight; - - int32_t ret = TSDB_CODE_SUCCESS; - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - *tbIdx = index.tableIndex; - - assert(tSqlExprIsParentOfLeaf(*pExpr)); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - // validate the null expression - int32_t code = validateNullExpr(*pExpr, pTableMeta, index.columnIndex, tscGetErrorMsgPayload(pCmd)); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // validate the like expression - code = validateLikeExpr(*pExpr, pTableMeta, index.columnIndex, tscGetErrorMsgPayload(pCmd)); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // validate the match expression - code = validateMatchExpr(*pExpr, pTableMeta, index.columnIndex, tscGetErrorMsgPayload(pCmd)); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); - if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP && index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range - if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // set join query condition - if (pRight->tokenId == TK_ID) { // no need to keep the timestamp join condition - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY); - pCondExpr->tsJoin = true; - - assert(index.tableIndex >= 0 && index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); - SJoinNode **leftNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; - if (*leftNode == NULL) { - *leftNode = calloc(1, sizeof(SJoinNode)); - if (*leftNode == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - int16_t leftIdx = index.tableIndex; - - if (getColumnIndexByName(&pRight->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - if (index.tableIndex < 0 || index.tableIndex >= TSDB_MAX_JOIN_TABLE_NUM) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - SJoinNode **rightNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; - if (*rightNode == NULL) { - *rightNode = calloc(1, sizeof(SJoinNode)); - if (*rightNode == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - int16_t rightIdx = index.tableIndex; - - if ((*leftNode)->tsJoin == NULL) { - (*leftNode)->tsJoin = taosArrayInit(2, sizeof(int16_t)); - } - - if ((*rightNode)->tsJoin == NULL) { - (*rightNode)->tsJoin = taosArrayInit(2, sizeof(int16_t)); - } - - taosArrayPush((*leftNode)->tsJoin, &rightIdx); - taosArrayPush((*rightNode)->tsJoin, &leftIdx); - - /* - * To release expression, e.g., m1.ts = m2.ts, - * since this expression is used to set the join query type - */ - tSqlExprDestroy(*pExpr); - if (type) { - *type |= TSQL_EXPR_JOIN; - } - } else { - tSqlExpr *rexpr = NULL; - if ((*pExpr)->tokenId == TK_NE) { - handleNeOptr(&rexpr, *pExpr); - } else { - rexpr = *pExpr; - } - - ret = setNormalExprToCond(tsExpr, rexpr, parentOptr); - if (type) { - *type |= TSQL_EXPR_TS; - } - } - - *pExpr = NULL; // remove this expression - } else if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) || index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - // query on tags, check for tag query condition - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - // in case of in operator, keep it in a seprate attribute - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - if (!validTableNameOptr(*pExpr)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - - if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pCondExpr->pTableCond == NULL) { - pCondExpr->pTableCond = *pExpr; - pCondExpr->relType = parentOptr; - pCondExpr->tableCondIndex = index.tableIndex; - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - if (type) { - *type |= TSQL_EXPR_TAG; - } - *pExpr = NULL; - } else { - if (pRight != NULL && pRight->tokenId == TK_ID) { // join on tag columns for stable query - if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - pQueryInfo->type |= TSDB_QUERY_TYPE_JOIN_QUERY; - ret = setExprToCond(&pCondExpr->pJoinExpr, *pExpr, NULL, parentOptr, pCmd->payload); - *pExpr = NULL; - if (type) { - *type |= TSQL_EXPR_JOIN; - } - } else { - // do nothing - // ret = setExprToCond(pCmd, &pCondExpr->pTagCond, - // *pExpr, NULL, parentOptr); - tSqlExpr *rexpr = NULL; - if ((*pExpr)->tokenId == TK_NE && (pSchema->type != TSDB_DATA_TYPE_BINARY && pSchema->type != TSDB_DATA_TYPE_NCHAR && pSchema->type != TSDB_DATA_TYPE_BOOL)) { - handleNeOptr(&rexpr, *pExpr); - *pExpr = rexpr; - } - - if (type) { - *type |= TSQL_EXPR_TAG; - } - } - } - } else { // query on other columns - if (type) { - *type |= TSQL_EXPR_COLUMN; - } - - if (pRight->tokenId == TK_ID) { // other column cannot be served as the join column - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - tSqlExpr *rexpr = NULL; - if ((*pExpr)->tokenId == TK_NE && (pSchema->type != TSDB_DATA_TYPE_BINARY && pSchema->type != TSDB_DATA_TYPE_NCHAR && pSchema->type != TSDB_DATA_TYPE_BOOL)) { - handleNeOptr(&rexpr, *pExpr); - } else { - rexpr = *pExpr; - } - - ret = setNormalExprToCond(columnExpr, rexpr, parentOptr); - *pExpr = NULL; // remove it from expr tree - } - - return ret; -} - -int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr, - int32_t* type, int32_t* tbIdx, int32_t parentOptr, tSqlExpr** columnExpr, tSqlExpr** tsExpr) { - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - tSqlExpr *columnLeft = NULL; - tSqlExpr *columnRight = NULL; - tSqlExpr *tsLeft = NULL; - tSqlExpr *tsRight = NULL; - - int32_t ret = 0; - - const char* msg1 = "query condition between columns/tags/timestamp/join fields must use 'AND'"; - const char* msg2 = "query condition between tables must use 'AND'"; - - if ((*pExpr)->flags & (1 << EXPR_FLAG_TS_ERROR)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - tSqlExpr* pLeft = (*pExpr)->pLeft; - tSqlExpr* pRight = (*pExpr)->pRight; - - if (!isValidExpr(pLeft, pRight, (*pExpr)->tokenId)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int32_t leftType = 0; - int32_t rightType = 0; - int32_t leftTbIdx = 0; - int32_t rightTbIdx = 0; - - if (!tSqlExprIsParentOfLeaf(*pExpr)) { - ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, type ? &leftType : NULL, &leftTbIdx, (*pExpr)->tokenId, &columnLeft, &tsLeft); - if (ret != TSDB_CODE_SUCCESS) { - goto err_ret; - } - - ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pRight, pCondExpr, type ? &rightType : NULL, &rightTbIdx, (*pExpr)->tokenId, &columnRight, &tsRight); - if (ret != TSDB_CODE_SUCCESS) { - goto err_ret; - } - - /* - * if left child and right child do not belong to the same group, the sub - * expression is not valid for parent node, it must be TK_AND operator. - */ - if (type != NULL && ((leftType != rightType) || GET_MIXED_TYPE(leftType)) && ((*pExpr)->tokenId == TK_OR)) { - ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - goto err_ret; - } - - if (((leftTbIdx != rightTbIdx) || (leftTbIdx == -1 || rightTbIdx == -1)) && ((*pExpr)->tokenId == TK_OR)) { - ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - goto err_ret; - } - - if (columnLeft && columnRight) { - setNormalExprToCond(&columnLeft, columnRight, (*pExpr)->tokenId); - - *columnExpr = columnLeft; - } else { - *columnExpr = columnLeft ? columnLeft : columnRight; - } - - if (tsLeft && tsRight) { - setNormalExprToCond(&tsLeft, tsRight, (*pExpr)->tokenId); - - *tsExpr = tsLeft; - } else { - *tsExpr = tsLeft ? tsLeft : tsRight; - } - - if (type) { - *type = leftType|rightType; - } - *tbIdx = (leftTbIdx == rightTbIdx) ? leftTbIdx : -1; - - return TSDB_CODE_SUCCESS; - } - - exchangeExpr(*pExpr); - - if (pLeft->tokenId == TK_ID && pRight->tokenId == TK_TIMESTAMP && (pRight->flags & (1 << EXPR_FLAG_TIMESTAMP_VAR))) { - ret = TSDB_CODE_TSC_INVALID_OPERATION; - goto err_ret; - } - - if ((pLeft->flags & (1 << EXPR_FLAG_TS_ERROR)) || (pRight->flags & (1 << EXPR_FLAG_TS_ERROR))) { - ret = TSDB_CODE_TSC_INVALID_OPERATION; - goto err_ret; - } - - ret = handleExprInQueryCond(pCmd, pQueryInfo, pExpr, pCondExpr, type, tbIdx, parentOptr, columnExpr, tsExpr); - if (ret) { - goto err_ret; - } - - return TSDB_CODE_SUCCESS; - -err_ret: - - tSqlExprDestroy(columnLeft); - tSqlExprDestroy(columnRight); - tSqlExprDestroy(tsLeft); - tSqlExprDestroy(tsRight); - return ret; -} - -static void doExtractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, tSqlExpr** pOut, int32_t tableIndex) { - if (*pExpr == NULL) { - *pOut = NULL; - return; - } - - if (tSqlExprIsParentOfLeaf(*pExpr)) { - tSqlExpr* pLeft = (*pExpr)->pLeft; - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return; - } - - if (index.tableIndex != tableIndex) { - return; - } - - *pOut = *pExpr; - (*pExpr) = NULL; - - } else { - *pOut = tSqlExprCreate(NULL, NULL, (*pExpr)->tokenId); - - doExtractExprForSTable(pCmd, &(*pExpr)->pLeft, pQueryInfo, &((*pOut)->pLeft), tableIndex); - doExtractExprForSTable(pCmd, &(*pExpr)->pRight, pQueryInfo, &((*pOut)->pRight), tableIndex); - } -} - -static tSqlExpr* extractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, int32_t tableIndex) { - tSqlExpr* pResExpr = NULL; - - if (*pExpr != NULL) { - doExtractExprForSTable(pCmd, pExpr, pQueryInfo, &pResExpr, tableIndex); - tSqlExprCompact(&pResExpr); - } - - return pResExpr; -} - -int tableNameCompar(const void* lhs, const void* rhs) { - char* left = *(char**)lhs; - char* right = *(char**)rhs; - - int32_t ret = strcmp(left, right); - - if (ret == 0) { - return 0; - } - - return ret > 0 ? 1 : -1; -} - -static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, const char* account, - tSqlExpr* pExpr, int16_t tableCondIndex, SStringBuilder* sb) { - const char* msg = "table name too long"; - - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableCondIndex); - - STagCond* pTagCond = &pQueryInfo->tagCond; - pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->id.uid; - - assert(pExpr->tokenId == TK_LIKE - || pExpr->tokenId == TK_MATCH - || pExpr->tokenId == TK_NMATCH - || pExpr->tokenId == TK_IN); - - if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_NMATCH) { - char* str = taosStringBuilderGetResult(sb, NULL); - pQueryInfo->tagCond.tbnameCond.cond = strdup(str); - pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str); - return TSDB_CODE_SUCCESS; - } else { - SStringBuilder sb1; - memset(&sb1, 0, sizeof(sb1)); - taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); - - // remove the duplicated input table names - int32_t num = 0; - char* tableNameString = taosStringBuilderGetResult(sb, NULL); - - char** segments = strsplit(tableNameString + QUERY_COND_REL_PREFIX_IN_LEN, TBNAME_LIST_SEP, &num); - qsort(segments, num, POINTER_BYTES, tableNameCompar); - - int32_t j = 1; - for (int32_t i = 1; i < num; ++i) { - if (strcmp(segments[i], segments[i - 1]) != 0) { - segments[j++] = segments[i]; - } - } - num = j; - - char name[TSDB_DB_NAME_LEN] = {0}; - tNameGetDbName(&pTableMetaInfo->name, name); - SStrToken dbToken = {.type = TK_STRING, .z = name, .n = (uint32_t)strlen(name)}; - - for (int32_t i = 0; i < num; ++i) { - if (i >= 1) { - taosStringBuilderAppendStringLen(&sb1, TBNAME_LIST_SEP, 1); - } - - char idBuf[TSDB_TABLE_FNAME_LEN] = {0}; - int32_t xlen = (int32_t)strlen(segments[i]); - SStrToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; - - int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen); - if (ret != TSDB_CODE_SUCCESS) { - taosStringBuilderDestroy(&sb1); - tfree(segments); - - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - return ret; - } - - taosStringBuilderAppendString(&sb1, idBuf); - } - - char* str = taosStringBuilderGetResult(&sb1, NULL); - pQueryInfo->tagCond.tbnameCond.cond = strdup(str); - pQueryInfo->tagCond.tbnameCond.len = (int32_t)strlen(str); - - taosStringBuilderDestroy(&sb1); - tfree(segments); - return TSDB_CODE_SUCCESS; - } -} - -int32_t mergeTimeRange(SSqlCmd* pCmd, STimeWindow* res, STimeWindow* win, int32_t optr) { - const char* msg0 = "only one time stamp window allowed"; - -#define SET_EMPTY_RANGE(w) do { (w)->skey = INT64_MAX; (w)->ekey = INT64_MIN; } while (0) -#define IS_EMPTY_RANGE(w) ((w)->skey == INT64_MAX && (w)->ekey == INT64_MIN) - - if (optr == TSDB_RELATION_AND) { - if (res->skey > win->ekey || win->skey > res->ekey) { - SET_EMPTY_RANGE(res); - return TSDB_CODE_SUCCESS; - } - - if (res->skey < win->skey) { - res->skey = win->skey; - } - - if (res->ekey > win->ekey) { - res->ekey = win->ekey; - } - - return TSDB_CODE_SUCCESS; - } - - if (res->skey > win->ekey || win->skey > res->ekey) { - if (IS_EMPTY_RANGE(res)) { - res->skey = win->skey; - res->ekey = win->ekey; - return TSDB_CODE_SUCCESS; - } - - if (IS_EMPTY_RANGE(win)) { - return TSDB_CODE_SUCCESS; - } - - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); - } - - if (res->skey > win->skey) { - res->skey = win->skey; - } - - if (res->ekey < win->ekey) { - res->ekey = win->ekey; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t createTimeRangeExpr(tSqlExpr** pExpr, STimeWindow* win, uint32_t tokenId) { - *pExpr = calloc(1, sizeof(tSqlExpr)); - - (*pExpr)->type = SQL_NODE_VALUE; - (*pExpr)->tokenId = tokenId; - (*pExpr)->value.nType = TSDB_DATA_TYPE_VALUE_ARRAY; - (*pExpr)->value.nLen = 2; - (*pExpr)->value.arr = taosArrayInit(2, sizeof(int64_t)); - - taosArrayPush((*pExpr)->value.arr, &win->skey); - taosArrayPush((*pExpr)->value.arr, &win->ekey); - - return TSDB_CODE_SUCCESS; -} - -static int32_t convertTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) { - const char* msg0 = "invalid timestamp or operator for timestamp"; - int32_t code = 0; - STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; - - if (pExpr == NULL) { - pQueryInfo->onlyHasTagCond &= true; - return TSDB_CODE_SUCCESS; - } - pQueryInfo->onlyHasTagCond &= false; - - - if (!tSqlExprIsParentOfLeaf(pExpr)) { - code = convertTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pLeft); - if (code) { - return code; - } - - code = convertTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pRight); - if (code) { - return code; - } - } else { - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - tSqlExpr* pRight = pExpr->pRight; - - if (getTimeRange(&win, pRight, pExpr->tokenId, tinfo.precision) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); - } - - createTimeRangeExpr(&pExpr->pRight, &win, pRight->tokenId); - - tSqlExprDestroy(pRight); - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t validateJoinExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) { - const char* msg1 = "super table join requires tags column"; - const char* msg2 = "timestamp join condition missing"; - const char* msg3 = "condition missing for join query"; - - if (!QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { - if (pQueryInfo->numOfTables == 1) { - pQueryInfo->onlyHasTagCond &= true; - return TSDB_CODE_SUCCESS; - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - pQueryInfo->onlyHasTagCond &= false; - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // for stable join, tag columns - // must be present for join - if (pCondExpr->pJoinExpr == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - if (!pCondExpr->tsJoin) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - return TSDB_CODE_SUCCESS; -} - -static void cleanQueryExpr(SCondExpr* pCondExpr) { - if (pCondExpr->pTableCond) { - tSqlExprDestroy(pCondExpr->pTableCond); - } - - if (pCondExpr->pColumnCond) { - tSqlExprDestroy(pCondExpr->pColumnCond); - } - - if (pCondExpr->pTimewindow) { - tSqlExprDestroy(pCondExpr->pTimewindow); - } - - if (pCondExpr->pJoinExpr) { - tSqlExprDestroy(pCondExpr->pJoinExpr); - } -} - -/* -static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (QUERY_IS_JOIN_QUERY(pQueryInfo->type) && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - - if (getColumnIndexByName(pCmd, &pCondExpr->pJoinExpr->pLeft->ColName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - tscError("%p: invalid column name (left)", pQueryInfo); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - tscColumnListInsert(pTableMetaInfo->tagColList, &index, &pSchema[index.columnIndex]); - - if (getColumnIndexByName(pCmd, &pCondExpr->pJoinExpr->pRight->ColName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - tscError("%p: invalid column name (right)", pQueryInfo); - } - - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - tscColumnListInsert(pTableMetaInfo->tagColList, &index, &pSchema[index.columnIndex]); - } -} -*/ - -static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { - const char *msg1 = "invalid tag operator"; - const char* msg2 = "not supported filter condition"; - - do { - if (p->nodeType != TSQL_NODE_EXPR) { - break; - } - - if (!p->_node.pLeft || !p->_node.pRight) { - break; - } - - if (IS_ARITHMETIC_OPTR(p->_node.optr)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (!IS_RELATION_OPTR(p->_node.optr)) { - break; - } - - tVariant * vVariant = NULL; - int32_t schemaType = -1; - - if (p->_node.pLeft->nodeType == TSQL_NODE_VALUE && p->_node.pRight->nodeType == TSQL_NODE_COL) { - if (!p->_node.pRight->pSchema) { - break; - } - - vVariant = p->_node.pLeft->pVal; - schemaType = p->_node.pRight->pSchema->type; - } else if (p->_node.pLeft->nodeType == TSQL_NODE_COL && p->_node.pRight->nodeType == TSQL_NODE_VALUE) { - if (!p->_node.pLeft->pSchema) { - break; - } - - vVariant = p->_node.pRight->pVal; - schemaType = p->_node.pLeft->pSchema->type; - } else { - break; - } - - if (schemaType >= TSDB_DATA_TYPE_TINYINT && schemaType <= TSDB_DATA_TYPE_BIGINT) { - schemaType = TSDB_DATA_TYPE_BIGINT; - } else if (schemaType == TSDB_DATA_TYPE_FLOAT || schemaType == TSDB_DATA_TYPE_DOUBLE) { - schemaType = TSDB_DATA_TYPE_DOUBLE; - } - - int32_t retVal = TSDB_CODE_SUCCESS; - - int32_t bufLen = 0; - if (IS_NUMERIC_TYPE(vVariant->nType)) { - bufLen = 60; // The maximum length of string that a number is converted to. - } else { - bufLen = vVariant->nLen + 1; - } - - if (schemaType == TSDB_DATA_TYPE_BINARY) { - char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); - retVal = tVariantDump(vVariant, tmp, schemaType, false); - free(tmp); - } else if (schemaType == TSDB_DATA_TYPE_NCHAR) { - // pRight->value.nLen + 1 is larger than the actual nchar string length - char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); - retVal = tVariantDump(vVariant, tmp, schemaType, false); - free(tmp); - } else { - double tmp; - retVal = tVariantDump(vVariant, (char*)&tmp, schemaType, false); - } - - if (retVal != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } while (0); - - return TSDB_CODE_SUCCESS; -} - -static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) { - int32_t ret = TSDB_CODE_SUCCESS; - - if (pCondExpr->pTagCond == NULL) { - return ret; - } - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - tSqlExpr* p1 = extractExprForSTable(pCmd, &pCondExpr->pTagCond, pQueryInfo, i); - if (p1 == NULL) { // no query condition on this table - continue; - } - - tExprNode* p = NULL; - - SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL); - //if (ret == TSDB_CODE_SUCCESS) { - // ret = filterInitFromTree(p, &pQueryInfo->tagFilter, (int32_t)taosArrayGetSize(colList)); - //} - - SBufferWriter bw = tbufInitWriter(NULL, false); - - TRY(0) { - exprTreeToBinary(&bw, p); - } CATCH(code) { - tbufCloseWriter(&bw); - UNUSED(code); - // TODO: more error handling - } END_TRY - - // add to required table column list - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - int64_t uid = pTableMetaInfo->pTableMeta->id.uid; - int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - size_t num = taosArrayGetSize(colList); - for(int32_t j = 0; j < num; ++j) { - SColIndex* pIndex = taosArrayGet(colList, j); - SColumnIndex index = {.tableIndex = i, .columnIndex = pIndex->colIndex - numOfCols}; - - SSchema* s = tscGetTableSchema(pTableMetaInfo->pTableMeta); - tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMetaInfo->pTableMeta->id.uid, - &s[pIndex->colIndex]); - } - - tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &bw); - tSqlExprCompact(&pCondExpr->pTagCond); - - if (ret == TSDB_CODE_SUCCESS) { - ret = validateTagCondExpr(pCmd, p); - } - - tSqlExprDestroy(p1); - tExprTreeDestroy(p, NULL); //TODO - - taosArrayDestroy(colList); - if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table"); - } - - if (ret) { - break; - } - } - - return ret; -} - -int32_t validateJoinNodes(SQueryInfo* pQueryInfo, SSqlObj* pSql) { - const char* msg1 = "timestamp required for join tables"; - const char* msg2 = "tag required for join stables"; - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - SJoinNode *node = pQueryInfo->tagCond.joinInfo.joinTables[i]; - - if (node == NULL || node->tsJoin == NULL || taosArrayGetSize(node->tsJoin) <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg1); - } - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - SJoinNode *node = pQueryInfo->tagCond.joinInfo.joinTables[i]; - - if (node == NULL || node->tagJoin == NULL || taosArrayGetSize(node->tagJoin) <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); - } - } - } - - return TSDB_CODE_SUCCESS; -} - - -void mergeJoinNodesImpl(int8_t* r, int8_t* p, int16_t* tidx, SJoinNode** nodes, int32_t type) { - SJoinNode *node = nodes[*tidx]; - SArray* arr = (type == 0) ? node->tsJoin : node->tagJoin; - size_t size = taosArrayGetSize(arr); - - p[*tidx] = 1; - - for (int32_t j = 0; j < size; j++) { - int16_t* idx = taosArrayGet(arr, j); - r[*idx] = 1; - if (p[*idx] == 0) { - mergeJoinNodesImpl(r, p, idx, nodes, type); - } - } -} - -int32_t mergeJoinNodes(SQueryInfo* pQueryInfo, SSqlObj* pSql) { - const char* msg1 = "not all join tables have same timestamp"; - const char* msg2 = "not all join tables have same tag"; - - int8_t r[TSDB_MAX_JOIN_TABLE_NUM] = {0}; - int8_t p[TSDB_MAX_JOIN_TABLE_NUM] = {0}; - - for (int16_t i = 0; i < pQueryInfo->numOfTables; ++i) { - mergeJoinNodesImpl(r, p, &i, pQueryInfo->tagCond.joinInfo.joinTables, 0); - - taosArrayClear(pQueryInfo->tagCond.joinInfo.joinTables[i]->tsJoin); - - for (int32_t j = 0; j < TSDB_MAX_JOIN_TABLE_NUM; ++j) { - if (r[j]) { - taosArrayPush(pQueryInfo->tagCond.joinInfo.joinTables[i]->tsJoin, &j); - } - } - - memset(r, 0, sizeof(r)); - memset(p, 0, sizeof(p)); - } - - if (taosArrayGetSize(pQueryInfo->tagCond.joinInfo.joinTables[0]->tsJoin) != pQueryInfo->numOfTables) { - return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg1); - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - for (int16_t i = 0; i < pQueryInfo->numOfTables; ++i) { - mergeJoinNodesImpl(r, p, &i, pQueryInfo->tagCond.joinInfo.joinTables, 1); - - taosArrayClear(pQueryInfo->tagCond.joinInfo.joinTables[i]->tagJoin); - - for (int32_t j = 0; j < TSDB_MAX_JOIN_TABLE_NUM; ++j) { - if (r[j]) { - taosArrayPush(pQueryInfo->tagCond.joinInfo.joinTables[i]->tagJoin, &j); - } - } - - memset(r, 0, sizeof(r)); - memset(p, 0, sizeof(p)); - } - - if (taosArrayGetSize(pQueryInfo->tagCond.joinInfo.joinTables[0]->tagJoin) != pQueryInfo->numOfTables) { - return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); - } - - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t getQueryTimeRange(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr) { - int32_t ret = TSDB_CODE_SUCCESS; - - if (*pExpr == NULL) { - return ret; - } - - //multiple tables's query time range mixed together - - tExprNode* p = NULL; - SFilterInfo *filter = NULL; - - SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - ret = exprTreeFromSqlExpr(pCmd, &p, *pExpr, pQueryInfo, colList, NULL); - taosArrayDestroy(colList); - - if (ret != TSDB_CODE_SUCCESS) { - goto _ret; - } - - ret = filterInitFromTree(p, &filter, FI_OPTION_NO_REWRITE|FI_OPTION_TIMESTAMP); - if (ret != TSDB_CODE_SUCCESS) { - goto _ret; - } - - ret = filterGetTimeRange(filter, &pQueryInfo->window); - - filterFreeInfo(filter); - -_ret: - tExprTreeDestroy(p, NULL); - - if (ret) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), tstrerror(ret)); - } - - return ret; -} - - - -int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql) { - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - const char* msg1 = "invalid expression"; -// const char* msg2 = "invalid filter expression"; - - int32_t ret = TSDB_CODE_SUCCESS; - - // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space - SStringBuilder sb; memset(&sb, 0, sizeof(sb)); - SCondExpr condExpr = {0}; - - if ((*pExpr)->pLeft == NULL || (*pExpr)->pRight == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg1); - } - - int32_t type = 0; - int32_t tbIdx = 0; - int32_t *etype = &type; - -#if 0 - //DISABLE PARENT CONDITION GROUP TYPE CHECK - if (taosArrayGetSize(pQueryInfo->pUpstream) > 0) { - etype = NULL; - } -#endif - - if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, etype, &tbIdx, (*pExpr)->tokenId, &condExpr.pColumnCond, &condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - if (taosArrayGetSize(pQueryInfo->pUpstream) > 0 && condExpr.pTimewindow != NULL) { - setNormalExprToCond(&condExpr.pColumnCond, condExpr.pTimewindow, TK_AND); - condExpr.pTimewindow = NULL; - } - - tSqlExprCompact(pExpr); - - // after expression compact, the expression tree is only include tag query condition - condExpr.pTagCond = (*pExpr); - *pExpr = NULL; - - // 1. check if it is a join query - if ((ret = validateJoinExpr(&pSql->cmd, pQueryInfo, &condExpr)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - // 2. get the query time range - if ((ret = convertTimeRangeFromExpr(&pSql->cmd, pQueryInfo, condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - if ((ret = getQueryTimeRange(&pSql->cmd, pQueryInfo, &condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - - // 3. get the tag query condition - if ((ret = getTagQueryCondExpr(&pSql->cmd, pQueryInfo, &condExpr)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - // 4. get the table name query condition - if ((ret = getTablenameCond(&pSql->cmd, pQueryInfo, condExpr.pTableCond, &sb)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - // 5. other column query condition - if ((ret = checkColumnQueryCondInfo(&pSql->cmd, pQueryInfo, condExpr.pColumnCond, TK_AND)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - if ((ret = getColQueryCondExpr(&pSql->cmd, pQueryInfo, &condExpr.pColumnCond)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - - // 6. join condition - if ((ret = getJoinCondInfo(&pSql->cmd, pQueryInfo, condExpr.pJoinExpr)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } - - // 7. query condition for table name - pQueryInfo->tagCond.relType = (condExpr.relType == TK_AND) ? TSDB_RELATION_AND : TSDB_RELATION_OR; - - ret = setTableCondForSTableQuery(&pSql->cmd, pQueryInfo, getAccountId(pSql), condExpr.pTableCond, condExpr.tableCondIndex, &sb); - taosStringBuilderDestroy(&sb); - if (ret) { - goto PARSE_WHERE_EXIT; - } - - //if (!validateFilterExpr(pQueryInfo)) { - // ret = invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); - // goto PARSE_WHERE_EXIT; - //} - - //doAddJoinTagsColumnsIntoTagList(&pSql->cmd, pQueryInfo, &condExpr); - if (condExpr.tsJoin) { - ret = validateJoinNodes(pQueryInfo, pSql); - if (ret) { - goto PARSE_WHERE_EXIT; - } - - ret = mergeJoinNodes(pQueryInfo, pSql); - if (ret) { - goto PARSE_WHERE_EXIT; - } - } - -PARSE_WHERE_EXIT: - - cleanQueryExpr(&condExpr); - return ret; -} - -int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t timePrecision) { - // this is join condition, do nothing - if (pRight->tokenId == TK_ID) { - return TSDB_CODE_SUCCESS; - } - - /* - * filter primary ts filter expression like: - * where ts in ('2015-12-12 4:8:12') - */ - if (pRight->tokenId == TK_SET || optr == TK_IN || optr == TK_NE) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int64_t val = 0; - bool parsed = false; - if (pRight->value.nType == TSDB_DATA_TYPE_BINARY) { - pRight->value.nLen = strdequote(pRight->value.pz); - - char* seg = strnchr(pRight->value.pz, '-', pRight->value.nLen, false); - if (seg != NULL) { - if (taosParseTime(pRight->value.pz, &val, pRight->value.nLen, timePrecision, tsDaylight) == TSDB_CODE_SUCCESS) { - parsed = true; - } else { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else { - SStrToken token = {.z = pRight->value.pz, .n = pRight->value.nLen, .type = TK_ID}; - int32_t len = tGetToken(pRight->value.pz, &token.type); - - if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->value.nLen) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - } - - if (!parsed) { - /* - * failed to parse timestamp in regular formation, try next - * it may be a epoch time in string format - */ - if (pRight->flags & (1 << EXPR_FLAG_NS_TIMESTAMP)) { - pRight->value.i64 = convertTimePrecision(pRight->value.i64, TSDB_TIME_PRECISION_NANO, timePrecision); - pRight->flags &= ~(1 << EXPR_FLAG_NS_TIMESTAMP); - } - - tVariantDump(&pRight->value, (char*)&val, TSDB_DATA_TYPE_BIGINT, true); - } - - if (optr == TK_LE) { - win->ekey = val; - } else if (optr == TK_LT) { - win->ekey = val - 1; - } else if (optr == TK_GT) { - win->skey = val + 1; - } else if (optr == TK_GE) { - win->skey = val; - } else if (optr == TK_EQ) { - win->ekey = win->skey = val; - } else if (optr == TK_NE) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} - -// todo error !!!! -int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char rep[] = {'(', ')', '*', ',', '.', '/', '\\', '+', '-', '%', ' '}; - - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name; - for (int32_t j = 0; j < (TSDB_COL_NAME_LEN - 1) && fieldName[j] != 0; ++j) { - for (int32_t k = 0; k < tListLen(rep); ++k) { - if (fieldName[j] == rep[k]) { - fieldName[j] = '_'; - break; - } - } - } - - fieldName[TSDB_COL_NAME_LEN - 1] = 0; - } - - // the column name may be identical, here check again - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name; - for (int32_t j = i + 1; j < pQueryInfo->fieldsInfo.numOfOutput; ++j) { - if (strncasecmp(fieldName, tscFieldInfoGetField(&pQueryInfo->fieldsInfo, j)->name, (TSDB_COL_NAME_LEN - 1)) == 0) { - const char* msg = "duplicated column name in new table"; - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t validateFillNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode) { - SArray* pFillToken = pSqlNode->fillType; - if (pSqlNode->fillType == NULL) { - return TSDB_CODE_SUCCESS; - } - - tVariantListItem* pItem = taosArrayGet(pFillToken, 0); - - const int32_t START_INTERPO_COL_IDX = 1; - - const char* msg1 = "value is expected"; - const char* msg2 = "invalid fill option"; - const char* msg3 = "top/bottom not support fill"; - const char* msg4 = "illegal value or data overflow"; - const char* msg5 = "fill only available for interval query"; - const char* msg6 = "not supported function now"; - - if ((!isTimeWindowQuery(pQueryInfo)) && (!tscIsPointInterpQuery(pQueryInfo))) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - /* - * fill options are set at the end position, when all columns are set properly - * the columns may be increased due to group by operation - */ - if (checkQueryRangeForFill(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - - if (pItem->pVar.nType != TSDB_DATA_TYPE_BINARY) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - size_t numOfFields = tscNumOfFields(pQueryInfo); - - if (pQueryInfo->fillVal == NULL) { - pQueryInfo->fillVal = calloc(numOfFields, sizeof(int64_t)); - pQueryInfo->numOfFillVal = (int32_t)numOfFields; - if (pQueryInfo->fillVal == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - if (strncasecmp(pItem->pVar.pz, "none", 4) == 0 && pItem->pVar.nLen == 4) { - pQueryInfo->fillType = TSDB_FILL_NONE; - } else if (strncasecmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4) { - pQueryInfo->fillType = TSDB_FILL_NULL; - for (int32_t i = START_INTERPO_COL_IDX; i < numOfFields; ++i) { - TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - setNull((char*)&pQueryInfo->fillVal[i], pField->type, pField->bytes); - } - } else if (strncasecmp(pItem->pVar.pz, "prev", 4) == 0 && pItem->pVar.nLen == 4) { - pQueryInfo->fillType = TSDB_FILL_PREV; - if (tscIsPointInterpQuery(pQueryInfo) && pQueryInfo->order.order == TSDB_ORDER_DESC) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - } else if (strncasecmp(pItem->pVar.pz, "next", 4) == 0 && pItem->pVar.nLen == 4) { - pQueryInfo->fillType = TSDB_FILL_NEXT; - } else if (strncasecmp(pItem->pVar.pz, "linear", 6) == 0 && pItem->pVar.nLen == 6) { - pQueryInfo->fillType = TSDB_FILL_LINEAR; - } else if (strncasecmp(pItem->pVar.pz, "value", 5) == 0 && pItem->pVar.nLen == 5) { - pQueryInfo->fillType = TSDB_FILL_SET_VALUE; - - size_t num = taosArrayGetSize(pFillToken); - if (num == 1) { // no actual value, return with error code - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - int32_t startPos = 1; - int32_t numOfFillVal = (int32_t)(num - 1); - - /* for point interpolation query, we do not have the timestamp column */ - if (tscIsPointInterpQuery(pQueryInfo)) { - startPos = 0; - - if (numOfFillVal > numOfFields) { - numOfFillVal = (int32_t)numOfFields; - } - } else { - numOfFillVal = (int16_t)((num > (int32_t)numOfFields) ? (int32_t)numOfFields : num); - } - - int32_t j = 1; - - for (int32_t i = startPos; i < numOfFillVal; ++i, ++j) { - TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - if (pField->type == TSDB_DATA_TYPE_BINARY || pField->type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull((char*) &pQueryInfo->fillVal[i], pField->type); - continue; - } - - tVariant* p = taosArrayGet(pFillToken, j); - int32_t ret = tVariantDump(p, (char*)&pQueryInfo->fillVal[i], pField->type, true); - if (ret != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - } - - if ((num < numOfFields) || ((num - 1 < numOfFields) && (tscIsPointInterpQuery(pQueryInfo)))) { - tVariantListItem* lastItem = taosArrayGetLast(pFillToken); - - for (int32_t i = numOfFillVal; i < numOfFields; ++i) { - TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - if (pField->type == TSDB_DATA_TYPE_BINARY || pField->type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull((char*) &pQueryInfo->fillVal[i], pField->type); - } else { - tVariantDump(&lastItem->pVar, (char*)&pQueryInfo->fillVal[i], pField->type, true); - } - } - } - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_TOP || pExpr->base.functionId == TSDB_FUNC_BOTTOM) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - - return TSDB_CODE_SUCCESS; -} - -static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { - /* set default timestamp order information for all queries */ - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - pQueryInfo->order.order = TSDB_ORDER_ASC; - if (isTopBottomQuery(pQueryInfo)) { - pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - } else { // in case of select tbname from super_table, the default order column can not be the primary ts column - pQueryInfo->order.orderColId = INT32_MIN; // todo define a macro - } - - /* for super table query, set default ascending order for group output */ - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; - } - - if (pQueryInfo->distinct) { - pQueryInfo->order.order = TSDB_ORDER_ASC; - pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - } -} - -int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSchema* pSchema) { - const char* msg0 = "only one column allowed in orderby"; - const char* msg1 = "invalid column name in orderby clause"; - const char* msg2 = "too many order by columns"; - const char* msg3 = "only primary timestamp/tbname/first tag in groupby clause allowed"; - const char* msg4 = "only tag in groupby clause allowed in order clause"; - const char* msg5 = "only primary timestamp/column in top/bottom function allowed as order column"; - const char* msg6 = "only primary timestamp allowed as the second order column"; - const char* msg7 = "only primary timestamp/column in groupby clause allowed as order column"; - const char* msg8 = "only column in groupby clause allowed as order column"; - const char* msg9 = "orderby column must projected in subquery"; - const char* msg10 = "not support distinct mixed with order by"; - - setDefaultOrderInfo(pQueryInfo); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pSqlNode->pSortOrder == NULL) { - return TSDB_CODE_SUCCESS; - } - char* pMsgBuf = tscGetErrorMsgPayload(pCmd); - SArray* pSortOrder = pSqlNode->pSortOrder; - - /* - * for table query, there is only one or none order option is allowed, which is the - * ts or values(top/bottom) order is supported. - * - * for super table query, the order option must be less than 3. - */ - size_t size = taosArrayGetSize(pSortOrder); - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_TMP_TABLE(pTableMetaInfo)) { - if (size > 1) { - return invalidOperationMsg(pMsgBuf, msg0); - } - } else { - if (size > 2) { - return invalidOperationMsg(pMsgBuf, msg2); - } - } - if (size > 0 && pQueryInfo->distinct) { - return invalidOperationMsg(pMsgBuf, msg10); - } - - // handle the first part of order by - tVariant* pVar = taosArrayGet(pSortOrder, 0); - - // e.g., order by 1 asc, return directly with out further check. - if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) { - return TSDB_CODE_SUCCESS; - } - - SStrToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // super table query - if (getColumnIndexByName(&columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsgBuf, msg1); - } - - bool orderByTags = false; - bool orderByTS = false; - bool orderByGroupbyCol = false; - - if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - int32_t relTagIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - // it is a tag column - if (pQueryInfo->groupbyExpr.columnInfo == NULL) { - return invalidOperationMsg(pMsgBuf, msg4); - } - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); - if (relTagIndex == pColIndex->colIndex) { - orderByTags = true; - } - } else if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - orderByTags = true; - } - - if (PRIMARYKEY_TIMESTAMP_COL_INDEX == index.columnIndex) { - orderByTS = true; - } - - SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; - if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { - SColIndex* pColIndex = taosArrayGet(columnInfo, 0); - if (PRIMARYKEY_TIMESTAMP_COL_INDEX != index.columnIndex && pColIndex->colIndex == index.columnIndex) { - orderByGroupbyCol = true; - } - } - - if (!(orderByTags || orderByTS || orderByGroupbyCol) && !isTopBottomQuery(pQueryInfo)) { - return invalidOperationMsg(pMsgBuf, msg3); - } else { // order by top/bottom result value column is not supported in case of interval query. - assert(!(orderByTags && orderByTS && orderByGroupbyCol)); - } - - size_t s = taosArrayGetSize(pSortOrder); - if (s == 1) { - if (orderByTags) { - pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); - pQueryInfo->groupbyExpr.orderType = p1->sortOrder; - } else if (orderByGroupbyCol) { - tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); - - pQueryInfo->groupbyExpr.orderType = p1->sortOrder; - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; - } else if (isTopBottomQuery(pQueryInfo)) { - /* order of top/bottom query in interval is not valid */ - - int32_t pos = tscExprTopBottomIndex(pQueryInfo); - assert(pos > 0); - SExprInfo* pExpr = tscExprGet(pQueryInfo, pos - 1); - assert(pExpr->base.functionId == TSDB_FUNC_TS); - - pExpr = tscExprGet(pQueryInfo, pos); - - if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(pMsgBuf, msg5); - } - - tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); - pQueryInfo->order.order = p1->sortOrder; - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; - return TSDB_CODE_SUCCESS; - } else { - tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); - - pQueryInfo->order.order = p1->sortOrder; - pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - - // orderby ts query on super table - if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - bool found = false; - for (int32_t i = 0; i < tscNumOfExprs(pQueryInfo); ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - found = true; - break; - } - } - if (!found && pQueryInfo->pDownstream) { - return invalidOperationMsg(pMsgBuf, msg9); - } - addPrimaryTsColIntoResult(pQueryInfo, pCmd); - } - } - } else { - tVariantListItem *pItem = taosArrayGet(pSqlNode->pSortOrder, 0); - if (orderByTags) { - pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - pQueryInfo->groupbyExpr.orderType = pItem->sortOrder; - } else if (orderByGroupbyCol){ - pQueryInfo->order.order = pItem->sortOrder; - pQueryInfo->order.orderColId = index.columnIndex; - } else { - pQueryInfo->order.order = pItem->sortOrder; - pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - } - - pItem = taosArrayGet(pSqlNode->pSortOrder, 1); - tVariant* pVar2 = &pItem->pVar; - SStrToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; - if (getColumnIndexByName(&cname, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsgBuf, msg1); - } - - if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(pMsgBuf, msg6); - } else { - tVariantListItem* p1 = taosArrayGet(pSortOrder, 1); - pQueryInfo->order.order = p1->sortOrder; - pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - } - } - - } else if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) { // check order by clause for normal table & temp table - if (getColumnIndexByName(&columnName, pQueryInfo, &index, pMsgBuf) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsgBuf, msg1); - } - - if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX && !isTopBottomQuery(pQueryInfo)) { - bool validOrder = false; - SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; - if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { - SColIndex* pColIndex = taosArrayGet(columnInfo, 0); - validOrder = (pColIndex->colIndex == index.columnIndex); - } - - if (!validOrder) { - return invalidOperationMsg(pMsgBuf, msg7); - } - - tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); - pQueryInfo->groupbyExpr.orderIndex = pSchema[index.columnIndex].colId; - pQueryInfo->groupbyExpr.orderType = p1->sortOrder; - } - - if (isTopBottomQuery(pQueryInfo)) { - SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; - if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { - SColIndex* pColIndex = taosArrayGet(columnInfo, 0); - - if (pColIndex->colIndex == index.columnIndex) { - return invalidOperationMsg(pMsgBuf, msg8); - } - } else { - int32_t pos = tscExprTopBottomIndex(pQueryInfo); - assert(pos > 0); - SExprInfo* pExpr = tscExprGet(pQueryInfo, pos - 1); - assert(pExpr->base.functionId == TSDB_FUNC_TS); - - pExpr = tscExprGet(pQueryInfo, pos); - - if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(pMsgBuf, msg5); - } - } - - tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); - pQueryInfo->order.order = pItem->sortOrder; - - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; - return TSDB_CODE_SUCCESS; - } - - tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); - pQueryInfo->order.order = pItem->sortOrder; - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; - } else { - // handle the temp table order by clause. You can order by any single column in case of the temp table, created by - // inner subquery. - assert(UTIL_TABLE_IS_TMP_TABLE(pTableMetaInfo) && taosArrayGetSize(pSqlNode->pSortOrder) == 1); - - if (getColumnIndexByName(&columnName, pQueryInfo, &index, pMsgBuf) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsgBuf, msg1); - } - - tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); - pQueryInfo->order.order = pItem->sortOrder; - pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { - const int32_t DEFAULT_TABLE_INDEX = 0; - - const char* msg1 = "invalid table name"; - const char* msg3 = "manipulation of tag available for super table"; - const char* msg4 = "set tag value only available for table"; - const char* msg5 = "only support add one tag"; - const char* msg6 = "column can only be modified by super table"; - - const char* msg7 = "no tags can be dropped"; - const char* msg8 = "only support one tag"; - const char* msg9 = "tag name too long"; - - const char* msg10 = "invalid tag name"; - const char* msg11 = "primary tag cannot be dropped"; - const char* msg12 = "update normal column not supported"; - const char* msg13 = "invalid tag value"; - const char* msg14 = "tag value too long"; - - const char* msg15 = "no columns can be dropped"; - const char* msg16 = "only support one column"; - const char* msg17 = "invalid column name"; - const char* msg18 = "primary timestamp column cannot be dropped"; - const char* msg19 = "invalid new tag name"; - const char* msg20 = "table is not super table"; - const char* msg21 = "only binary/nchar column length could be modified"; - const char* msg23 = "only column length coulbe be modified"; - const char* msg24 = "invalid binary/nchar column length"; - - int32_t code = TSDB_CODE_SUCCESS; - - SSqlCmd* pCmd = &pSql->cmd; - SAlterTableInfo* pAlterSQL = pInfo->pAlterInfo; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, DEFAULT_TABLE_INDEX); - - if (tscValidateName(&(pAlterSQL->name)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - code = tscSetTableFullName(&pTableMetaInfo->name, &(pAlterSQL->name), pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = tscGetTableMeta(pSql, pTableMetaInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - char* pMsg = tscGetErrorMsgPayload(pCmd); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - if (pAlterSQL->tableType == TSDB_SUPER_TABLE && !(UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo))) { - return invalidOperationMsg(pMsg, msg20); - } - - if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN || - pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN) { - if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(pMsg, msg3); - } - } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) && (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo))) { - return invalidOperationMsg(pMsg, msg4); - } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_COLUMN) && - UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) { - return invalidOperationMsg(pMsg, msg6); - } - - if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { - SArray* pFieldList = pAlterSQL->pAddColumns; - if (taosArrayGetSize(pFieldList) > 1) { - return invalidOperationMsg(pMsg, msg5); - } - - TAOS_FIELD* p = taosArrayGet(pFieldList, 0); - int32_t ret = validateOneTag(pCmd, p); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); - } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { - if (tscGetNumOfTags(pTableMeta) == 1) { - return invalidOperationMsg(pMsg, msg7); - } - - // numOfTags == 1 - if (taosArrayGetSize(pAlterSQL->varList) > 1) { - return invalidOperationMsg(pMsg, msg8); - } - - tVariantListItem* pItem = taosArrayGet(pAlterSQL->varList, 0); - if (pItem->pVar.nLen >= TSDB_COL_NAME_LEN) { - return invalidOperationMsg(pMsg, msg9); - } - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - SStrToken name = {.z = pItem->pVar.pz, .n = pItem->pVar.nLen, .type = TK_STRING}; - - if (getColumnIndexByName(&name, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - if (index.columnIndex < numOfCols) { - return invalidOperationMsg(pMsg, msg10); - } else if (index.columnIndex == numOfCols) { - return invalidOperationMsg(pMsg, msg11); - } - - char name1[128] = {0}; - strncpy(name1, pItem->pVar.pz, pItem->pVar.nLen); - - TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypes[TSDB_DATA_TYPE_INT].bytes); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - } else if (pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { - SArray* pVarList = pAlterSQL->varList; - if (taosArrayGetSize(pVarList) > 2) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - tVariantListItem* pSrcItem = taosArrayGet(pAlterSQL->varList, 0); - tVariantListItem* pDstItem = taosArrayGet(pAlterSQL->varList, 1); - - if (pSrcItem->pVar.nLen >= TSDB_COL_NAME_LEN || pDstItem->pVar.nLen >= TSDB_COL_NAME_LEN) { - return invalidOperationMsg(pMsg, msg9); - } - - if (pSrcItem->pVar.nType != TSDB_DATA_TYPE_BINARY || pDstItem->pVar.nType != TSDB_DATA_TYPE_BINARY) { - return invalidOperationMsg(pMsg, msg10); - } - - SColumnIndex srcIndex = COLUMN_INDEX_INITIALIZER; - SColumnIndex destIndex = COLUMN_INDEX_INITIALIZER; - - SStrToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; - if (getColumnIndexByName(&srcToken, pQueryInfo, &srcIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsg, msg17); - } - - SStrToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; - if (getColumnIndexByName(&destToken, pQueryInfo, &destIndex, tscGetErrorMsgPayload(pCmd)) == TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsg, msg19); - } - - tVariantListItem* pItem = taosArrayGet(pVarList, 0); - - char name[TSDB_COL_NAME_LEN] = {0}; - strncpy(name, pItem->pVar.pz, pItem->pVar.nLen); - TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypes[TSDB_DATA_TYPE_INT].bytes); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - - pItem = taosArrayGet(pVarList, 1); - memset(name, 0, tListLen(name)); - - strncpy(name, pItem->pVar.pz, pItem->pVar.nLen); - f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypes[TSDB_DATA_TYPE_INT].bytes); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - } else if (pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { - // Note: update can only be applied to table not super table. - // the following is used to handle tags value for table created according to super table - pCmd->command = TSDB_SQL_UPDATE_TAGS_VAL; - - SArray* pVarList = pAlterSQL->varList; - tVariantListItem* item = taosArrayGet(pVarList, 0); - int16_t numOfTags = tscGetNumOfTags(pTableMeta); - - SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SStrToken name = {.type = TK_STRING, .z = item->pVar.pz, .n = item->pVar.nLen}; - if (getColumnIndexByName(&name, pQueryInfo, &columnIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (columnIndex.columnIndex < tscGetNumOfColumns(pTableMeta)) { - return invalidOperationMsg(pMsg, msg12); - } - - tVariantListItem* pItem = taosArrayGet(pVarList, 1); - SSchema* pTagsSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex); - - if (IS_VAR_DATA_TYPE(pTagsSchema->type) && (pItem->pVar.nLen > pTagsSchema->bytes * TSDB_NCHAR_SIZE)) { - return invalidOperationMsg(pMsg, msg14); - } - - pAlterSQL->tagData.data = calloc(1, pTagsSchema->bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); - - if (tVariantDump(&pItem->pVar, pAlterSQL->tagData.data, pTagsSchema->type, true) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsg, msg13); - } - - pAlterSQL->tagData.dataLen = pTagsSchema->bytes; - - // validate the length of binary - if ((pTagsSchema->type == TSDB_DATA_TYPE_BINARY || pTagsSchema->type == TSDB_DATA_TYPE_NCHAR) && - varDataTLen(pAlterSQL->tagData.data) > pTagsSchema->bytes) { - return invalidOperationMsg(pMsg, msg14); - } - - int32_t schemaLen = sizeof(STColumn) * numOfTags; - int32_t size = sizeof(SUpdateTableTagValMsg) + pTagsSchema->bytes + schemaLen + TSDB_EXTRA_PAYLOAD_SIZE; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - tscError("0x%"PRIx64" failed to malloc for alter table pMsg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; - pUpdateMsg->head.vgId = htonl(pTableMeta->vgId); - pUpdateMsg->tid = htonl(pTableMeta->id.tid); - pUpdateMsg->uid = htobe64(pTableMeta->id.uid); - pUpdateMsg->colId = htons(pTagsSchema->colId); - pUpdateMsg->type = pTagsSchema->type; - pUpdateMsg->bytes = htons(pTagsSchema->bytes); - pUpdateMsg->tversion = htons(pTableMeta->tversion); - pUpdateMsg->numOfTags = htons(numOfTags); - pUpdateMsg->schemaLen = htonl(schemaLen); - - // the schema is located after the pMsg body, then followed by true tag value - char* d = pUpdateMsg->data; - SSchema* pTagCols = tscGetTableTagSchema(pTableMeta); - for (int i = 0; i < numOfTags; ++i) { - STColumn* pCol = (STColumn*) d; - pCol->colId = htons(pTagCols[i].colId); - pCol->bytes = htons(pTagCols[i].bytes); - pCol->type = pTagCols[i].type; - pCol->offset = 0; - - d += sizeof(STColumn); - } - - // copy the tag value to pMsg body - pItem = taosArrayGet(pVarList, 1); - tVariantDump(&pItem->pVar, pUpdateMsg->data + schemaLen, pTagsSchema->type, true); - - int32_t len = 0; - if (pTagsSchema->type != TSDB_DATA_TYPE_BINARY && pTagsSchema->type != TSDB_DATA_TYPE_NCHAR) { - len = tDataTypes[pTagsSchema->type].bytes; - } else { - len = varDataTLen(pUpdateMsg->data + schemaLen); - } - - pUpdateMsg->tagValLen = htonl(len); // length may be changed after dump data - - int32_t total = sizeof(SUpdateTableTagValMsg) + len + schemaLen; - pUpdateMsg->head.contLen = htonl(total); - - } else if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN) { - SArray* pFieldList = pAlterSQL->pAddColumns; - if (taosArrayGetSize(pFieldList) > 1) { - const char* msgx = "only support add one column"; - return invalidOperationMsg(pMsg, msgx); - } - - TAOS_FIELD* p = taosArrayGet(pFieldList, 0); - int32_t ret = validateOneColumn(pCmd, p); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); - } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) { - if (tscGetNumOfColumns(pTableMeta) == TSDB_MIN_COLUMNS) { // - return invalidOperationMsg(pMsg, msg15); - } - - size_t size = taosArrayGetSize(pAlterSQL->varList); - if (size > 1) { - return invalidOperationMsg(pMsg, msg16); - } - - tVariantListItem* pItem = taosArrayGet(pAlterSQL->varList, 0); - - SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SStrToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen}; - if (getColumnIndexByName(&name, pQueryInfo, &columnIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsg, msg17); - } - - if (columnIndex.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(pMsg, msg18); - } - - char name1[TSDB_COL_NAME_LEN] = {0}; - tstrncpy(name1, pItem->pVar.pz, sizeof(name1)); - TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypes[TSDB_DATA_TYPE_INT].bytes); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - } else if (pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_COLUMN) { - if (taosArrayGetSize(pAlterSQL->pAddColumns) >= 2) { - return invalidOperationMsg(pMsg, msg16); - } - - - TAOS_FIELD* pItem = taosArrayGet(pAlterSQL->pAddColumns, 0); - if (pItem->type != TSDB_DATA_TYPE_BINARY && pItem->type != TSDB_DATA_TYPE_NCHAR) { - return invalidOperationMsg(pMsg, msg21); - } - - SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SStrToken name = {.type = TK_STRING, .z = pItem->name, .n = (uint32_t)strlen(pItem->name)}; - if (getColumnIndexByName(&name, pQueryInfo, &columnIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsg, msg17); - } - - SSchema* pColSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex); - - if (pColSchema->type != TSDB_DATA_TYPE_BINARY && pColSchema->type != TSDB_DATA_TYPE_NCHAR) { - return invalidOperationMsg(pMsg, msg21); - } - - if (pItem->type != pColSchema->type) { - return invalidOperationMsg(pMsg, msg23); - } - - if ((pItem->type == TSDB_DATA_TYPE_BINARY && (pItem->bytes <= 0 || pItem->bytes > TSDB_MAX_BINARY_LEN)) || - (pItem->type == TSDB_DATA_TYPE_NCHAR && (pItem->bytes <= 0 || pItem->bytes > TSDB_MAX_NCHAR_LEN))) { - return invalidOperationMsg(pMsg, msg24); - } - - if (pItem->bytes <= pColSchema->bytes) { - return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, pMsg, pItem->name, NULL); - } - - SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema; - int16_t numOfColumns = pTableMetaInfo->pTableMeta->tableInfo.numOfColumns; - int16_t i; - uint32_t nLen = 0; - for (i = 0; i < numOfColumns; ++i) { - nLen += (i != columnIndex.columnIndex) ? pSchema[i].bytes : pItem->bytes; - } - if (nLen >= TSDB_MAX_BYTES_PER_ROW) { - return invalidOperationMsg(pMsg, msg24); - } - TAOS_FIELD f = tscCreateField(pColSchema->type, name.z, pItem->bytes); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - }else if (pAlterSQL->type == TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN) { - if (taosArrayGetSize(pAlterSQL->pAddColumns) >= 2) { - return invalidOperationMsg(pMsg, msg16); - } - - TAOS_FIELD* pItem = taosArrayGet(pAlterSQL->pAddColumns, 0); - if (pItem->type != TSDB_DATA_TYPE_BINARY && pItem->type != TSDB_DATA_TYPE_NCHAR) { - return invalidOperationMsg(pMsg, msg21); - } - - SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SStrToken name = {.type = TK_STRING, .z = pItem->name, .n = (uint32_t)strlen(pItem->name)}; - if (getColumnIndexByName(&name, pQueryInfo, &columnIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(pMsg, msg17); - } - - SSchema* pColSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex); - - if (columnIndex.columnIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - return invalidOperationMsg(pMsg, msg10); - } - - if (pColSchema->type != TSDB_DATA_TYPE_BINARY && pColSchema->type != TSDB_DATA_TYPE_NCHAR) { - return invalidOperationMsg(pMsg, msg21); - } - - if (pItem->type != pColSchema->type) { - return invalidOperationMsg(pMsg, msg23); - } - - if ((pItem->type == TSDB_DATA_TYPE_BINARY && (pItem->bytes <= 0 || pItem->bytes > TSDB_MAX_BINARY_LEN)) || - (pItem->type == TSDB_DATA_TYPE_NCHAR && (pItem->bytes <= 0 || pItem->bytes > TSDB_MAX_NCHAR_LEN))) { - return invalidOperationMsg(pMsg, msg24); - } - - if (pItem->bytes <= pColSchema->bytes) { - return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_TAG_LENGTH, pMsg, pItem->name, NULL); - } - - SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - int16_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta); - int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - int32_t tagIndex = columnIndex.columnIndex - numOfCols; - assert(tagIndex>=0); - uint32_t nLen = 0; - for (int i = 0; i < numOfTags; ++i) { - nLen += (i != tagIndex) ? pSchema[i].bytes : pItem->bytes; - } - if (nLen >= TSDB_MAX_TAGS_LEN) { - return invalidOperationMsg(pMsg, msg24); - } - - TAOS_FIELD f = tscCreateField(pColSchema->type, name.z, pItem->bytes); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char* msg0 = "sample interval can not be less than 10ms."; - const char* msg1 = "functions not allowed in select clause"; - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - if (pQueryInfo->interval.interval != 0 && - convertTimePrecision(pQueryInfo->interval.interval, tinfo.precision, TSDB_TIME_PRECISION_MILLI)< 10 && - pQueryInfo->interval.intervalUnit != 'n' && - pQueryInfo->interval.intervalUnit != 'y') { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); - } - - size_t size = taosArrayGetSize(pQueryInfo->exprList); - for (int32_t i = 0; i < size; ++i) { - int32_t functId = tscExprGet(pQueryInfo, i)->base.functionId; - if (!IS_STREAM_QUERY_VALID(aAggs[functId].status)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - bool isProjectionFunction = false; - const char* msg1 = "functions not compatible with interval"; - - // multi-output set/ todo refactor - size_t size = taosArrayGetSize(pQueryInfo->exprList); - - for (int32_t k = 0; k < size; ++k) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, k); - - if (pExpr->base.functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * pExpr->base.functionId - 1); - if (pUdfInfo->funcType == TSDB_UDF_TYPE_SCALAR) { - isProjectionFunction = true; - break; - } else { - continue; - } - } - - // projection query on primary timestamp, the selectivity function needs to be present. - if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - bool hasSelectivity = false; - for (int32_t j = 0; j < size; ++j) { - SExprInfo* pEx = tscExprGet(pQueryInfo, j); - if ((aAggs[pEx->base.functionId].status & TSDB_FUNCSTATE_SELECTIVITY) == TSDB_FUNCSTATE_SELECTIVITY) { - hasSelectivity = true; - break; - } - } - - if (hasSelectivity) { - continue; - } - } - - int32_t f = pExpr->base.functionId; - if ((f == TSDB_FUNC_PRJ && pExpr->base.numOfParams == 0) || f == TSDB_FUNC_DIFF || f == TSDB_FUNC_ARITHM || f == TSDB_FUNC_DERIVATIVE) { - isProjectionFunction = true; - break; - } - } - - if (isProjectionFunction) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - return isProjectionFunction == true ? TSDB_CODE_TSC_INVALID_OPERATION : TSDB_CODE_SUCCESS; -} - -typedef struct SDNodeDynConfOption { - char* name; // command name - int32_t len; // name string length -} SDNodeDynConfOption; - - -int32_t validateEp(char* ep) { - char buf[TSDB_EP_LEN + 1] = {0}; - tstrncpy(buf, ep, TSDB_EP_LEN); - - char* pos = strchr(buf, ':'); - if (NULL == pos) { - int32_t val = strtol(ep, NULL, 10); - if (val <= 0 || val > 65536) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else { - uint16_t port = atoi(pos + 1); - if (0 == port) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t validateDNodeConfig(SMiscInfo* pOptions) { - int32_t numOfToken = (int32_t) taosArrayGetSize(pOptions->a); - - if (numOfToken < 2 || numOfToken > 3) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - const int tokenLogEnd = 2; - const int tokenBalance = 2; - const int tokenMonitor = 3; - const int tokenDebugFlag = 4; - const int tokenDebugFlagEnd = 20; - const SDNodeDynConfOption cfgOptions[] = { - {"resetLog", 8}, {"resetQueryCache", 15}, {"balance", 7}, {"monitor", 7}, - {"debugFlag", 9}, {"monDebugFlag", 12}, {"vDebugFlag", 10}, {"mDebugFlag", 10}, - {"cDebugFlag", 10}, {"httpDebugFlag", 13}, {"qDebugflag", 10}, {"sdbDebugFlag", 12}, - {"uDebugFlag", 10}, {"tsdbDebugFlag", 13}, {"sDebugflag", 10}, {"rpcDebugFlag", 12}, - {"dDebugFlag", 10}, {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"tmrDebugFlag", 12}, - {"cqDebugFlag", 11}, - }; - - SStrToken* pOptionToken = taosArrayGet(pOptions->a, 1); - - if (numOfToken == 2) { - // reset log and reset query cache does not need value - for (int32_t i = 0; i < tokenLogEnd; ++i) { - const SDNodeDynConfOption* pOption = &cfgOptions[i]; - if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { - return TSDB_CODE_SUCCESS; - } - } - } else if ((strncasecmp(cfgOptions[tokenBalance].name, pOptionToken->z, pOptionToken->n) == 0) && - (cfgOptions[tokenBalance].len == pOptionToken->n)) { - SStrToken* pValToken = taosArrayGet(pOptions->a, 2); - int32_t vnodeId = 0; - int32_t dnodeId = 0; - strdequote(pValToken->z); - bool parseOk = taosCheckBalanceCfgOptions(pValToken->z, &vnodeId, &dnodeId); - if (!parseOk) { - return TSDB_CODE_TSC_INVALID_OPERATION; // options value is invalid - } - return TSDB_CODE_SUCCESS; - } else if ((strncasecmp(cfgOptions[tokenMonitor].name, pOptionToken->z, pOptionToken->n) == 0) && - (cfgOptions[tokenMonitor].len == pOptionToken->n)) { - SStrToken* pValToken = taosArrayGet(pOptions->a, 2); - int32_t val = strtol(pValToken->z, NULL, 10); - if (val != 0 && val != 1) { - return TSDB_CODE_TSC_INVALID_OPERATION; // options value is invalid - } - return TSDB_CODE_SUCCESS; - } else { - SStrToken* pValToken = taosArrayGet(pOptions->a, 2); - - int32_t val = strtol(pValToken->z, NULL, 10); - if (val < 0 || val > 256) { - /* options value is out of valid range */ - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - for (int32_t i = tokenDebugFlag; i < tokenDebugFlagEnd; ++i) { - const SDNodeDynConfOption* pOption = &cfgOptions[i]; - - // options is valid - if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { - return TSDB_CODE_SUCCESS; - } - } - } - - return TSDB_CODE_TSC_INVALID_OPERATION; -} - -int32_t validateLocalConfig(SMiscInfo* pOptions) { - int32_t numOfToken = (int32_t) taosArrayGetSize(pOptions->a); - if (numOfToken < 1 || numOfToken > 2) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - SDNodeDynConfOption LOCAL_DYNAMIC_CFG_OPTIONS[6] = {{"resetLog", 8}, {"rpcDebugFlag", 12}, {"tmrDebugFlag", 12}, - {"cDebugFlag", 10}, {"uDebugFlag", 10}, {"debugFlag", 9}}; - - - SStrToken* pOptionToken = taosArrayGet(pOptions->a, 0); - - if (numOfToken == 1) { - // reset log does not need value - for (int32_t i = 0; i < 1; ++i) { - SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i]; - if ((pOption->len == pOptionToken->n) && - (strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0)) { - return TSDB_CODE_SUCCESS; - } - } - } else { - SStrToken* pValToken = taosArrayGet(pOptions->a, 1); - - int32_t val = strtol(pValToken->z, NULL, 10); - if (!validateDebugFlag(val)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - for (int32_t i = 1; i < tListLen(LOCAL_DYNAMIC_CFG_OPTIONS); ++i) { - SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i]; - if ((pOption->len == pOptionToken->n) - && (strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0)) { - return TSDB_CODE_SUCCESS; - } - } - } - return TSDB_CODE_TSC_INVALID_OPERATION; -} - -int32_t validateColumnName(char* name) { - bool ret = taosIsKeyWordToken(name, (int32_t)strlen(name)); - if (ret) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - SStrToken token = {.z = name}; - token.n = tGetToken(name, &token.type); - - if (token.type != TK_STRING && token.type != TK_ID) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (token.type == TK_STRING) { - strdequote(token.z); - strntolower(token.z, token.z, token.n); - token.n = (uint32_t)strtrim(token.z); - - int32_t k = tGetToken(token.z, &token.type); - if (k != token.n) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return validateColumnName(token.z); - } else { - if (isNumber(&token)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - return TSDB_CODE_SUCCESS; -} - -bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo) { - if (!tscIsPointInterpQuery(pQueryInfo)) { - return true; - } - - if (pQueryInfo->window.skey == INT64_MIN || pQueryInfo->window.ekey == INT64_MAX) { - return false; - } - - return !(pQueryInfo->window.skey != pQueryInfo->window.ekey && pQueryInfo->interval.interval == 0); -} - -int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSqlObj* pSql) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - const char* msg0 = "soffset/offset can not be less than 0"; - const char* msg1 = "slimit/soffset only available for STable query"; - const char* msg2 = "slimit/soffset can not apply to projection query"; - - // handle the limit offset value, validate the limit - pQueryInfo->limit = pSqlNode->limit; - pQueryInfo->clauseLimit = pQueryInfo->limit.limit; - pQueryInfo->slimit = pSqlNode->slimit; - - tscDebug("0x%"PRIx64" limit:%" PRId64 ", offset:%" PRId64 " slimit:%" PRId64 ", soffset:%" PRId64, pSql->self, - pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->slimit.limit, pQueryInfo->slimit.offset); - - if (pQueryInfo->slimit.offset < 0 || pQueryInfo->limit.offset < 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); - } - - if (pQueryInfo->limit.limit == 0) { - tscDebug("0x%"PRIx64" limit 0, no output result", pSql->self); - pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - return TSDB_CODE_SUCCESS; - } - - // todo refactor - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - if (!tscQueryTags(pQueryInfo)) { // local handle the super table tag query - if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { - if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - // for projection query on super table, all queries are subqueries - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY)) { - pQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; - } - } - } - - if (pQueryInfo->slimit.limit == 0) { - tscDebug("0x%"PRIx64" slimit 0, no output result", pSql->self); - pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - return TSDB_CODE_SUCCESS; - } - - /* - * Get the distribution of all tables among all available virtual nodes that are qualified for the query condition - * and created according to this super table from management node. - * And then launching multiple async-queries against all qualified virtual nodes, during the first-stage - * query operation. - */ -// assert(allVgroupInfoRetrieved(pQueryInfo)); - - // No tables included. No results generated. Query results are empty. - if (pTableMetaInfo->vgroupList->numOfVgroups == 0) { - tscDebug("0x%"PRIx64" no table in super table, no output result", pSql->self); - pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - return TSDB_CODE_SUCCESS; - } - - // keep original limitation value in globalLimit - pQueryInfo->clauseLimit = pQueryInfo->limit.limit; - pQueryInfo->prjOffset = pQueryInfo->limit.offset; - pQueryInfo->vgroupLimit = -1; - - if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - /* - * The offset value should be removed during retrieve data from virtual node, since the - * global order are done at the client side, so the offset is applied at the client side. - * However, note that the maximum allowed number of result for each table should be less - * than or equal to the value of limit. - */ - if (pQueryInfo->limit.limit > 0) { - pQueryInfo->vgroupLimit = pQueryInfo->limit.limit + pQueryInfo->limit.offset; - pQueryInfo->limit.limit = -1; - } - - pQueryInfo->limit.offset = 0; - } - } else { - if (pQueryInfo->slimit.limit != -1 || pQueryInfo->slimit.offset != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDbInfo* pCreateDb) { - const char* msg1 = "invalid number of keep options"; - const char* msg2 = "invalid keep value"; - const char* msg3 = "invalid keep value, should be keep0 <= keep1 <= keep2"; - - pMsg->daysToKeep0 = htonl(-1); - pMsg->daysToKeep1 = htonl(-1); - pMsg->daysToKeep2 = htonl(-1); - - SArray* pKeep = pCreateDb->keep; - if (pKeep != NULL) { - size_t s = taosArrayGetSize(pKeep); -#ifdef _STORAGE - if (s >= 4 ||s <= 0) { -#else - if (s != 1) { -#endif - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - tVariantListItem* p0 = taosArrayGet(pKeep, 0); - tVariantListItem* p1 = (s > 1) ? taosArrayGet(pKeep, 1) : p0; - tVariantListItem* p2 = (s > 2) ? taosArrayGet(pKeep, 2) : p1; - - if ((int32_t)p0->pVar.i64 <= 0 || (int32_t)p1->pVar.i64 <= 0 || (int32_t)p2->pVar.i64 <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - if (!(((int32_t)p0->pVar.i64 <= (int32_t)p1->pVar.i64) && ((int32_t)p1->pVar.i64 <= (int32_t)p2->pVar.i64))) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - pMsg->daysToKeep0 = htonl((int32_t)p0->pVar.i64); - pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64); - pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64); - - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDbInfo* pCreateDbInfo) { - const char* msg = "invalid time precision"; - - pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default - - SStrToken* pToken = &pCreateDbInfo->precision; - if (pToken->n > 0) { - pToken->n = strdequote(pToken->z); - - if (strncmp(pToken->z, TSDB_TIME_PRECISION_MILLI_STR, pToken->n) == 0 && - strlen(TSDB_TIME_PRECISION_MILLI_STR) == pToken->n) { - // time precision for this db: million second - pMsg->precision = TSDB_TIME_PRECISION_MILLI; - } else if (strncmp(pToken->z, TSDB_TIME_PRECISION_MICRO_STR, pToken->n) == 0 && - strlen(TSDB_TIME_PRECISION_MICRO_STR) == pToken->n) { - pMsg->precision = TSDB_TIME_PRECISION_MICRO; - } else if (strncmp(pToken->z, TSDB_TIME_PRECISION_NANO_STR, pToken->n) == 0 && - strlen(TSDB_TIME_PRECISION_NANO_STR) == pToken->n) { - pMsg->precision = TSDB_TIME_PRECISION_NANO; - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - } - - - return TSDB_CODE_SUCCESS; -} - -static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDbInfo* pCreateDb) { - pMsg->maxTables = htonl(-1); // max tables can not be set anymore - pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); - pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); - pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); - pMsg->commitTime = htonl((int32_t)pCreateDb->commitTime); - pMsg->minRowsPerFileBlock = htonl(pCreateDb->minRowsPerBlock); - pMsg->maxRowsPerFileBlock = htonl(pCreateDb->maxRowsPerBlock); - pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod); - pMsg->compression = pCreateDb->compressionLevel; - pMsg->walLevel = (char)pCreateDb->walLevel; - pMsg->replications = pCreateDb->replica; - pMsg->quorum = pCreateDb->quorum; - pMsg->ignoreExist = pCreateDb->ignoreExists; - pMsg->update = pCreateDb->update; - pMsg->cacheLastRow = pCreateDb->cachelast; - pMsg->dbType = pCreateDb->dbType; - pMsg->partitions = htons(pCreateDb->partitions); -} - -int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDbInfo* pCreateDbSql) { - SCreateDbMsg* pMsg = (SCreateDbMsg *)(pCmd->payload); - setCreateDBOption(pMsg, pCreateDbSql); - - if (setKeepOption(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (setTimePrecision(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (tscCheckCreateDbParams(pCmd, pMsg) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} - -void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex) { - SQueryInfo* pParentQueryInfo = tscGetQueryInfo(&pParentObj->cmd); - - if (pParentQueryInfo->groupbyExpr.numOfGroupCols > 0) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - SExprInfo* pExpr = NULL; - - size_t size = taosArrayGetSize(pQueryInfo->exprList); - if (size > 0) { - pExpr = tscExprGet(pQueryInfo, (int32_t)size - 1); - } - - if (pExpr == NULL || pExpr->base.functionId != TSDB_FUNC_TAG) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pParentQueryInfo, tableIndex); - - uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; - int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, uid); - - SSchema* pTagSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, colId); - int16_t colIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, colId); - SColumnIndex index = {.tableIndex = 0, .columnIndex = colIndex}; - - char* name = pTagSchema->name; - int16_t type = pTagSchema->type; - int16_t bytes = pTagSchema->bytes; - - pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, getNewResColId(&pSql->cmd), bytes, true); - pExpr->base.colInfo.flag = TSDB_COL_TAG; - - // NOTE: tag column does not add to source column list - SColumnList ids = {0}; - insertResultField(pQueryInfo, (int32_t)size, &ids, bytes, (int8_t)type, name, pExpr); - - int32_t relIndex = index.columnIndex; - - pExpr->base.colInfo.colIndex = relIndex; - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); - pColIndex->colIndex = relIndex; - - tscColumnListInsert(pTableMetaInfo->tagColList, relIndex, uid, pTagSchema); - } - } -} - -// limit the output to be 1 for each state value -static void doLimitOutputNormalColOfGroupby(SExprInfo* pExpr) { - int32_t outputRow = 1; - tVariantCreateFromBinary(&pExpr->base.param[0], (char*)&outputRow, sizeof(int32_t), TSDB_DATA_TYPE_INT); - pExpr->base.numOfParams = 1; -} - -void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex, SSqlCmd* pCmd) { - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, tagIndex); - size_t size = tscNumOfExprs(pQueryInfo); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->colIndex); - SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pColIndex->colIndex}; - - SExprInfo* pExprInfo = tscAddFuncInSelectClause(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, - TSDB_COL_NORMAL, getNewResColId(pCmd)); - - strncpy(pExprInfo->base.token, pExprInfo->base.colInfo.name, tListLen(pExprInfo->base.token)); - - int32_t numOfFields = tscNumOfFields(pQueryInfo); - SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfFields - 1); - - doLimitOutputNormalColOfGroupby(pInfo->pExpr); - pInfo->visible = false; -} - -static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { - int32_t tagLength = 0; - size_t size = taosArrayGetSize(pQueryInfo->exprList); - -//todo is 0?? - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_TAGPRJ || pExpr->base.functionId == TSDB_FUNC_TAG) { - pExpr->base.functionId = TSDB_FUNC_TAG_DUMMY; - tagLength += pExpr->base.resBytes; - } else if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - pExpr->base.functionId = TSDB_FUNC_TS_DUMMY; - tagLength += pExpr->base.resBytes; - } - } - - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId < 0) { - continue; - } - - if ((pExpr->base.functionId != TSDB_FUNC_TAG_DUMMY && pExpr->base.functionId != TSDB_FUNC_TS_DUMMY) && - !(pExpr->base.functionId == TSDB_FUNC_PRJ && TSDB_COL_IS_UD_COL(pExpr->base.colInfo.flag))) { - SSchema* pColSchema = &pSchema[pExpr->base.colInfo.colIndex]; - getResultDataInfo(pColSchema->type, pColSchema->bytes, pExpr->base.functionId, (int32_t)pExpr->base.param[0].i64, &pExpr->base.resType, - &pExpr->base.resBytes, &pExpr->base.interBytes, tagLength, isSTable, NULL); - } - } -} - -static int32_t doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { - size_t size = taosArrayGetSize(pQueryInfo->exprList); - - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - - if (pExpr->base.functionId == TSDB_FUNC_PRJ && (!TSDB_COL_IS_UD_COL(pExpr->base.colInfo.flag) && (pExpr->base.colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX))) { - bool qualifiedCol = false; - for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); - - if (pExpr->base.colInfo.colId == pColIndex->colId) { - qualifiedCol = true; - doLimitOutputNormalColOfGroupby(pExpr); - pExpr->base.numOfParams = 1; - break; - } - } - - // it is not a tag column/tbname column/user-defined column, return error - if (!qualifiedCol) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - } - - return TSDB_CODE_SUCCESS; -} - -static bool tagColumnInGroupby(SGroupbyExpr* pGroupbyExpr, int16_t columnId) { - for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { - SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, j); - - if (columnId == pColIndex->colId && TSDB_COL_IS_TAG(pColIndex->flag )) { - return true; - } - } - - return false; -} - -static bool onlyTagPrjFunction(SQueryInfo* pQueryInfo) { - bool hasTagPrj = false; - bool hasColumnPrj = false; - - size_t size = taosArrayGetSize(pQueryInfo->exprList); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_PRJ) { - hasColumnPrj = true; - } else if (pExpr->base.functionId == TSDB_FUNC_TAGPRJ) { - hasTagPrj = true; - } - } - - return (hasTagPrj) && (hasColumnPrj == false); -} - -// check if all the tags prj columns belongs to the group by columns -static bool allTagPrjInGroupby(SQueryInfo* pQueryInfo) { - bool allInGroupby = true; - - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId != TSDB_FUNC_TAGPRJ) { - continue; - } - - if (!tagColumnInGroupby(&pQueryInfo->groupbyExpr, pExpr->base.colInfo.colId)) { - allInGroupby = false; - break; - } - } - - // all selected tag columns belong to the group by columns set, always correct - return allInGroupby; -} - -static void updateTagPrjFunction(SQueryInfo* pQueryInfo) { - size_t size = taosArrayGetSize(pQueryInfo->exprList); - - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_TAGPRJ) { - pExpr->base.functionId = TSDB_FUNC_TAG; - } - } -} - -/* - * check for selectivity function + tags column function both exist. - * 1. tagprj functions are not compatible with aggregated function when missing "group by" clause - * 2. if selectivity function and tagprj function both exist, there should be only - * one selectivity function exists. - */ -static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, char* msg) { - const char* msg1 = "only one selectivity function allowed in presence of tags function"; - const char* msg2 = "aggregation function should not be mixed up with projection"; - - bool tagTsColExists = false; - int16_t numOfSelectivity = 0; - int16_t numOfAggregation = 0; - - size_t numOfExprs = taosArrayGetSize(pQueryInfo->exprList); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = taosArrayGetP(pQueryInfo->exprList, i); - if (pExpr->base.functionId == TSDB_FUNC_TAGPRJ || - (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)) { - tagTsColExists = true; // selectivity + ts/tag column - break; - } - } - - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = taosArrayGetP(pQueryInfo->exprList, i); - - int16_t functionId = pExpr->base.functionId; - if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS || - functionId == TSDB_FUNC_ARITHM || functionId == TSDB_FUNC_TS_DUMMY) { - continue; - } - - if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - ++numOfAggregation; - } - - continue; - } - - if ((aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - numOfSelectivity++; - } else { - numOfAggregation++; - } - } - - if (tagTsColExists) { // check if the selectivity function exists - // When the tag projection function on tag column that is not in the group by clause, aggregation function and - // selectivity function exist in select clause is not allowed. - if (numOfAggregation > 0) { - return invalidOperationMsg(msg, msg1); - } - - /* - * if numOfSelectivity equals to 0, it is a super table projection query - */ - if (numOfSelectivity == 1) { - doUpdateSqlFunctionForTagPrj(pQueryInfo); - int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - } else if (numOfSelectivity > 1) { - /* - * If more than one selectivity functions exist, all the selectivity functions must be last_row. - * Otherwise, return with error code. - */ - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - int16_t functionId = pExpr->base.functionId; - if (functionId == TSDB_FUNC_TAGPRJ || (aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) == 0) { - continue; - } - - if ((functionId == TSDB_FUNC_LAST_ROW) || - (functionId == TSDB_FUNC_LAST_DST && (pExpr->base.colInfo.flag & TSDB_COL_NULL) != 0)) { - // do nothing - } else { - return invalidOperationMsg(msg, msg1); - } - } - - doUpdateSqlFunctionForTagPrj(pQueryInfo); - int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - } else { - if ((pQueryInfo->type & TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) { - if (numOfAggregation > 0 && pQueryInfo->groupbyExpr.numOfGroupCols == 0) { - return invalidOperationMsg(msg, msg2); - } - - if (numOfAggregation > 0 || numOfSelectivity > 0) { - // clear the projection type flag - pQueryInfo->type &= (~TSDB_QUERY_TYPE_PROJECTION_QUERY); - int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char* msg1 = "interval not allowed in group by normal column"; - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - SSchema* tagSchema = NULL; - if (!UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - tagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - } - - SSchema tmp = {.type = 0, .name = "", .colId = 0, .bytes = 0}; - SSchema* s = &tmp; - - for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i); - int16_t colIndex = pColIndex->colIndex; - - if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { - s = tGetTbnameColumnSchema(); - } else { - if (TSDB_COL_IS_TAG(pColIndex->flag)) { - if(tagSchema){ - s = &tagSchema[colIndex]; - } - } else { - s = &pSchema[colIndex]; - } - } - - if (TSDB_COL_IS_TAG(pColIndex->flag)) { - - int32_t f = TSDB_FUNC_TAG; - if (tscIsDiffDerivQuery(pQueryInfo)) { - f = TSDB_FUNC_TAGPRJ; - } - - int32_t pos = tscGetFirstInvisibleFieldPos(pQueryInfo); - - SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex}; - SExprInfo* pExpr = tscExprInsert(pQueryInfo, pos, f, &index, s->type, s->bytes, getNewResColId(pCmd), s->bytes, true); - - memset(pExpr->base.aliasName, 0, sizeof(pExpr->base.aliasName)); - tstrncpy(pExpr->base.aliasName, s->name, sizeof(pExpr->base.aliasName)); - tstrncpy(pExpr->base.token, s->name, sizeof(pExpr->base.aliasName)); - - pExpr->base.colInfo.flag = TSDB_COL_TAG; - - // NOTE: tag column does not add to source column list - SColumnList ids = createColumnList(1, 0, pColIndex->colIndex); - insertResultField(pQueryInfo, pos, &ids, s->bytes, (int8_t)s->type, s->name, pExpr); - } else { - // if this query is "group by" normal column, time window query is not allowed - if (isTimeWindowQuery(pQueryInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - size_t size = tscNumOfExprs(pQueryInfo); - - bool hasGroupColumn = false; - for (int32_t j = 0; j < size; ++j) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, j); - if ((pExpr->base.functionId == TSDB_FUNC_PRJ) && pExpr->base.colInfo.colId == pColIndex->colId) { - hasGroupColumn = true; - break; - } - } - - //if the group by column does not required by user, add an invisible column into the final result set. - if (!hasGroupColumn) { - doAddGroupColumnForSubquery(pQueryInfo, i, pCmd); - } - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t doTagFunctionCheck(SQueryInfo* pQueryInfo) { - bool tagProjection = false; - bool tableCounting = false; - - int32_t numOfCols = (int32_t) tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - int32_t functionId = pExpr->base.functionId; - - if (functionId == TSDB_FUNC_TAGPRJ) { - tagProjection = true; - continue; - } - - if (functionId == TSDB_FUNC_COUNT) { - assert(pExpr->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX); - tableCounting = true; - } - } - - return (tableCounting && tagProjection)? -1:0; -} - -int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* msg) { - const char* msg1 = "functions/columns not allowed in group by query"; - const char* msg2 = "projection query on columns not allowed"; - const char* msg3 = "group by/session/state_window not allowed on projection query"; - const char* msg4 = "retrieve tags not compatible with group by or interval query"; - const char* msg5 = "functions can not be mixed up"; - const char* msg6 = "TWA/Diff/Derivative/Irate only support group by tbname"; - - // only retrieve tags, group by is not supportted - if (tscQueryTags(pQueryInfo)) { - if (doTagFunctionCheck(pQueryInfo) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(msg, msg5); - } - - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 || isTimeWindowQuery(pQueryInfo)) { - return invalidOperationMsg(msg, msg4); - } else { - return TSDB_CODE_SUCCESS; - } - } - if (tscIsProjectionQuery(pQueryInfo) && tscIsSessionWindowQuery(pQueryInfo)) { - return invalidOperationMsg(msg, msg3); - } - - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { - // check if all the tags prj columns belongs to the group by columns - if (onlyTagPrjFunction(pQueryInfo) && allTagPrjInGroupby(pQueryInfo)) { - // It is a groupby aggregate query, the tag project function is not suitable for this case. - updateTagPrjFunction(pQueryInfo); - - return doAddGroupbyColumnsOnDemand(pCmd, pQueryInfo); - } - - // check all query functions in selection clause, multi-output functions are not allowed - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - int32_t f = pExpr->base.functionId; - - /* - * group by normal columns. - * Check if the column projection is identical to the group by column or not - */ - if (f == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - bool qualified = false; - for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); - if (pColIndex->colId == pExpr->base.colInfo.colId) { - qualified = true; - break; - } - } - - if (!qualified) { - return invalidOperationMsg(msg, msg2); - } - } - - if (f < 0) { - continue; - } - - if ((!pQueryInfo->stateWindow) && (f == TSDB_FUNC_DIFF || f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE)) { - for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); - if (j == 0) { - if (pColIndex->colIndex != TSDB_TBNAME_COLUMN_INDEX) { - return invalidOperationMsg(msg, msg6); - } - } else if (!TSDB_COL_IS_TAG(pColIndex->flag)) { - return invalidOperationMsg(msg, msg6); - } - } - } - - if (IS_MULTIOUTPUT(aAggs[f].status) && f != TSDB_FUNC_TOP && f != TSDB_FUNC_BOTTOM && f != TSDB_FUNC_DIFF && - f != TSDB_FUNC_DERIVATIVE && f != TSDB_FUNC_TAGPRJ && f != TSDB_FUNC_PRJ) { - return invalidOperationMsg(msg, msg1); - } - - if (f == TSDB_FUNC_COUNT && pExpr->base.colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) { - return invalidOperationMsg(msg, msg1); - } - } - - if (checkUpdateTagPrjFunctions(pQueryInfo, msg) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (doAddGroupbyColumnsOnDemand(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // projection query on super table does not compatible with "group by" syntax - if (tscIsProjectionQuery(pQueryInfo) && !(tscIsDiffDerivQuery(pQueryInfo))) { - return invalidOperationMsg(msg, msg3); - } - - return TSDB_CODE_SUCCESS; - } else { - return checkUpdateTagPrjFunctions(pQueryInfo, msg); - } -} - -int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode) { - const char* msg1 = "only one expression allowed"; - const char* msg2 = "invalid expression in select clause"; - const char* msg3 = "invalid function"; - - SArray* pExprList = pSqlNode->pSelNodeList; - size_t size = taosArrayGetSize(pExprList); - if (size != 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - bool server_status = false; - tSqlExprItem* pExprItem = taosArrayGet(pExprList, 0); - tSqlExpr* pExpr = pExprItem->pNode; - if (pExpr->Expr.operand.z == NULL) { - //handle 'select 1' - if (pExpr->exprToken.n == 1 && 0 == strncasecmp(pExpr->exprToken.z, "1", 1)) { - server_status = true; - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } - // TODO redefine the function - SDNodeDynConfOption functionsInfo[5] = {{"database()", 10}, - {"server_version()", 16}, - {"server_status()", 15}, - {"client_version()", 16}, - {"current_user()", 14}}; - - int32_t index = -1; - if (server_status == true) { - index = 2; - } else { - for (int32_t i = 0; i < tListLen(functionsInfo); ++i) { - if (strncasecmp(functionsInfo[i].name, pExpr->exprToken.z, functionsInfo[i].len) == 0 && - functionsInfo[i].len == pExpr->exprToken.n) { - index = i; - break; - } - } - } - - switch (index) { - case 0: - pQueryInfo->command = TSDB_SQL_CURRENT_DB;break; - case 1: - pQueryInfo->command = TSDB_SQL_SERV_VERSION;break; - case 2: - pQueryInfo->command = TSDB_SQL_SERV_STATUS;break; - case 3: - pQueryInfo->command = TSDB_SQL_CLI_VERSION;break; - case 4: - pQueryInfo->command = TSDB_SQL_CURRENT_USER;break; - default: { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - } - - SColumnIndex ind = {0}; - SExprInfo* pExpr1 = tscExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT, - tDataTypes[TSDB_DATA_TYPE_INT].bytes, getNewResColId(pCmd), tDataTypes[TSDB_DATA_TYPE_INT].bytes, false); - - tSqlExprItem* item = taosArrayGet(pExprList, 0); - const char* name = (item->aliasName != NULL)? item->aliasName:functionsInfo[index].name; - tstrncpy(pExpr1->base.aliasName, name, tListLen(pExpr1->base.aliasName)); - - return TSDB_CODE_SUCCESS; -} - -// can only perform the parameters based on the macro definitation -int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) { - char msg[512] = {0}; - - if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) { - snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 1-2 allowed", pCreate->walLevel); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - if (pCreate->replications != -1 && - (pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) { - snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, - TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - int32_t blocks = ntohl(pCreate->totalBlocks); - if (blocks != -1 && (blocks < TSDB_MIN_TOTAL_BLOCKS || blocks > TSDB_MAX_TOTAL_BLOCKS)) { - snprintf(msg, tListLen(msg), "invalid db option totalBlocks: %d valid range: [%d, %d]", blocks, - TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - if (pCreate->quorum != -1 && - (pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) { - snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum, - TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - int32_t val = htonl(pCreate->daysPerFile); - if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { - snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, - TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - val = htonl(pCreate->cacheBlockSize); - if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) { - snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val, - TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - val = htonl(pCreate->maxTables); - if (val != -1 && (val < TSDB_MIN_TABLES || val > TSDB_MAX_TABLES)) { - snprintf(msg, tListLen(msg), "invalid db option maxSessions: %d valid range: [%d, %d]", val, - TSDB_MIN_TABLES, TSDB_MAX_TABLES); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO && - pCreate->precision != TSDB_TIME_PRECISION_NANO) { - snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d, %d]", pCreate->precision, - TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - val = htonl(pCreate->commitTime); - if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { - snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, - TSDB_MIN_COMMIT_TIME, TSDB_MAX_COMMIT_TIME); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - val = htonl(pCreate->fsyncPeriod); - if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) { - snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val, - TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - if (pCreate->compression != -1 && - (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { - snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, - TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - val = (int16_t)htons(pCreate->partitions); - if (val != -1 && - (val < TSDB_MIN_DB_PARTITON_OPTION || val > TSDB_MAX_DB_PARTITON_OPTION)) { - snprintf(msg, tListLen(msg), "invalid topic option partition: %d valid range: [%d, %d]", val, - TSDB_MIN_DB_PARTITON_OPTION, TSDB_MAX_DB_PARTITON_OPTION); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - - return TSDB_CODE_SUCCESS; -} - -// for debug purpose -void tscPrintSelNodeList(SSqlObj* pSql, int32_t subClauseIndex) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - int32_t size = (int32_t)tscNumOfExprs(pQueryInfo); - if (size == 0) { - return; - } - - int32_t totalBufSize = 1024; - - char str[1024+1] = {0}; - int32_t offset = 0; - - offset += sprintf(str, "num:%d [", size); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - - char tmpBuf[1024] = {0}; - int32_t tmpLen = 0; - char *name = NULL; - - if (pExpr->base.functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * pExpr->base.functionId - 1); - name = pUdfInfo->name; - } else { - name = aAggs[pExpr->base.functionId].name; - } - - tmpLen = - sprintf(tmpBuf, "%s(uid:%" PRIu64 ", %d)", name, pExpr->base.uid, pExpr->base.colInfo.colId); - - if (tmpLen + offset >= totalBufSize - 1) break; - - - offset += sprintf(str + offset, "%s", tmpBuf); - - if (i < size - 1) { - str[offset++] = ','; - } - } - - assert(offset < totalBufSize); - str[offset] = ']'; - assert(offset < totalBufSize); - tscDebug("0x%"PRIx64" select clause:%s", pSql->self, str); -} - -int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* pInfo) { - const char* msg1 = "invalid table name"; - - SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; - - SArray* pFieldList = pCreateTable->colInfo.pColumns; - SArray* pTagList = pCreateTable->colInfo.pTagColumns; - - assert(pFieldList != NULL); - - // if sql specifies db, use it, otherwise use default db - SStrToken* pzTableName = &(pCreateTable->name); - - if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - int32_t code = tscSetTableFullName(&pTableMetaInfo->name, pzTableName, pSql); - if(code != TSDB_CODE_SUCCESS) { - return code; - } - - if (!validateTableColumnInfo(pFieldList, pCmd) || - (pTagList != NULL && !validateTagParams(pTagList, pFieldList, pCmd))) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int32_t col = 0; - size_t numOfFields = taosArrayGetSize(pFieldList); - - for (; col < numOfFields; ++col) { - TAOS_FIELD* p = taosArrayGet(pFieldList, col); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); - } - - pCmd->numOfCols = (int16_t)numOfFields; - - if (pTagList != NULL) { // create super table[optional] - size_t numOfTags = taosArrayGetSize(pTagList); - for (int32_t i = 0; i < numOfTags; ++i) { - TAOS_FIELD* p = taosArrayGet(pTagList, i); - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); - } - - pCmd->count =(int32_t) numOfTags; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { - const char* msg1 = "invalid table name"; - const char* msg3 = "tag value too long"; - const char* msg4 = "illegal value or data overflow"; - const char* msg5 = "tags number not matched"; - - SSqlCmd* pCmd = &pSql->cmd; - - SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - // two table: the first one is for current table, and the secondary is for the super table. - if (pQueryInfo->numOfTables < 2) { - tscAddEmptyMetaInfo(pQueryInfo); - } - - const int32_t TABLE_INDEX = 0; - const int32_t STABLE_INDEX = 1; - - STableMetaInfo* pStableMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); - - // super table name, create table by using dst - int32_t numOfTables = (int32_t) taosArrayGetSize(pCreateTable->childTableInfo); - for(int32_t j = 0; j < numOfTables; ++j) { - SCreatedTableInfo* pCreateTableInfo = taosArrayGet(pCreateTable->childTableInfo, j); - - SStrToken* pToken = &pCreateTableInfo->stableName; - if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - int32_t code = tscSetTableFullName(&pStableMetaInfo->name, pToken, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // get table meta from mnode - code = tNameExtractFullName(&pStableMetaInfo->name, pCreateTableInfo->tagdata.name); - - SArray* pValList = pCreateTableInfo->pTagVals; - code = tscGetTableMeta(pSql, pStableMetaInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - size_t valSize = taosArrayGetSize(pValList); - - // too long tag values will return invalid sql, not be truncated automatically - SSchema *pTagSchema = tscGetTableTagSchema(pStableMetaInfo->pTableMeta); - STableComInfo tinfo = tscGetTableInfo(pStableMetaInfo->pTableMeta); - STagData *pTag = &pCreateTableInfo->tagdata; - - SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SArray* pNameList = NULL; - size_t nameSize = 0; - int32_t schemaSize = tscGetNumOfTags(pStableMetaInfo->pTableMeta); - int32_t ret = TSDB_CODE_SUCCESS; - - if (pCreateTableInfo->pTagNames) { - pNameList = pCreateTableInfo->pTagNames; - nameSize = taosArrayGetSize(pNameList); - - if (valSize != nameSize) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - if (schemaSize < valSize) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - bool findColumnIndex = false; - - for (int32_t i = 0; i < nameSize; ++i) { - SStrToken* sToken = taosArrayGet(pNameList, i); - if (TK_STRING == sToken->type) { - tscDequoteAndTrimToken(sToken); - } - - tVariantListItem* pItem = taosArrayGet(pValList, i); - - findColumnIndex = false; - - // todo speedup by using hash list - for (int32_t t = 0; t < schemaSize; ++t) { - if (strncmp(sToken->z, pTagSchema[t].name, sToken->n) == 0 && strlen(pTagSchema[t].name) == sToken->n) { - SSchema* pSchema = &pTagSchema[t]; - - char tagVal[TSDB_MAX_TAGS_LEN] = {0}; - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - if (pItem->pVar.nLen > pSchema->bytes) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } else if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { - if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) { - ret = convertTimestampStrToInt64(&(pItem->pVar), tinfo.precision); - if (ret != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - } else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) { - pItem->pVar.i64 = convertTimePrecision(pItem->pVar.i64, TSDB_TIME_PRECISION_NANO, tinfo.precision); - } - } - - ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); - - // check again after the convert since it may be converted from binary to nchar. - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - int16_t len = varDataTLen(tagVal); - if (len > pSchema->bytes) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - - if (ret != TSDB_CODE_SUCCESS) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); - - findColumnIndex = true; - break; - } - } - - if (!findColumnIndex) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return tscInvalidOperationMsg(pCmd->payload, "invalid tag name", sToken->z); - } - } - } else { - if (schemaSize != valSize) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - for (int32_t i = 0; i < valSize; ++i) { - SSchema* pSchema = &pTagSchema[i]; - tVariantListItem* pItem = taosArrayGet(pValList, i); - - char tagVal[TSDB_MAX_TAGS_LEN]; - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - if (pItem->pVar.nLen > pSchema->bytes) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } else if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { - if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) { - ret = convertTimestampStrToInt64(&(pItem->pVar), tinfo.precision); - if (ret != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - } else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) { - pItem->pVar.i64 = convertTimePrecision(pItem->pVar.i64, TSDB_TIME_PRECISION_NANO, tinfo.precision); - } - } - - - ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); - - // check again after the convert since it may be converted from binary to nchar. - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - int16_t len = varDataTLen(tagVal); - if (len > pSchema->bytes) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - - if (ret != TSDB_CODE_SUCCESS) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); - } - } - - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - tdDestroyKVRowBuilder(&kvRowBuilder); - if (row == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - tdSortKVRowByColIdx(row); - pTag->dataLen = kvRowLen(row); - - if (pTag->data == NULL) { - pTag->data = malloc(pTag->dataLen); - } - - kvRowCpy(pTag->data, row); - free(row); - - // table name - if (tscValidateName(&(pCreateTableInfo->name)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); - ret = tscSetTableFullName(&pTableMetaInfo->name, &pCreateTableInfo->name, pSql); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - pCreateTableInfo->fullname = calloc(1, tNameLen(&pTableMetaInfo->name) + 1); - ret = tNameExtractFullName(&pTableMetaInfo->name, pCreateTableInfo->fullname); - if (ret != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { - const char* msg1 = "invalid table name"; - const char* msg2 = "functions not allowed in CQ"; - const char* msg3 = "fill only available for interval query"; - const char* msg4 = "fill option not supported in stream computing"; - const char* msg5 = "sql too long"; // todo ADD support - const char* msg6 = "from missing in subclause"; - const char* msg7 = "time interval is required"; - const char* msg8 = "the first column should be primary timestamp column"; - const char* msg9 = "Continuous query do not support sub query"; - const char* msg10 = "illegal number of columns"; - - SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - assert(pQueryInfo->numOfTables == 1); - - SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - // if sql specifies db, use it, otherwise use default db - SStrToken* pName = &(pCreateTable->name); - SSqlNode* pSqlNode = pCreateTable->pSelect; - - if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - SRelationInfo* pFromInfo = pInfo->pCreateTableInfo->pSelect->from; - if (pFromInfo == NULL || taosArrayGetSize(pFromInfo->list) == 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - if (pFromInfo->type == SQL_FROM_NODE_SUBQUERY){ - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - - SRelElement* p1 = taosArrayGet(pFromInfo->list, 0); - SStrToken srcToken = {.z = p1->tableName.z, .n = p1->tableName.n, .type = TK_STRING}; - if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - int32_t code = tscSetTableFullName(&pTableMetaInfo->name, &srcToken, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = tscGetTableMeta(pSql, pTableMetaInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (validateSelectNodeList(&pSql->cmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, false) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pSqlNode->pWhere != NULL) { // query condition in stream computing - if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - // set interval value - if (validateIntervalNode(pSql, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (isTimeWindowQuery(pQueryInfo) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - // project query primary column must be timestamp type - if (tscIsProjectionQuery(pQueryInfo)) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); - if (pExpr->base.colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); - } - } else { - if (pQueryInfo->interval.interval == 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - } - - // set the created table[stream] name - code = tscSetTableFullName(&pTableMetaInfo->name, pName, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (pSqlNode->sqlstr.n > TSDB_MAX_SAVED_SQL_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - - if (tsRewriteFieldNameIfNecessary(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pQueryInfo->fieldsInfo.numOfOutput <= 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg10); - } - - if (validateSqlFunctionInStreamSql(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - /* - * check if fill operation is available, the fill operation is parsed and executed during query execution, - * not here. - */ - if (pSqlNode->fillType != NULL) { - if (pQueryInfo->interval.interval == 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - tVariantListItem* pItem = taosArrayGet(pSqlNode->fillType, 0); - if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) { - if (!((strncmp(pItem->pVar.pz, "none", 4) == 0 && pItem->pVar.nLen == 4) || - (strncmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4))) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - } - } - - // set the number of stream table columns - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - return TSDB_CODE_SUCCESS; -} - -int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char* msg3 = "start(end) time of query range required or time range too large"; - - if (pQueryInfo->interval.interval == 0) { - return TSDB_CODE_SUCCESS; - } - - bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER); - if (initialWindows) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - int64_t timeRange = TABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); - - int64_t intervalRange = 0; - if (pQueryInfo->interval.intervalUnit == 'n' || pQueryInfo->interval.intervalUnit == 'y') { - int64_t f = 1; - if (pQueryInfo->interval.intervalUnit == 'n') { - f = 30L * MILLISECOND_PER_DAY; - } else if (pQueryInfo->interval.intervalUnit == 'y') { - f = 365L * MILLISECOND_PER_DAY; - } - - intervalRange = pQueryInfo->interval.interval * f; - } else { - intervalRange = pQueryInfo->interval.interval; - } - // number of result is not greater than 10,000,000 - if ((timeRange == 0) || (timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - return TSDB_CODE_SUCCESS; -} - -// TODO normalize the function expression and compare it -int32_t tscGetExprFilters(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNodeList, tSqlExpr* pSqlExpr, SExprInfo** pExpr) { - const char* msg1 = "invalid sql expression in having"; - - *pExpr = NULL; - size_t nx = tscNumOfExprs(pQueryInfo); - - // parameters is needed for functions - if (pSqlExpr->Expr.paramList == NULL && pSqlExpr->functionId != TSDB_FUNC_COUNT) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - tSqlExprItem *pParam = NULL; - SSchema schema = {0}; - - if (pSqlExpr->Expr.paramList != NULL) { - pParam = taosArrayGet(pSqlExpr->Expr.paramList, 0); - SStrToken* pToken = &pParam->pNode->columnName; - - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - getColumnIndexByName(pToken, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - schema = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - } else { - schema = (SSchema) {.colId = PRIMARYKEY_TIMESTAMP_COL_INDEX, .type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = TSDB_KEYSIZE}; - } - - for(int32_t i = 0; i < nx; ++i) { - SExprInfo* pExprInfo = tscExprGet(pQueryInfo, i); - if (pExprInfo->base.functionId == pSqlExpr->functionId && pExprInfo->base.colInfo.colId == schema.colId) { - ++pQueryInfo->havingFieldNum; - *pExpr = pExprInfo; - return TSDB_CODE_SUCCESS; - } - } - -// size_t num = taosArrayGetSize(pSelectNodeList); -// for(int32_t i = 0; i < num; ++i) { -// tSqlExprItem* pItem = taosArrayGet(pSelectNodeList, i); -// -// if (tSqlExprCompare(pItem->pNode, pSqlExpr) == 0) { // exists, not added it, -// -// SColumnIndex index = COLUMN_INDEX_INITIALIZER; -// int32_t functionId = pSqlExpr->functionId; -// if (pSqlExpr->Expr.paramList == NULL) { -// index.columnIndex = 0; -// index.tableIndex = 0; -// } else { -// tSqlExprItem* pParamElem = taosArrayGet(pSqlExpr->Expr.paramList, 0); -// SStrToken* pToken = &pParamElem->pNode->columnName; -// getColumnIndexByName(pToken, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)); -// } -// -// size_t numOfNodeInSel = tscNumOfExprs(pQueryInfo); -// for(int32_t k = 0; k < numOfNodeInSel; ++k) { -// SExprInfo* pExpr1 = tscExprGet(pQueryInfo, k); -// -// if (pExpr1->base.functionId != functionId) { -// continue; -// } -// -// if (pExpr1->base.colInfo.colIndex != index.columnIndex) { -// continue; -// } -// -// ++pQueryInfo->havingFieldNum; -// *pExpr = pExpr1; -// break; -// } -// -// assert(*pExpr != NULL); -// return TSDB_CODE_SUCCESS; -// } -// } - - tSqlExprItem item = {.pNode = pSqlExpr, .aliasName = NULL, .distinct = false}; - - int32_t outputIndex = (int32_t)tscNumOfExprs(pQueryInfo); - - // ADD TRUE FOR TEST - if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, true, NULL) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - ++pQueryInfo->havingFieldNum; - - size_t n = tscNumOfExprs(pQueryInfo); - *pExpr = tscExprGet(pQueryInfo, (int32_t)n - 1); - - SInternalField* pField = taosArrayGet(pQueryInfo->fieldsInfo.internalField, n - 1); - pField->visible = false; - - return TSDB_CODE_SUCCESS; -} - -static int32_t handleExprInHavingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNodeList, tSqlExpr* pExpr, int32_t sqlOptr) { - const char* msg1 = "non binary column not support like/match operator"; - const char* msg2 = "invalid operator for binary column in having clause"; - const char* msg3 = "invalid operator for bool column in having clause"; - - SColumnFilterInfo* pColFilter = NULL; - // TODO refactor: validate the expression - /* - * in case of TK_AND filter condition, we first find the corresponding column and build the query condition together - * the already existed condition. - */ - SExprInfo *expr = NULL; - if (sqlOptr == TK_AND) { - int32_t ret = tscGetExprFilters(pCmd, pQueryInfo, pSelectNodeList, pExpr->pLeft, &expr); - if (ret) { - return ret; - } - - // this is a new filter condition on this column - if (expr->base.flist.numOfFilters == 0) { - pColFilter = addColumnFilterInfo(&expr->base.flist); - } else { // update the existed column filter information, find the filter info here - pColFilter = &expr->base.flist.filterInfo[0]; - } - - if (pColFilter == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } else if (sqlOptr == TK_OR) { - int32_t ret = tscGetExprFilters(pCmd, pQueryInfo, pSelectNodeList, pExpr->pLeft, &expr); - if (ret) { - return ret; - } - - // TODO fixme: failed to invalid the filter expression: "col1 = 1 OR col2 = 2" - // TODO refactor - pColFilter = addColumnFilterInfo(&expr->base.flist); - if (pColFilter == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } else { // error; - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - pColFilter->filterstr = - ((expr->base.resType == TSDB_DATA_TYPE_BINARY || expr->base.resType == TSDB_DATA_TYPE_NCHAR) ? 1 : 0); - - if (pColFilter->filterstr) { - if (pExpr->tokenId != TK_EQ - && pExpr->tokenId != TK_NE - && pExpr->tokenId != TK_ISNULL - && pExpr->tokenId != TK_NOTNULL - && pExpr->tokenId != TK_LIKE - && pExpr->tokenId != TK_MATCH - && pExpr->tokenId != TK_NMATCH - ) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } else { - if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_NMATCH) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (expr->base.resType == TSDB_DATA_TYPE_BOOL) { - if (pExpr->tokenId != TK_EQ && pExpr->tokenId != TK_NE) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - int32_t ret = doExtractColumnFilterInfo(pCmd, pQueryInfo, pTableMeta->tableInfo.precision, pColFilter, - expr->base.resType, pExpr); - if (ret) { - return ret; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t getHavingExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNodeList, tSqlExpr* pExpr, int32_t parentOptr) { - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - const char* msg1 = "invalid having clause"; - - tSqlExpr* pLeft = pExpr->pLeft; - tSqlExpr* pRight = pExpr->pRight; - - if (pExpr->tokenId == TK_AND || pExpr->tokenId == TK_OR) { - int32_t ret = getHavingExpr(pCmd, pQueryInfo, pSelectNodeList, pExpr->pLeft, pExpr->tokenId); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - return getHavingExpr(pCmd, pQueryInfo, pSelectNodeList, pExpr->pRight, pExpr->tokenId); - } - - if (pLeft == NULL || pRight == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pLeft->type == pRight->type) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - exchangeExpr(pExpr); - - pLeft = pExpr->pLeft; - pRight = pExpr->pRight; - if (pLeft->type != SQL_NODE_SQLFUNCTION) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pRight->type != SQL_NODE_VALUE) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pExpr->tokenId >= TK_BITAND) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pLeft->Expr.paramList) { - size_t size = taosArrayGetSize(pLeft->Expr.paramList); - for (int32_t i = 0; i < size; i++) { - tSqlExprItem* pParamItem = taosArrayGet(pLeft->Expr.paramList, i); - - tSqlExpr* pExpr1 = pParamItem->pNode; - if (pExpr1->tokenId != TK_ALL && - pExpr1->tokenId != TK_ID && - pExpr1->tokenId != TK_STRING && - pExpr1->tokenId != TK_INTEGER && - pExpr1->tokenId != TK_FLOAT) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pExpr1->tokenId == TK_ID && (pExpr1->columnName.z == NULL && pExpr1->columnName.n == 0)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pExpr1->tokenId == TK_ID) { - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if ((getColumnIndexByName(&pExpr1->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - if (index.columnIndex <= 0 || - index.columnIndex >= tscGetNumOfColumns(pTableMeta)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - } - } - } - - pLeft->functionId = isValidFunction(pLeft->Expr.operand.z, pLeft->Expr.operand.n); - if (pLeft->functionId < 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - return handleExprInHavingClause(pCmd, pQueryInfo, pSelectNodeList, pExpr, parentOptr); -} - -int32_t validateHavingClause(SQueryInfo* pQueryInfo, tSqlExpr* pExpr, SSqlCmd* pCmd, SArray* pSelectNodeList, - int32_t joinQuery, int32_t timeWindowQuery) { - const char* msg1 = "having only works with group by"; - const char* msg2 = "functions or others can not be mixed up"; - const char* msg3 = "invalid expression in having clause"; - - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - if (pQueryInfo->groupbyExpr.numOfGroupCols <= 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - if (pExpr->pLeft == NULL || pExpr->pRight == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - if (pQueryInfo->colList == NULL) { - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - } - - int32_t ret = 0; - - if ((ret = getHavingExpr(pCmd, pQueryInfo, pSelectNodeList, pExpr, TK_AND)) != TSDB_CODE_SUCCESS) { - return ret; - } - - //REDO function check - if (!functionCompatibleCheck(pQueryInfo, joinQuery, timeWindowQuery)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, char* msgBuf, SSqlObj* pSql) { - const char* msg1 = "invalid table name"; - - int32_t numOfTables = (int32_t) taosArrayGetSize(pSqlNode->from->list); - assert(pSqlNode->from->type == SQL_FROM_NODE_TABLES); - - for(int32_t j = 0; j < numOfTables; ++j) { - SRelElement* item = taosArrayGet(pSqlNode->from->list, j); - - SStrToken* t = &item->tableName; - if (t->type == TK_INTEGER || t->type == TK_FLOAT) { - return invalidOperationMsg(msgBuf, msg1); - } - - tscDequoteAndTrimToken(t); - if (tscValidateName(t) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(msgBuf, msg1); - } - - SName name = {0}; - int32_t code = tscSetTableFullName(&name, t, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - taosArrayPush(tableNameList, &name); - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameList, char* msgBuf, SSqlObj* pSql) { - int32_t numOfSub = (int32_t) taosArrayGetSize(pSqlNode->from->list); - - for(int32_t j = 0; j < numOfSub; ++j) { - SRelElement* sub = taosArrayGet(pSqlNode->from->list, j); - - int32_t num = (int32_t)taosArrayGetSize(sub->pSubquery); - for (int32_t i = 0; i < num; ++i) { - SSqlNode* p = taosArrayGetP(sub->pSubquery, i); - if (p->from->type == SQL_FROM_NODE_TABLES) { - int32_t code = getTableNameFromSqlNode(p, tableNameList, msgBuf, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else { - getTableNameFromSubquery(p, tableNameList, msgBuf, pSql); - } - } - } - - return TSDB_CODE_SUCCESS; -} - -void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); -static void freeElem(void* p) { - tfree(*(char**)p); -} - -int32_t tnameComparFn(const void* p1, const void* p2) { - SName* pn1 = (SName*)p1; - SName* pn2 = (SName*)p2; - - int32_t ret = strncmp(pn1->acctId, pn2->acctId, tListLen(pn1->acctId)); - if (ret != 0) { - return ret > 0? 1:-1; - } else { - ret = strncmp(pn1->dbname, pn2->dbname, tListLen(pn1->dbname)); - if (ret != 0) { - return ret > 0? 1:-1; - } else { - ret = strncmp(pn1->tname, pn2->tname, tListLen(pn1->tname)); - if (ret != 0) { - return ret > 0? 1:-1; - } else { - return 0; - } - } - } -} - -int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { - SSqlCmd* pCmd = &pSql->cmd; - - // the table meta has already been loaded from local buffer or mnode already - if (pCmd->pTableMetaMap != NULL) { - return TSDB_CODE_SUCCESS; - } - - int32_t code = TSDB_CODE_SUCCESS; - - SArray* tableNameList = NULL; - SArray* pVgroupList = NULL; - SArray* plist = NULL; - STableMeta* pTableMeta = NULL; - size_t tableMetaCapacity = 0; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - pCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - - tableNameList = taosArrayInit(4, sizeof(SName)); - size_t size = taosArrayGetSize(pInfo->list); - for (int32_t i = 0; i < size; ++i) { - SSqlNode* pSqlNode = taosArrayGetP(pInfo->list, i); - if (pSqlNode->from == NULL) { - goto _end; - } - - // load the table meta in the from clause - if (pSqlNode->from->type == SQL_FROM_NODE_TABLES) { - code = getTableNameFromSqlNode(pSqlNode, tableNameList, tscGetErrorMsgPayload(pCmd), pSql); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - } else { - code = getTableNameFromSubquery(pSqlNode, tableNameList, tscGetErrorMsgPayload(pCmd), pSql); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - } - } - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - - plist = taosArrayInit(4, POINTER_BYTES); - pVgroupList = taosArrayInit(4, POINTER_BYTES); - - taosArraySort(tableNameList, tnameComparFn); - taosArrayRemoveDuplicate(tableNameList, tnameComparFn, NULL); - - STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf); - size_t numOfTables = taosArrayGetSize(tableNameList); - for (int32_t i = 0; i < numOfTables; ++i) { - SName* pname = taosArrayGet(tableNameList, i); - tNameExtractFullName(pname, name); - - size_t len = strlen(name); - - if (NULL == taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&pTableMeta, &tableMetaCapacity)) { - // not found - tfree(pTableMeta); - } - - if (pTableMeta && pTableMeta->id.uid > 0) { - tscDebug("0x%"PRIx64" retrieve table meta %s from local buf", pSql->self, name); - - // avoid mem leak, may should update pTableMeta - void* pVgroupIdList = NULL; - if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - code = tscCreateTableMetaFromSTableMeta((STableMeta **)(&pTableMeta), name, &tableMetaCapacity, (STableMeta **)(&pSTMeta)); - pSql->pBuf = (void *)pSTMeta; - - // create the child table meta from super table failed, try load it from mnode - if (code != TSDB_CODE_SUCCESS) { - char* t = strdup(name); - taosArrayPush(plist, &t); - continue; - } - } else if (pTableMeta->tableType == TSDB_SUPER_TABLE) { - // the vgroup list of super table is not kept in local buffer, so here need retrieve it from the mnode each time - tscDebug("0x%"PRIx64" try to acquire cached super table %s vgroup id list", pSql->self, name); - void* pv = taosCacheAcquireByKey(tscVgroupListBuf, name, len); - if (pv == NULL) { - char* t = strdup(name); - taosArrayPush(pVgroupList, &t); - tscDebug("0x%"PRIx64" failed to retrieve stable %s vgroup id list in cache, try fetch from mnode", pSql->self, name); - } else { - tFilePage* pdata = (tFilePage*) pv; - pVgroupIdList = taosArrayInit((size_t) pdata->num, sizeof(int32_t)); - if (pVgroupIdList == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - taosArrayAddBatch(pVgroupIdList, pdata->data, (int32_t) pdata->num); - taosCacheRelease(tscVgroupListBuf, &pv, false); - } - } - - if (taosHashGet(pCmd->pTableMetaMap, name, len) == NULL) { - STableMeta* pMeta = tscTableMetaDup(pTableMeta); - STableMetaVgroupInfo tvi = { .pTableMeta = pMeta, .vgroupIdList = pVgroupIdList}; - taosHashPut(pCmd->pTableMetaMap, name, len, &tvi, sizeof(STableMetaVgroupInfo)); - } - } else { - // Add to the retrieve table meta array list. - // If the tableMeta is missing, the cached vgroup list for the corresponding super table will be ignored. - tscDebug("0x%"PRIx64" failed to retrieve table meta %s from local buf", pSql->self, name); - - char* t = strdup(name); - taosArrayPush(plist, &t); - } - } - - size_t funcSize = 0; - if (pInfo->funcs) { - funcSize = taosArrayGetSize(pInfo->funcs); - } - - if (funcSize > 0) { - for (size_t i = 0; i < funcSize; ++i) { - SStrToken* t = taosArrayGet(pInfo->funcs, i); - if (NULL == t) { - continue; - } - - if (t->n >= TSDB_FUNC_NAME_LEN) { - code = tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), "too long function name", t->z); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - } - - int32_t functionId = isValidFunction(t->z, t->n); - if (functionId < 0) { - struct SUdfInfo info = {0}; - info.name = strndup(t->z, t->n); - if (pQueryInfo->pUdfInfo == NULL) { - pQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(struct SUdfInfo)); - } - - info.functionId = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo) * (-1) - 1;; - taosArrayPush(pQueryInfo->pUdfInfo, &info); - } - } - } - - // load the table meta for a given table name list - if (taosArrayGetSize(plist) > 0 || taosArrayGetSize(pVgroupList) > 0 || (pQueryInfo->pUdfInfo && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0)) { - code = getMultiTableMetaFromMnode(pSql, plist, pVgroupList, pQueryInfo->pUdfInfo, tscTableMetaCallBack, true); - } - -_end: - if (plist != NULL) { - taosArrayDestroyEx(plist, freeElem); - } - - if (pVgroupList != NULL) { - taosArrayDestroyEx(pVgroupList, freeElem); - } - - if (tableNameList != NULL) { - taosArrayDestroy(tableNameList); - } - - tfree(pTableMeta); - - return code; -} - -static int32_t doLoadAllTableMeta(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, int32_t numOfTables) { - const char* msg1 = "invalid table name"; - const char* msg2 = "invalid table alias name"; - const char* msg3 = "alias name too long"; - const char* msg4 = "self join not allowed"; - - int32_t code = TSDB_CODE_SUCCESS; - SSqlCmd* pCmd = &pSql->cmd; - - if (numOfTables > taosHashGetSize(pCmd->pTableMetaMap)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - for (int32_t i = 0; i < numOfTables; ++i) { - if (pQueryInfo->numOfTables <= i) { // more than one table - tscAddEmptyMetaInfo(pQueryInfo); - } - - SRelElement *item = taosArrayGet(pSqlNode->from->list, i); - SStrToken *oriName = &item->tableName; - - if (oriName->type == TK_INTEGER || oriName->type == TK_FLOAT) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - tscDequoteAndTrimToken(oriName); - if (tscValidateName(oriName) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - code = tscSetTableFullName(&pTableMetaInfo->name, oriName, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - SStrToken* aliasName = &item->aliasName; - if (TPARSER_HAS_TOKEN(*aliasName)) { - if (aliasName->type == TK_INTEGER || aliasName->type == TK_FLOAT) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - tscDequoteAndTrimToken(aliasName); - if (tscValidateName(aliasName) != TSDB_CODE_SUCCESS || aliasName->n >= TSDB_TABLE_NAME_LEN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - strncpy(pTableMetaInfo->aliasName, aliasName->z, aliasName->n); - } else { - strncpy(pTableMetaInfo->aliasName, tNameGetTableName(&pTableMetaInfo->name), tListLen(pTableMetaInfo->aliasName)); - } - - char fname[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, fname); - STableMetaVgroupInfo* p = taosHashGet(pCmd->pTableMetaMap, fname, strnlen(fname, TSDB_TABLE_FNAME_LEN)); - - pTableMetaInfo->pTableMeta = tscTableMetaDup(p->pTableMeta); - pTableMetaInfo->tableMetaCapacity = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); - assert(pTableMetaInfo->pTableMeta != NULL); - - if (p->vgroupIdList != NULL) { - size_t s = taosArrayGetSize(p->vgroupIdList); - - size_t vgroupsz = sizeof(SVgroupMsg) * s + sizeof(SVgroupsInfo); - pTableMetaInfo->vgroupList = calloc(1, vgroupsz); - if (pTableMetaInfo->vgroupList == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pTableMetaInfo->vgroupList->numOfVgroups = (int32_t) s; - for(int32_t j = 0; j < s; ++j) { - int32_t* id = taosArrayGet(p->vgroupIdList, j); - - // check if current buffer contains the vgroup info. If not, add it - SNewVgroupInfo existVgroupInfo = {.inUse = -1,}; - taosHashGetClone(tscVgroupMap, id, sizeof(*id), NULL, &existVgroupInfo); - - assert(existVgroupInfo.inUse >= 0); - SVgroupMsg *pVgroup = &pTableMetaInfo->vgroupList->vgroups[j]; - - pVgroup->numOfEps = existVgroupInfo.numOfEps; - pVgroup->vgId = existVgroupInfo.vgId; - memcpy(&pVgroup->epAddr, &existVgroupInfo.ep, sizeof(pVgroup->epAddr)); - } - } - } - - return code; -} - -static STableMeta* extractTempTableMetaFromSubquery(SQueryInfo* pUpstream) { - STableMetaInfo* pUpstreamTableMetaInfo = tscGetMetaInfo(pUpstream, 0); - - int32_t numOfColumns = pUpstream->fieldsInfo.numOfOutput; - STableMeta *meta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * numOfColumns); - meta->tableType = TSDB_TEMP_TABLE; - - STableComInfo *info = &meta->tableInfo; - info->numOfColumns = numOfColumns; - info->precision = pUpstreamTableMetaInfo->pTableMeta->tableInfo.precision; - info->numOfTags = 0; - - int32_t n = 0; - for(int32_t i = 0; i < numOfColumns; ++i) { - SInternalField* pField = tscFieldInfoGetInternalField(&pUpstream->fieldsInfo, i); - if (!pField->visible) { - continue; - } - - meta->schema[n].bytes = pField->field.bytes; - meta->schema[n].type = pField->field.type; - - SExprInfo* pExpr = pField->pExpr; - meta->schema[n].colId = pExpr->base.resColId; - tstrncpy(meta->schema[n].name, pField->pExpr->base.aliasName, TSDB_COL_NAME_LEN); - info->rowSize += meta->schema[n].bytes; - - n += 1; - } - - info->numOfColumns = n; - - return meta; -} - -static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pSql, SQueryInfo* pQueryInfo, char* msgBuf) { - SRelElement* subInfo = taosArrayGet(pSqlNode->from->list, index); - - // union all is not support currently - SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0); - if (taosArrayGetSize(subInfo->pSubquery) >= 2) { - return invalidOperationMsg(msgBuf, "not support union in subquery"); - } - - SQueryInfo* pSub = calloc(1, sizeof(SQueryInfo)); - tscInitQueryInfo(pSub); - - SArray *pUdfInfo = NULL; - if (pQueryInfo->pUdfInfo) { - pUdfInfo = taosArrayDup(pQueryInfo->pUdfInfo); - } - - pSub->pUdfInfo = pUdfInfo; - pSub->udfCopy = true; - - pSub->pDownstream = pQueryInfo; - int32_t code = validateSqlNode(pSql, p, pSub); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // create dummy table meta info - STableMetaInfo* pTableMetaInfo1 = calloc(1, sizeof(STableMetaInfo)); - if (pTableMetaInfo1 == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pTableMetaInfo1->pTableMeta = extractTempTableMetaFromSubquery(pSub); - pTableMetaInfo1->tableMetaCapacity = tscGetTableMetaSize(pTableMetaInfo1->pTableMeta); - - if (subInfo->aliasName.n > 0) { - if (subInfo->aliasName.n >= TSDB_TABLE_FNAME_LEN) { - tfree(pTableMetaInfo1); - return invalidOperationMsg(msgBuf, "subquery alias name too long"); - } - - tstrncpy(pTableMetaInfo1->aliasName, subInfo->aliasName.z, subInfo->aliasName.n + 1); - } - - taosArrayPush(pQueryInfo->pUpstream, &pSub); - - // NOTE: order mix up in subquery not support yet. - pQueryInfo->order = pSub->order; - - STableMetaInfo** tmp = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); - if (tmp == NULL) { - tfree(pTableMetaInfo1); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pQueryInfo->pTableMetaInfo = tmp; - - pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = pTableMetaInfo1; - pQueryInfo->numOfTables += 1; - - // all columns are added into the table column list - STableMeta* pMeta = pTableMetaInfo1->pTableMeta; - int32_t startOffset = (int32_t) taosArrayGetSize(pQueryInfo->colList); - - for(int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) { - tscColumnListInsert(pQueryInfo->colList, i + startOffset, pMeta->id.uid, &pMeta->schema[i]); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInfo) { - assert(pSqlNode != NULL && (pSqlNode->from == NULL || taosArrayGetSize(pSqlNode->from->list) > 0)); - - const char* msg1 = "point interpolation query needs timestamp"; - const char* msg2 = "too many tables in from clause"; - const char* msg3 = "start(end) time of query range required or time range too large"; - const char* msg4 = "interval query not supported, since the result of sub query not include valid timestamp column"; - const char* msg5 = "only tag query not compatible with normal column filter"; - const char* msg6 = "not support stddev/percentile/interp in the outer query yet"; - const char* msg7 = "derivative/twa/irate requires timestamp column exists in subquery"; - const char* msg8 = "condition missing for join query"; - const char* msg9 = "not support 3 level select"; - - int32_t code = TSDB_CODE_SUCCESS; - SSqlCmd* pCmd = &pSql->cmd; - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo == NULL) { - pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); - } - - /* - * handle the sql expression without from subclause - * select server_status(); - * select server_version(); - * select client_version(); - * select database(); - */ - if (pSqlNode->from == NULL) { - assert(pSqlNode->fillType == NULL && pSqlNode->pGroupby == NULL && pSqlNode->pWhere == NULL && - pSqlNode->pSortOrder == NULL); - return doLocalQueryProcess(pCmd, pQueryInfo, pSqlNode); - } - - if (pSqlNode->from->type == SQL_FROM_NODE_SUBQUERY) { - clearAllTableMetaInfo(pQueryInfo, false, pSql->self); - pQueryInfo->numOfTables = 0; - - // parse the subquery in the first place - int32_t numOfSub = (int32_t)taosArrayGetSize(pSqlNode->from->list); - for (int32_t i = 0; i < numOfSub; ++i) { - // check if there is 3 level select - SRelElement* subInfo = taosArrayGet(pSqlNode->from->list, i); - SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0); - if (p->from->type == SQL_FROM_NODE_SUBQUERY) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - - code = doValidateSubquery(pSqlNode, i, pSql, pQueryInfo, tscGetErrorMsgPayload(pCmd)); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - int32_t timeWindowQuery = - (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap)); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TABLE_QUERY); - - // parse the group by clause in the first place - if (validateGroupbyNode(pQueryInfo, pSqlNode->pGroupby, pCmd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - - if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, timeWindowQuery, true) != - TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // todo NOT support yet - for (int32_t i = 0; i < tscNumOfExprs(pQueryInfo); ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - int32_t f = pExpr->base.functionId; - if (f == TSDB_FUNC_STDDEV || f == TSDB_FUNC_PERCT || f == TSDB_FUNC_INTERP) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - if ((timeWindowQuery || pQueryInfo->stateWindow) && f == TSDB_FUNC_LAST) { - pExpr->base.numOfParams = 1; - pExpr->base.param[0].i64 = TSDB_ORDER_ASC; - pExpr->base.param[0].nType = TSDB_DATA_TYPE_INT; - } - } - - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, 0); - - if (pSchema->type != TSDB_DATA_TYPE_TIMESTAMP) { - int32_t numOfExprs = (int32_t)tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - - int32_t f = pExpr->base.functionId; - if (f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - } - } - } - - // validate the query filter condition info - if (pSqlNode->pWhere != NULL) { - if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else { - if (pQueryInfo->numOfTables > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); - } - } - - // validate the interval info - if (validateIntervalNode(pSql, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } else { - if (validateSessionNode(pCmd, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // parse the window_state - if (validateStateWindowNode(pCmd, pQueryInfo, pSqlNode, false) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (isTimeWindowQuery(pQueryInfo)) { - // check if the first column of the nest query result is timestamp column - SColumn* pCol = taosArrayGetP(pQueryInfo->colList, 0); - if (pCol->info.type != TSDB_DATA_TYPE_TIMESTAMP) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - - if (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - } - - // disable group result mixed up if interval/session window query exists. - if (isTimeWindowQuery(pQueryInfo)) { - size_t num = taosArrayGetSize(pQueryInfo->pUpstream); - for(int32_t i = 0; i < num; ++i) { - SQueryInfo* pUp = taosArrayGetP(pQueryInfo->pUpstream, i); - pUp->multigroupResult = false; - } - } - - // parse the having clause in the first place - int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1); - if (validateHavingClause(pQueryInfo, pSqlNode->pHaving, pCmd, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery) != - TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if ((code = validateLimitNode(pCmd, pQueryInfo, pSqlNode, pSql)) != TSDB_CODE_SUCCESS) { - return code; - } - - // set order by info - if (validateOrderbyNode(pCmd, pQueryInfo, pSqlNode, tscGetTableSchema(pTableMeta)) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if ((code = doFunctionsCompatibleCheck(pCmd, pQueryInfo, tscGetErrorMsgPayload(pCmd))) != TSDB_CODE_SUCCESS) { - return code; - } - -// updateFunctionInterBuf(pQueryInfo, false); - updateLastScanOrderIfNeeded(pQueryInfo); - - if ((code = validateFillNode(pCmd, pQueryInfo, pSqlNode)) != TSDB_CODE_SUCCESS) { - return code; - } - } else { - pQueryInfo->command = TSDB_SQL_SELECT; - - size_t numOfTables = taosArrayGetSize(pSqlNode->from->list); - if (numOfTables > TSDB_MAX_JOIN_TABLE_NUM) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - - // set all query tables, which are maybe more than one. - code = doLoadAllTableMeta(pSql, pQueryInfo, pSqlNode, (int32_t) numOfTables); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - - int32_t type = isSTable? TSDB_QUERY_TYPE_STABLE_QUERY:TSDB_QUERY_TYPE_TABLE_QUERY; - TSDB_QUERY_SET_TYPE(pQueryInfo->type, type); - - // parse the group by clause in the first place - if (validateGroupbyNode(pQueryInfo, pSqlNode->pGroupby, pCmd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - pQueryInfo->onlyHasTagCond = true; - // set where info - if (pSqlNode->pWhere != NULL) { - if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - pSqlNode->pWhere = NULL; - } else { - if (taosArrayGetSize(pSqlNode->from->list) > 1) { // Cross join not allowed yet - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "cross join not supported yet"); - } - } - - int32_t joinQuery = (pSqlNode->from != NULL && taosArrayGetSize(pSqlNode->from->list) > 1); - int32_t timeWindowQuery = - (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap)); - - if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery, false) != - TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (isSTable && tscQueryTags(pQueryInfo) && pQueryInfo->distinct && !pQueryInfo->onlyHasTagCond) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // parse the window_state - if (validateStateWindowNode(pCmd, pQueryInfo, pSqlNode, isSTable) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // set order by info - if (validateOrderbyNode(pCmd, pQueryInfo, pSqlNode, tscGetTableSchema(pTableMetaInfo->pTableMeta)) != - TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // set interval value - if (validateIntervalNode(pSql, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (tscQueryTags(pQueryInfo)) { - SExprInfo* pExpr1 = tscExprGet(pQueryInfo, 0); - - if (pExpr1->base.functionId != TSDB_FUNC_TID_TAG) { - if ((pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) || IS_TSWINDOW_SPECIFIED(pQueryInfo->window)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } - } - } - - // parse the having clause in the first place - if (validateHavingClause(pQueryInfo, pSqlNode->pHaving, pCmd, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery) != - TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - /* - * transfer sql functions that need secondary merge into another format - * in dealing with super table queries such as: count/first/last - */ - if (validateSessionNode(pCmd, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (isTimeWindowQuery(pQueryInfo) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (isSTable) { - tscTansformFuncForSTableQuery(pQueryInfo); - if (hasUnsupportFunctionsForSTableQuery(pCmd, pQueryInfo)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - // no result due to invalid query time range - if (pQueryInfo->window.skey > pQueryInfo->window.ekey) { - pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - return TSDB_CODE_SUCCESS; - } - - if (!hasTimestampForPointInterpQuery(pQueryInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - // in case of join query, time range is required. - if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { - uint64_t timeRange = (uint64_t)pQueryInfo->window.ekey - pQueryInfo->window.skey; - if (timeRange == 0 && pQueryInfo->window.skey == 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - } - - if ((code = validateLimitNode(pCmd, pQueryInfo, pSqlNode, pSql)) != TSDB_CODE_SUCCESS) { - return code; - } - - if ((code = doFunctionsCompatibleCheck(pCmd, pQueryInfo,tscGetErrorMsgPayload(pCmd))) != TSDB_CODE_SUCCESS) { - return code; - } - - updateLastScanOrderIfNeeded(pQueryInfo); - tscFieldInfoUpdateOffset(pQueryInfo); -// updateFunctionInterBuf(pQueryInfo, isSTable); - - if ((code = validateFillNode(pCmd, pQueryInfo, pSqlNode)) != TSDB_CODE_SUCCESS) { - return code; - } - } - - { // set the query info - pQueryInfo->projectionQuery = tscIsProjectionQuery(pQueryInfo); - pQueryInfo->hasFilter = tscHasColumnFilter(pQueryInfo); - pQueryInfo->simpleAgg = isSimpleAggregateRv(pQueryInfo); - pQueryInfo->onlyTagQuery = onlyTagPrjFunction(pQueryInfo); - pQueryInfo->groupbyColumn = tscGroupbyColumn(pQueryInfo); - pQueryInfo->arithmeticOnAgg = tsIsArithmeticQueryOnAggResult(pQueryInfo); - pQueryInfo->orderProjectQuery = tscOrderedProjectionQueryOnSTable(pQueryInfo, 0); - - SExprInfo** p = NULL; - int32_t numOfExpr = 0; - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - code = createProjectionExpr(pQueryInfo, pTableMetaInfo, &p, &numOfExpr); - if (pQueryInfo->exprList1 == NULL) { - pQueryInfo->exprList1 = taosArrayInit(4, POINTER_BYTES); - } - - taosArrayAddBatch(pQueryInfo->exprList1, (void*) p, numOfExpr); - tfree(p); - } - -#if 0 - SQueryNode* p = qCreateQueryPlan(pQueryInfo); - char* s = queryPlanToString(p); - printf("%s\n", s); - tfree(s); - qDestroyQueryPlan(p); -#endif - - return TSDB_CODE_SUCCESS; // Does not build query message here -} - -int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, uint64_t *uid) { - tExprNode* pLeft = NULL; - tExprNode* pRight= NULL; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - - if (pSqlExpr->pLeft != NULL) { - int32_t ret = exprTreeFromSqlExpr(pCmd, &pLeft, pSqlExpr->pLeft, pQueryInfo, pCols, uid); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - } - - if (pSqlExpr->pRight != NULL) { - int32_t ret = exprTreeFromSqlExpr(pCmd, &pRight, pSqlExpr->pRight, pQueryInfo, pCols, uid); - if (ret != TSDB_CODE_SUCCESS) { - tExprTreeDestroy(pLeft, NULL); - return ret; - } - } - - if (pSqlExpr->pLeft == NULL && pSqlExpr->pRight == NULL && pSqlExpr->tokenId == 0) { - *pExpr = calloc(1, sizeof(tExprNode)); - return TSDB_CODE_SUCCESS; - } - - if (pSqlExpr->pLeft == NULL) { // it is the leaf node - assert(pSqlExpr->pRight == NULL); - - if (pSqlExpr->type == SQL_NODE_VALUE) { - int32_t ret = TSDB_CODE_SUCCESS; - *pExpr = calloc(1, sizeof(tExprNode)); - (*pExpr)->nodeType = TSQL_NODE_VALUE; - (*pExpr)->pVal = calloc(1, sizeof(tVariant)); - tVariantAssign((*pExpr)->pVal, &pSqlExpr->value); - - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - if (pCols != NULL && taosArrayGetSize(pCols) > 0) { - SColIndex* idx = taosArrayGet(pCols, 0); - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, idx->colIndex); - // convert time by precision - if (pSchema != NULL && TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && TSDB_DATA_TYPE_BINARY == (*pExpr)->pVal->nType) { - ret = setColumnFilterInfoForTimestamp(pCmd, pQueryInfo, (*pExpr)->pVal); - } - } - return ret; - } else if (pSqlExpr->type == SQL_NODE_SQLFUNCTION) { - // arithmetic expression on the results of aggregation functions - *pExpr = calloc(1, sizeof(tExprNode)); - (*pExpr)->nodeType = TSQL_NODE_COL; - (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); - strncpy((*pExpr)->pSchema->name, pSqlExpr->exprToken.z, pSqlExpr->exprToken.n); - - // set the input column data byte and type. - size_t size = taosArrayGetSize(pQueryInfo->exprList); - - for (int32_t i = 0; i < size; ++i) { - SExprInfo* p1 = taosArrayGetP(pQueryInfo->exprList, i); - - if (strcmp((*pExpr)->pSchema->name, p1->base.aliasName) == 0) { - (*pExpr)->pSchema->type = (uint8_t)p1->base.resType; - (*pExpr)->pSchema->bytes = p1->base.resBytes; - (*pExpr)->pSchema->colId = p1->base.resColId; - - if (uid != NULL) { - *uid = p1->base.uid; - } - - break; - } - } - } else if (pSqlExpr->type == SQL_NODE_TABLE_COLUMN) { // column name, normal column arithmetic expression - int32_t ret = getColumnIndexByName(&pSqlExpr->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - pQueryInfo->curTableIdx = index.tableIndex; - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index.tableIndex)->pTableMeta; - int32_t numOfColumns = tscGetNumOfColumns(pTableMeta); - - *pExpr = calloc(1, sizeof(tExprNode)); - (*pExpr)->nodeType = TSQL_NODE_COL; - (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); - *(*pExpr)->pSchema = *pSchema; - - if (pCols != NULL) { // record the involved columns - SColIndex colIndex = {0}; - tstrncpy(colIndex.name, pSchema->name, sizeof(colIndex.name)); - colIndex.colId = pSchema->colId; - colIndex.colIndex = index.columnIndex; - colIndex.flag = (index.columnIndex >= numOfColumns)? 1:0; - - taosArrayPush(pCols, &colIndex); - } - - return TSDB_CODE_SUCCESS; - } else if (pSqlExpr->tokenId == TK_SET) { - int32_t colType = -1; - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, pQueryInfo->curTableIdx)->pTableMeta; - if (pCols != NULL) { - size_t colSize = taosArrayGetSize(pCols); - - if (colSize > 0) { - SColIndex* idx = taosArrayGet(pCols, colSize - 1); - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, idx->colIndex); - if (pSchema != NULL) { - colType = pSchema->type; - } - } - } - tVariant *pVal; - if (colType >= TSDB_DATA_TYPE_TINYINT && colType <= TSDB_DATA_TYPE_BIGINT) { - colType = TSDB_DATA_TYPE_BIGINT; - } else if (colType == TSDB_DATA_TYPE_FLOAT || colType == TSDB_DATA_TYPE_DOUBLE) { - colType = TSDB_DATA_TYPE_DOUBLE; - } - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pQueryInfo->curTableIdx); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - if (serializeExprListToVariant(pSqlExpr->Expr.paramList, &pVal, colType, tinfo.precision) == false) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression"); - } - *pExpr = calloc(1, sizeof(tExprNode)); - (*pExpr)->nodeType = TSQL_NODE_VALUE; - (*pExpr)->pVal = pVal; - } else { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression"); - } - - } else { - *pExpr = (tExprNode *)calloc(1, sizeof(tExprNode)); - (*pExpr)->nodeType = TSQL_NODE_EXPR; - - (*pExpr)->_node.hasPK = false; - (*pExpr)->_node.pLeft = pLeft; - (*pExpr)->_node.pRight = pRight; - - SStrToken t = {.type = pSqlExpr->tokenId}; - (*pExpr)->_node.optr = convertRelationalOperator(&t); - - assert((*pExpr)->_node.optr != 0); - - // check for dividing by 0 - if ((*pExpr)->_node.optr == TSDB_BINARY_OP_DIVIDE) { - if (pRight->nodeType == TSQL_NODE_VALUE) { - if (pRight->pVal->nType == TSDB_DATA_TYPE_INT && pRight->pVal->i64 == 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } else if (pRight->pVal->nType == TSDB_DATA_TYPE_FLOAT && pRight->pVal->dKey == 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - } - - // NOTE: binary|nchar data allows the >|< type filter - if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) { - if (pRight != NULL && pRight->nodeType == TSQL_NODE_VALUE) { - if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL && pLeft->pSchema->type == TSDB_DATA_TYPE_BOOL) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - } - } - - return TSDB_CODE_SUCCESS; -} - -bool hasNormalColumnFilter(SQueryInfo* pQueryInfo) { - size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i); - if (pCol->info.flist.numOfFilters > 0) { - return true; - } - } - - return false; -} - -#if 0 -void normalizeSqlNode(SSqlNode* pSqlNode, const char* dbName) { - assert(pSqlNode != NULL); - - if (pSqlNode->from->type == SQL_FROM_NODE_TABLES) { -// SRelElement *item = taosArrayGet(pSqlNode->from->list, 0); -// item->TableName.name; - } - - // 1. pSqlNode->pSelNodeList - if (pSqlNode->pSelNodeList != NULL && taosArrayGetSize(pSqlNode->pSelNodeList) > 0) { - SArray* pSelNodeList = pSqlNode->pSelNodeList; - size_t numOfExpr = taosArrayGetSize(pSelNodeList); - for (int32_t i = 0; i < numOfExpr; ++i) { - tSqlExprItem* pItem = taosArrayGet(pSelNodeList, i); - int32_t type = pItem->pNode->type; - if (type == SQL_NODE_VALUE || type == SQL_NODE_EXPR) { - continue; - } - - if (type == SQL_NODE_TABLE_COLUMN) { - } - } - } - -// 2. pSqlNode->pWhere -// 3. pSqlNode->pHaving -// 4. pSqlNode->pSortOrder -// pSqlNode->from -} - -#endif - - diff --git a/2.0/src/client/src/tscServer.c b/2.0/src/client/src/tscServer.c deleted file mode 100644 index 41a55869e6ec3a9223ab6849c8324d1aebc23a8a..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscServer.c +++ /dev/null @@ -1,3222 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include "os.h" -#include "qPlan.h" -#include "qTableMeta.h" -#include "tlockfree.h" -#include "tmsgtype.h" -#include "trpc.h" -#include "tscGlobalmerge.h" -#include "tscLog.h" -#include "tscProfile.h" -#include "tscUtil.h" -#include "tsclient.h" -#include "ttimer.h" - -int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; - -int (*tscProcessMsgRsp[TSDB_SQL_MAX])(SSqlObj *pSql); -void tscProcessActivityTimer(void *handle, void *tmrId); -int tscKeepConn[TSDB_SQL_MAX] = {0}; - -TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt); -void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts); -void tscSaveSubscriptionProgress(void* sub); -static int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo); - -static int32_t minMsgSize() { return tsRpcHeadSize + 100; } -static int32_t getWaitingTimeInterval(int32_t count) { - int32_t initial = 100; // 100 ms by default - if (count <= 1) { - return 0; - } - - return initial * ((2u)<<(count - 2)); -} - -static int32_t vgIdCompare(const void *lhs, const void *rhs) { - int32_t left = *(int32_t *)lhs; - int32_t right = *(int32_t *)rhs; - - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; - } -} -static int32_t removeDupVgid(int32_t *src, int32_t sz) { - if (src == NULL || sz <= 0) { - return 0; - } - qsort(src, sz, sizeof(src[0]), vgIdCompare); - - int32_t ret = 1; - for (int i = 1; i < sz; i++) { - if (src[i] != src[i - 1]) { - src[ret++] = src[i]; - } - } - return ret; -} - -static void tscSetDnodeEpSet(SRpcEpSet* pEpSet, SVgroupMsg* pVgroupInfo) { - assert(pEpSet != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); - - // Issue the query to one of the vnode among a vgroup randomly. - // change the inUse property would not affect the isUse attribute of STableMeta - pEpSet->inUse = rand() % pVgroupInfo->numOfEps; - - // apply the FQDN string length check here - bool existed = false; - - pEpSet->numOfEps = pVgroupInfo->numOfEps; - for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - pEpSet->port[i] = pVgroupInfo->epAddr[i].port; - - int32_t len = (int32_t) strnlen(pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); - if (len > 0) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); - existed = true; - } - } - - assert(existed); -} - -static void tscDumpMgmtEpSet(SSqlObj *pSql) { - SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - taosCorBeginRead(&pCorEpSet->version); - pSql->epSet = pCorEpSet->epSet; - taosCorEndRead(&pCorEpSet->version); -} -static void tscEpSetHtons(SRpcEpSet *s) { - for (int32_t i = 0; i < s->numOfEps; i++) { - s->port[i] = htons(s->port[i]); - } -} - -bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { - if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { - return false; - } - - for (int32_t i = 0; i < s1->numOfEps; i++) { - if (s1->port[i] != s2->port[i] - || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) - return false; - } - return true; -} - -void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { - // no need to update if equal - SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - taosCorBeginWrite(&pCorEpSet->version); - pCorEpSet->epSet = *pEpSet; - taosCorEndWrite(&pCorEpSet->version); -} - -static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgroupInfo) { - if (pVgroupInfo == NULL) { return;} - int8_t inUse = pVgroupInfo->inUse; - pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; - pEpSet->numOfEps = pVgroupInfo->numOfEps; - for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->ep[i].fqdn, sizeof(pEpSet->fqdn[i])); - pEpSet->port[i] = pVgroupInfo->ep[i].port; - } -} - -static void tscUpdateVgroupInfo(SSqlObj *pSql, SRpcEpSet *pEpSet) { - SSqlCmd *pCmd = &pSql->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { - return; - } - - int32_t vgId = -1; - if (pTableMetaInfo->pTableMeta->tableType == TSDB_SUPER_TABLE) { - vgId = extractSTableQueryVgroupId(pTableMetaInfo); - } else { - vgId = pTableMetaInfo->pTableMeta->vgId; - } - - assert(vgId > 0); - - SNewVgroupInfo vgroupInfo = {.vgId = -1}; - taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); - assert(vgroupInfo.numOfEps > 0 && vgroupInfo.vgId > 0); - - tscDebug("before: Endpoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); - vgroupInfo.inUse = pEpSet->inUse; - vgroupInfo.numOfEps = pEpSet->numOfEps; - for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { - tstrncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); - vgroupInfo.ep[i].port = pEpSet->port[i]; - } - - tscDebug("after: EndPoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); - taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo)); - - // Update the local cached epSet info cached by SqlObj - int32_t inUse = pSql->epSet.inUse; - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - tscDebug("0x%"PRIx64" update the epSet in SqlObj, in use before:%d, after:%d", pSql->self, inUse, pSql->epSet.inUse); - -} - -int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo) { - assert(pTableMetaInfo != NULL); - - int32_t vgIndex = pTableMetaInfo->vgroupIndex; - int32_t vgId = -1; - - if (pTableMetaInfo->pVgroupTables == NULL) { - SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; - assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); - vgId = pVgroupInfo->vgroups[vgIndex].vgId; - } else { - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - assert(vgIndex >= 0 && vgIndex < numOfVgroups); - - SVgroupTableInfo *pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); - vgId = pTableIdList->vgInfo.vgId; - } - - return vgId; -} - -void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { - STscObj *pObj = (STscObj *)param; - if (pObj == NULL) return; - - if (pObj != pObj->signature) { - tscError("heartbeat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature); - return; - } - - SSqlObj *pSql = tres; - SSqlRes *pRes = &pSql->res; - - if (code == TSDB_CODE_SUCCESS) { - SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp; - SRpcEpSet *epSet = &pRsp->epSet; - if (epSet->numOfEps > 0) { - tscEpSetHtons(epSet); - - //SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - //if (!tscEpSetIsEqual(&pCorEpSet->epSet, epSet)) { - // tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse); - // for (int8_t i = 0; i < epSet->numOfEps; i++) { - // tscTrace("endpoint %d: fqdn=%s, port=%d", i, epSet->fqdn[i], epSet->port[i]); - // } - //} - //concurrency problem, update mgmt epset anyway - tscUpdateMgmtEpSet(pSql, epSet); - } - - pSql->pTscObj->connId = htonl(pRsp->connId); - - if (pRsp->killConnection) { - tscKillConnection(pObj); - return; - } else { - if (pRsp->queryId) { - tscKillQuery(pObj, htonl(pRsp->queryId)); - } - - if (pRsp->streamId) { - tscKillStream(pObj, htonl(pRsp->streamId)); - } - } - - int32_t total = htonl(pRsp->totalDnodes); - int32_t online = htonl(pRsp->onlineDnodes); - assert(online <= total); - - if (online < total) { - tscError("0x%"PRIx64", HB, total dnode:%d, online dnode:%d", pSql->self, total, online); - pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - } - - if (pRes->length == NULL) { - pRes->length = calloc(2, sizeof(int32_t)); - } - - pRes->length[0] = total; - pRes->length[1] = online; - } else { - tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code)); - if (pRes->length == NULL) { - pRes->length = calloc(2, sizeof(int32_t)); - } - - pRes->length[1] = 0; - if (pRes->length[0] == 0) { - pRes->length[0] = 1; // make sure that the value of the total node is greater than the online node - } - } - - if (pObj->hbrid != 0) { - int32_t waitingDuring = tsShellActivityTimer * 500; - tscDebug("0x%"PRIx64" send heartbeat in %dms", pSql->self, waitingDuring); - - taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer); - } else { - tscDebug("0x%"PRIx64" start to close tscObj:%p, not send heartbeat again", pSql->self, pObj); - } -} - -void tscProcessActivityTimer(void *handle, void *tmrId) { - int64_t rid = (int64_t) handle; - STscObj *pObj = taosAcquireRef(tscRefId, rid); - if (pObj == NULL) { - return; - } - - SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid); - if (pHB == NULL) { - taosReleaseRef(tscRefId, rid); - return; - } - - assert(pHB->self == pObj->hbrid); - - pHB->retry = 0; - int32_t code = tscBuildAndSendRequest(pHB, NULL); - taosReleaseRef(tscObjRef, pObj->hbrid); - - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" failed to sent HB to server, reason:%s", pHB->self, tstrerror(code)); - } - - taosReleaseRef(tscRefId, rid); -} - -int tscSendMsgToServer(SSqlObj *pSql) { - STscObj* pObj = pSql->pTscObj; - SSqlCmd* pCmd = &pSql->cmd; - - char *pMsg = rpcMallocCont(pCmd->payloadLen); - if (NULL == pMsg) { - tscError("0x%"PRIx64" msg:%s malloc failed", pSql->self, TMSG_INFO(pSql->cmd.msgType)); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - // set the mgmt ip list - if (pSql->cmd.command >= TSDB_SQL_MGMT) { - tscDumpMgmtEpSet(pSql); - } - - memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); - - SRpcMsg rpcMsg = { - .msgType = pSql->cmd.msgType, - .pCont = pMsg, - .contLen = pSql->cmd.payloadLen, - .ahandle = (void*)pSql->self, - .handle = NULL, - .code = 0 - }; - - rpcSendRequest(pObj->pRpcObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid); - return TSDB_CODE_SUCCESS; -} - -// handle three situation -// 1. epset retry, only return last failure ep -// 2. no epset retry, like 'taos -h invalidFqdn', return invalidFqdn -// 3. other situation, no expected -void tscSetFqdnErrorMsg(SSqlObj* pSql, SRpcEpSet* pEpSet) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - char* msgBuf = tscGetErrorMsgPayload(pCmd); - - if (pEpSet) { - sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),pEpSet->fqdn[(pEpSet->inUse)%(pEpSet->numOfEps)]); - } else if (pCmd->command >= TSDB_SQL_MGMT) { - SRpcEpSet tEpset; - - SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - taosCorBeginRead(&pCorEpSet->version); - tEpset = pCorEpSet->epSet; - taosCorEndRead(&pCorEpSet->version); - - sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),tEpset.fqdn[(tEpset.inUse)%(tEpset.numOfEps)]); - } else { - sprintf(msgBuf, "%s", tstrerror(pRes->code)); - } -} - -void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { - TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle; - SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle); - if (pSql == NULL) { - rpcFreeCont(rpcMsg->pCont); - return; - } - - assert(pSql->self == handle); - - STscObj *pObj = pSql->pTscObj; - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - pSql->rpcRid = -1; - if (pObj->signature != pObj) { - tscDebug("0x%"PRIx64" DB connection is closed, cmd:%d pObj:%p signature:%p", pSql->self, pCmd->command, pObj, pObj->signature); - - taosRemoveRef(tscObjRef, handle); - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); - return; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) { - tscDebug("0x%"PRIx64" sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", - pSql->self, pCmd->command, pQueryInfo->type, pObj, pObj->signature); - - taosRemoveRef(tscObjRef, handle); - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); - return; - } - - if (pEpSet) { - if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { - if (pCmd->command < TSDB_SQL_MGMT) { - tscUpdateVgroupInfo(pSql, pEpSet); - } else { - tscUpdateMgmtEpSet(pSql, pEpSet); - } - } - } - - int32_t cmd = pCmd->command; - - // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema - if (cmd == TSDB_SQL_INSERT && rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - pSql->cmd.insertParam.schemaAttached = 1; - } - - // single table query error need to be handled here. - if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && - (((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) || - rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { - - // 1. super table subquery - // 2. nest queries are all not updated the tablemeta and retry parse the sql after cleanup local tablemeta/vgroup id buffer - if ((TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY | - TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) || - (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY)) || (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->distinct)) { - // do nothing in case of super table subquery - } else { - pSql->retry += 1; - tscWarn("0x%" PRIx64 " it shall renew table meta, code:%s, retry:%d", pSql->self, tstrerror(rpcMsg->code), pSql->retry); - - pSql->res.code = rpcMsg->code; // keep the previous error code - if (pSql->retry > pSql->maxRetry) { - tscError("0x%" PRIx64 " max retry %d reached, give up", pSql->self, pSql->maxRetry); - } else { - // wait for a little bit moment and then retry - // todo do not sleep in rpc callback thread, add this process into queue to process - if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - int32_t duration = getWaitingTimeInterval(pSql->retry); - taosMsleep(duration); - } - - pSql->retryReason = rpcMsg->code; - rpcMsg->code = tscRenewTableMeta(pSql, 0); - // if there is an error occurring, proceed to the following error handling procedure. - if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); - return; - } - } - } - } - - pRes->rspLen = 0; - - if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { - tscDebug("0x%"PRIx64" query is cancelled, code:%s", pSql->self, tstrerror(pRes->code)); - } else { - pRes->code = rpcMsg->code; - } - - if (pRes->code == TSDB_CODE_SUCCESS) { - tscDebug("0x%"PRIx64" reset retry counter to be 0 due to success rsp, old:%d", pSql->self, pSql->retry); - pSql->retry = 0; - } - - if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { - assert(rpcMsg->msgType == pCmd->msgType + 1); - pRes->code = rpcMsg->code; - pRes->rspType = rpcMsg->msgType; - pRes->rspLen = rpcMsg->contLen; - - if (pRes->rspLen > 0 && rpcMsg->pCont) { - char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); - if (tmp == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - } else { - pRes->pRsp = tmp; - memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen); - } - } else { - tfree(pRes->pRsp); - } - - /* - * There is not response callback function for submit response. - * The actual inserted number of points is the first number. - */ - if (rpcMsg->msgType == TDMT_VND_SUBMIT_RSP && pRes->pRsp != NULL) { - SShellSubmitRspMsg *pMsg = (SShellSubmitRspMsg*)pRes->pRsp; - pMsg->code = htonl(pMsg->code); - pMsg->numOfRows = htonl(pMsg->numOfRows); - pMsg->affectedRows = htonl(pMsg->affectedRows); - pMsg->failedRows = htonl(pMsg->failedRows); - pMsg->numOfFailedBlocks = htonl(pMsg->numOfFailedBlocks); - - pRes->numOfRows += pMsg->affectedRows; - tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s inserted rows:%d rspLen:%d", pSql->self, sqlCmd[pCmd->command], - tstrerror(pRes->code), pMsg->affectedRows, pRes->rspLen); - } else { - tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s rspLen:%d", pSql->self, sqlCmd[pCmd->command], tstrerror(pRes->code), pRes->rspLen); - } - } - - if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) { - rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); - } - - bool shouldFree = tscShouldBeFreed(pSql); - if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - if (rpcMsg->code != TSDB_CODE_SUCCESS) { - pRes->code = rpcMsg->code; - } - - rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; - if (rpcMsg->code == TSDB_CODE_RPC_FQDN_ERROR) { - tscAllocPayload(pCmd, TSDB_FQDN_LEN + 64); - tscSetFqdnErrorMsg(pSql, pEpSet); - } - - (*pSql->fp)(pSql->param, pSql, rpcMsg->code); - } - - if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it - tscDebug("0x%"PRIx64" sqlObj is automatically freed", pSql->self); - taosRemoveRef(tscObjRef, handle); - } - - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); -} - -int doBuildAndSendMsg(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pCmd->command == TSDB_SQL_SELECT || - pCmd->command == TSDB_SQL_FETCH || - pCmd->command == TSDB_SQL_RETRIEVE_MNODE || - pCmd->command == TSDB_SQL_INSERT || - pCmd->command == TSDB_SQL_CONNECT || - pCmd->command == TSDB_SQL_HB || - pCmd->command == TSDB_SQL_RETRIEVE_FUNC || - pCmd->command == TSDB_SQL_STABLEVGROUP) { - pRes->code = tscBuildMsg[pCmd->command](pSql, NULL); - } - - if (pRes->code != TSDB_CODE_SUCCESS) { - tscAsyncResultOnError(pSql); - return TSDB_CODE_SUCCESS; - } - - int32_t code = tscSendMsgToServer(pSql); - - // NOTE: if code is TSDB_CODE_SUCCESS, pSql may have been released here already by other threads. - if (code != TSDB_CODE_SUCCESS) { - pRes->code = code; - tscAsyncResultOnError(pSql); - return TSDB_CODE_SUCCESS; - } - - return TSDB_CODE_SUCCESS; -} - -int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - - SSqlCmd *pCmd = &pSql->cmd; - uint32_t type = 0; - - if (pQueryInfo == NULL) { - pQueryInfo = tscGetQueryInfo(pCmd); - } - - STableMetaInfo *pTableMetaInfo = NULL; - - if (pQueryInfo != NULL) { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo != NULL) { - tNameExtractFullName(&pTableMetaInfo->name, name); - } - - type = pQueryInfo->type; - - // while numOfTables equals to 0, it must be Heartbeat - assert((pQueryInfo->numOfTables == 0 && (pQueryInfo->command == TSDB_SQL_HB || pSql->cmd.command == TSDB_SQL_RETRIEVE_FUNC)) || pQueryInfo->numOfTables > 0); - } - - tscDebug("0x%"PRIx64" SQL cmd:%s will be processed, name:%s, type:%d", pSql->self, sqlCmd[pCmd->command], name, type); - if (pCmd->command < TSDB_SQL_MGMT) { // the pTableMetaInfo cannot be NULL - if (pTableMetaInfo == NULL) { - pSql->res.code = TSDB_CODE_TSC_APP_ERROR; - return pSql->res.code; - } - } else if (pCmd->command >= TSDB_SQL_LOCAL) { - return (*tscProcessMsgRsp[pCmd->command])(pSql); - } - - return doBuildAndSendMsg(pSql); -} - -int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SRetrieveTableReq *pRetrieveMsg = (SRetrieveTableReq *) pSql->cmd.payload; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - pRetrieveMsg->free = htons(pQueryInfo->type); - pRetrieveMsg->qId = htobe64(pSql->res.qId); - - // todo valid the vgroupId at the client side - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - int32_t vgIndex = pTableMetaInfo->vgroupIndex; - int32_t vgId = -1; - - if (pTableMetaInfo->pVgroupTables == NULL) { - SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; - assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); - vgId = pVgroupInfo->vgroups[vgIndex].vgId; - } else { - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - assert(vgIndex >= 0 && vgIndex < numOfVgroups); - - SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); - vgId = pTableIdList->vgInfo.vgId; - } - - pRetrieveMsg->header.vgId = htonl(vgId); - tscDebug("0x%"PRIx64" build fetch msg from vgId:%d, vgIndex:%d, qId:0x%" PRIx64, pSql->self, vgId, vgIndex, pSql->res.qId); - } else { - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); - tscDebug("0x%"PRIx64" build fetch msg from only one vgroup, vgId:%d, qId:0x%" PRIx64, pSql->self, pTableMeta->vgId, - pSql->res.qId); - } - - pSql->cmd.payloadLen = sizeof(SRetrieveTableReq); - pSql->cmd.msgType = TDMT_VND_FETCH; - - pRetrieveMsg->header.contLen = htonl(sizeof(SRetrieveTableReq)); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - - // pSql->cmd.payloadLen is set during copying data into payload - pSql->cmd.msgType = TDMT_VND_SUBMIT; - - SNewVgroupInfo vgroupInfo = {0}; - taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - - tscDebug("0x%"PRIx64" submit msg built, numberOfEP:%d", pSql->self, pSql->epSet.numOfEps); - - return TSDB_CODE_SUCCESS; -} - -/* - * for table query, simply return the size <= 1k - */ -static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) { - const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; - - SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); - int32_t srcColFilterSize = 0; - int32_t srcTagFilterSize = tscGetTagFilterSerializeLen(pQueryInfo); - - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - int32_t exprSize = (int32_t)(sizeof(SSqlExpr) * numOfExprs * 2); - - int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; - int32_t sqlLen = (int32_t) strlen(pSql->sqlstr) + 1; - - int32_t tableSerialize = 0; - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - if (pTableMetaInfo->pVgroupTables != NULL) { - size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); - - int32_t totalTables = 0; - for (int32_t i = 0; i < numOfGroups; ++i) { - SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, i); - totalTables += (int32_t) taosArrayGetSize(pTableInfo->itemList); - } - - tableSerialize = totalTables * sizeof(STableIdInfo); - } - - if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) { - STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); - if (pCond != NULL && pCond->cond != NULL) { - srcColFilterSize = pCond->len; - } - } - - SCond* pCond = &pQueryInfo->tagCond.tbnameCond; - if (pCond->len > 0) { - srcColListSize += pCond->len; - } - - return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize + - exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; -} - -static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, char *pMsg, - int32_t *succeed) { - TSKEY dfltKey = htobe64(pQueryMsg->window.skey); - - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { - - int32_t vgId = -1; - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - int32_t index = pTableMetaInfo->vgroupIndex; - assert(index >= 0); - - SVgroupMsg* pVgroupInfo = NULL; - if (pTableMetaInfo->vgroupList && pTableMetaInfo->vgroupList->numOfVgroups > 0) { - assert(index < pTableMetaInfo->vgroupList->numOfVgroups); - pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; - } else { - tscError("0x%"PRIx64" No vgroup info found", pSql->self); - - *succeed = 0; - return pMsg; - } - - vgId = pVgroupInfo->vgId; - tscSetDnodeEpSet(&pSql->epSet, pVgroupInfo); - tscDebug("0x%"PRIx64" query on stable, vgIndex:%d, numOfVgroups:%d", pSql->self, index, pTableMetaInfo->vgroupList->numOfVgroups); - } else { - vgId = pTableMeta->vgId; - - SNewVgroupInfo vgroupInfo = {0}; - taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - } - - if (pSql->epSet.numOfEps > 0){ - pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; - } - pQueryMsg->head.vgId = htonl(vgId); - - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->tid = htonl(pTableMeta->id.tid); - pTableIdInfo->uid = htobe64(pTableMeta->id.uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->id.uid, dfltKey)); - - pQueryMsg->numOfTables = htonl(1); // set the number of tables - pMsg += sizeof(STableIdInfo); - } else { // it is a subquery of the super table query, this EP info is acquired from vgroupInfo - int32_t index = pTableMetaInfo->vgroupIndex; - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - assert(index >= 0 && index < numOfVgroups); - - SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); - - // set the vgroup info - tscSetDnodeEpSet(&pSql->epSet, &pTableIdList->vgInfo); - pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); - - int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList); - pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables - - tscDebug("0x%"PRIx64" query on stable, vgId:%d, numOfTables:%d, vgIndex:%d, numOfVgroups:%d", pSql->self, - pTableIdList->vgInfo.vgId, numOfTables, index, numOfVgroups); - - // serialize each table id info - for(int32_t i = 0; i < numOfTables; ++i) { - STableIdInfo* pItem = taosArrayGet(pTableIdList->itemList, i); - - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->tid = htonl(pItem->tid); - pTableIdInfo->uid = htobe64(pItem->uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pItem->uid, dfltKey)); - pMsg += sizeof(STableIdInfo); - } - } - - char n[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, n); - - tscDebug("0x%"PRIx64" vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql->self, htonl(pQueryMsg->head.vgId), n, pTableMeta->id.tid, pTableMeta->id.uid); - return pMsg; -} - -// TODO refactor -static int32_t serializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) { - // append the filter information after the basic column information - for (int32_t f = 0; f < numOfFilters; ++f) { - SColumnFilterInfo *pColFilter = &pColFilters[f]; - - SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)(*pMsg); - pFilterMsg->filterstr = htons(pColFilter->filterstr); - - (*pMsg) += sizeof(SColumnFilterInfo); - - if (pColFilter->filterstr) { - pFilterMsg->len = htobe64(pColFilter->len); - memcpy(*pMsg, (void *)pColFilter->pz, (size_t)(pColFilter->len + 1)); - (*pMsg) += (pColFilter->len + 1); // append the additional filter binary info - } else { - pFilterMsg->lowerBndi = htobe64(pColFilter->lowerBndi); - pFilterMsg->upperBndi = htobe64(pColFilter->upperBndi); - } - - pFilterMsg->lowerRelOptr = htons(pColFilter->lowerRelOptr); - pFilterMsg->upperRelOptr = htons(pColFilter->upperRelOptr); - - if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { - tscError("invalid filter info"); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t serializeSqlExpr(SSqlExpr* pExpr, STableMetaInfo* pTableMetaInfo, char** pMsg, int64_t id, bool validateColumn) { - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - // the queried table has been removed and a new table with the same name has already been created already - // return error msg - if (pExpr->uid != pTableMeta->id.uid) { - tscError("0x%"PRIx64" table has already been destroyed", id); - return TSDB_CODE_TSC_INVALID_TABLE_NAME; - } - - if (validateColumn && !tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { - tscError("0x%"PRIx64" table schema is not matched with parsed sql", id); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - assert(pExpr->resColId < 0); - SSqlExpr* pSqlExpr = (SSqlExpr *)(*pMsg); - - SColIndex* pIndex = &pSqlExpr->colInfo; - - pIndex->colId = htons(pExpr->colInfo.colId); - pIndex->colIndex = htons(pExpr->colInfo.colIndex); - pIndex->flag = htons(pExpr->colInfo.flag); - pSqlExpr->uid = htobe64(pExpr->uid); - pSqlExpr->colType = htons(pExpr->colType); - pSqlExpr->colBytes = htons(pExpr->colBytes); - pSqlExpr->resType = htons(pExpr->resType); - pSqlExpr->resBytes = htons(pExpr->resBytes); - pSqlExpr->interBytes = htonl(pExpr->interBytes); - pSqlExpr->functionId = htons(pExpr->functionId); - pSqlExpr->numOfParams = htons(pExpr->numOfParams); - pSqlExpr->resColId = htons(pExpr->resColId); - pSqlExpr->flist.numOfFilters = htons(pExpr->flist.numOfFilters); - - (*pMsg) += sizeof(SSqlExpr); - for (int32_t j = 0; j < pExpr->numOfParams; ++j) { // todo add log - pSqlExpr->param[j].nType = htonl(pExpr->param[j].nType); - pSqlExpr->param[j].nLen = htonl(pExpr->param[j].nLen); - - if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) { - memcpy((*pMsg), pExpr->param[j].pz, pExpr->param[j].nLen); - (*pMsg) += pExpr->param[j].nLen; - } else { - pSqlExpr->param[j].i64 = htobe64(pExpr->param[j].i64); - } - } - - serializeColFilterInfo(pExpr->flist.filterInfo, pExpr->flist.numOfFilters, pMsg); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = NULL; - STableMeta *pTableMeta = NULL; - STableMetaInfo *pTableMetaInfo = NULL; - - int32_t code = TSDB_CODE_SUCCESS; - int32_t size = tscEstimateQueryMsgSize(pSql); - assert(size > 0); - - if (TSDB_CODE_SUCCESS != tscAllocPayloadFast(pCmd, size)) { - tscError("%p failed to malloc for query msg", pSql); - return TSDB_CODE_TSC_INVALID_OPERATION; // todo add test for this - } - - pQueryInfo = tscGetQueryInfo(pCmd); - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - pTableMeta = pTableMetaInfo->pTableMeta; - - SQueryAttr query = {{0}}; - tscCreateQueryFromQueryInfo(pQueryInfo, &query, pSql); - query.vgId = pTableMeta->vgId; - - SArray* tableScanOperator = createTableScanPlan(&query); - SArray* queryOperator = createExecOperatorPlan(&query); - - SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; - tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); - - int32_t numOfTags = query.numOfTags; - int32_t sqlLen = (int32_t) strlen(pSql->sqlstr); - - if (taosArrayGetSize(tableScanOperator) == 0) { - pQueryMsg->tableScanOperator = htonl(-1); - } else { - int32_t* tablescanOp = taosArrayGet(tableScanOperator, 0); - pQueryMsg->tableScanOperator = htonl(*tablescanOp); - } - - pQueryMsg->window.skey = htobe64(query.window.skey); - pQueryMsg->window.ekey = htobe64(query.window.ekey); - - pQueryMsg->order = htons(query.order.order); - pQueryMsg->orderColId = htons(query.order.orderColId); - pQueryMsg->fillType = htons(query.fillType); - pQueryMsg->limit = htobe64(query.limit.limit); - pQueryMsg->offset = htobe64(query.limit.offset); - pQueryMsg->numOfCols = htons(query.numOfCols); - - pQueryMsg->interval.interval = htobe64(query.interval.interval); - pQueryMsg->interval.sliding = htobe64(query.interval.sliding); - pQueryMsg->interval.offset = htobe64(query.interval.offset); - pQueryMsg->interval.intervalUnit = query.interval.intervalUnit; - pQueryMsg->interval.slidingUnit = query.interval.slidingUnit; - pQueryMsg->interval.offsetUnit = query.interval.offsetUnit; - - pQueryMsg->stableQuery = query.stableQuery; - pQueryMsg->topBotQuery = query.topBotQuery; - pQueryMsg->groupbyColumn = query.groupbyColumn; - pQueryMsg->hasTagResults = query.hasTagResults; - pQueryMsg->timeWindowInterpo = query.timeWindowInterpo; - pQueryMsg->queryBlockDist = query.queryBlockDist; - pQueryMsg->stabledev = query.stabledev; - pQueryMsg->tsCompQuery = query.tsCompQuery; - pQueryMsg->simpleAgg = query.simpleAgg; - pQueryMsg->pointInterpQuery = query.pointInterpQuery; - pQueryMsg->needReverseScan = query.needReverseScan; - pQueryMsg->stateWindow = query.stateWindow; - pQueryMsg->numOfTags = htonl(numOfTags); - pQueryMsg->sqlstrLen = htonl(sqlLen); - pQueryMsg->sw.gap = htobe64(query.sw.gap); - pQueryMsg->sw.primaryColId = htonl(PRIMARYKEY_TIMESTAMP_COL_INDEX); - - pQueryMsg->secondStageOutput = htonl(query.numOfExpr2); - pQueryMsg->numOfOutput = htons((int16_t)query.numOfOutput); // this is the stage one output column number - - pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); - pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); - pQueryMsg->tbnameCondLen = htonl(pQueryInfo->tagCond.tbnameCond.len); - pQueryMsg->queryType = htonl(pQueryInfo->type); - pQueryMsg->prevResultLen = htonl(pQueryInfo->bufLen); - - // set column list ids - size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); - char *pMsg = (char *)(pQueryMsg->tableCols) + numOfCols * sizeof(SColumnInfo); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfo *pCol = &query.tableCols[i]; - - pQueryMsg->tableCols[i].colId = htons(pCol->colId); - pQueryMsg->tableCols[i].bytes = htons(pCol->bytes); - pQueryMsg->tableCols[i].type = htons(pCol->type); - //pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters); - pQueryMsg->tableCols[i].flist.numOfFilters = 0; - pQueryMsg->tableCols[i].flist.filterInfo = 0; - // append the filter information after the basic column information - //serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg); - } - - if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0 && !onlyQueryTags(&query) ) { - STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); - if (pCond != NULL && pCond->cond != NULL) { - pQueryMsg->colCondLen = htons(pCond->len); - memcpy(pMsg, pCond->cond, pCond->len); - - pMsg += pCond->len; - } - } else { - pQueryMsg->colCondLen = 0; - } - - for (int32_t i = 0; i < query.numOfOutput; ++i) { - code = serializeSqlExpr(&query.pExpr1[i].base, pTableMetaInfo, &pMsg, pSql->self, true); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - } - - for (int32_t i = 0; i < query.numOfExpr2; ++i) { - code = serializeSqlExpr(&query.pExpr2[i].base, pTableMetaInfo, &pMsg, pSql->self, false); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - } - - int32_t succeed = 1; - - // serialize the table info (sid, uid, tags) - pMsg = doSerializeTableInfo(pQueryMsg, pSql, pTableMetaInfo, pMsg, &succeed); - if (succeed == 0) { - code = TSDB_CODE_TSC_APP_ERROR; - goto _end; - } - - SGroupbyExpr *pGroupbyExpr = query.pGroupbyExpr; - if (pGroupbyExpr != NULL && pGroupbyExpr->numOfGroupCols > 0) { - pQueryMsg->orderByIdx = htons(pGroupbyExpr->orderIndex); - pQueryMsg->orderType = htons(pGroupbyExpr->orderType); - - for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { - SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j); - - *((int16_t *)pMsg) = htons(pCol->colId); - pMsg += sizeof(pCol->colId); - - *((int16_t *)pMsg) += htons(pCol->colIndex); - pMsg += sizeof(pCol->colIndex); - - *((int16_t *)pMsg) += htons(pCol->flag); - pMsg += sizeof(pCol->flag); - - memcpy(pMsg, pCol->name, tListLen(pCol->name)); - pMsg += tListLen(pCol->name); - } - } - - if (query.fillType != TSDB_FILL_NONE) { - for (int32_t i = 0; i < query.numOfOutput; ++i) { - *((int64_t *)pMsg) = htobe64(query.fillVal[i]); - pMsg += sizeof(query.fillVal[0]); - } - } - - if (query.numOfTags > 0 && query.tagColList != NULL) { - for (int32_t i = 0; i < query.numOfTags; ++i) { - SColumnInfo* pTag = &query.tagColList[i]; - - SColumnInfo* pTagCol = (SColumnInfo*) pMsg; - pTagCol->colId = htons(pTag->colId); - pTagCol->bytes = htons(pTag->bytes); - pTagCol->type = htons(pTag->type); - pTagCol->flist.numOfFilters = 0; - - pMsg += sizeof(SColumnInfo); - } - } - - // serialize tag column query condition - if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) { - STagCond* pTagCond = &pQueryInfo->tagCond; - - SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid); - if (pCond != NULL && pCond->cond != NULL) { - pQueryMsg->tagCondLen = htons(pCond->len); - memcpy(pMsg, pCond->cond, pCond->len); - - pMsg += pCond->len; - } - } else { - pQueryMsg->tagCondLen = 0; - } - - if (pQueryInfo->bufLen > 0) { - memcpy(pMsg, pQueryInfo->buf, pQueryInfo->bufLen); - pMsg += pQueryInfo->bufLen; - } - - SCond* pCond = &pQueryInfo->tagCond.tbnameCond; - if (pCond->len > 0) { - strncpy(pMsg, pCond->cond, pCond->len); - pMsg += pCond->len; - } - - // compressed ts block - pQueryMsg->tsBuf.tsOffset = htonl((int32_t)(pMsg - pCmd->payload)); - - if (pQueryInfo->tsBuf != NULL) { - // note: here used the index instead of actual vnode id. - int32_t vnodeIndex = pTableMetaInfo->vgroupIndex; - code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsBuf.tsLen, &pQueryMsg->tsBuf.tsNumOfBlocks); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - - pMsg += pQueryMsg->tsBuf.tsLen; - - pQueryMsg->tsBuf.tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); - pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); - pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); - } else { - pQueryMsg->tsBuf.tsLen = 0; - pQueryMsg->tsBuf.tsNumOfBlocks = 0; - } - - int32_t numOfOperator = (int32_t) taosArrayGetSize(queryOperator); - pQueryMsg->numOfOperator = htonl(numOfOperator); - for(int32_t i = 0; i < numOfOperator; ++i) { - int32_t *operator = taosArrayGet(queryOperator, i); - *(int32_t*)pMsg = htonl(*operator); - - pMsg += sizeof(int32_t); - } - - // support only one udf - if (pQueryInfo->pUdfInfo != NULL && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { - pQueryMsg->udfContentOffset = htonl((int32_t) (pMsg - pCmd->payload)); - for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i); - *(int8_t*) pMsg = pUdfInfo->resType; - pMsg += sizeof(pUdfInfo->resType); - - *(int16_t*) pMsg = htons(pUdfInfo->resBytes); - pMsg += sizeof(pUdfInfo->resBytes); - - STR_TO_VARSTR(pMsg, pUdfInfo->name); - - pMsg += varDataTLen(pMsg); - - *(int32_t*) pMsg = htonl(pUdfInfo->funcType); - pMsg += sizeof(pUdfInfo->funcType); - - *(int32_t*) pMsg = htonl(pUdfInfo->bufSize); - pMsg += sizeof(pUdfInfo->bufSize); - - pQueryMsg->udfContentLen = htonl(pUdfInfo->contLen); - memcpy(pMsg, pUdfInfo->content, pUdfInfo->contLen); - - pMsg += pUdfInfo->contLen; - } - } else { - pQueryMsg->udfContentOffset = 0; - pQueryMsg->udfContentLen = 0; - } - - memcpy(pMsg, pSql->sqlstr, sqlLen); - pMsg += sqlLen; - - int32_t msgLen = (int32_t)(pMsg - pCmd->payload); - - tscDebug("0x%"PRIx64" msg built success, len:%d bytes", pSql->self, msgLen); - pCmd->payloadLen = msgLen; - pSql->cmd.msgType = TDMT_VND_QUERY; - - pQueryMsg->head.contLen = htonl(msgLen); - assert(msgLen + minMsgSize() <= (int32_t)pCmd->allocSize); - - _end: - freeQueryAttr(&query); - taosArrayDestroy(tableScanOperator); - taosArrayDestroy(queryOperator); - return code; -} - -int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateDbMsg); - - pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TDMT_MND_CREATE_DB : TSDB_MSG_TYPE_CREATE_TP; - - SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload; - -// assert(pCmd->numOfClause == 1); - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateDbMsg->db); - assert(code == TSDB_CODE_SUCCESS); - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildCreateFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - SCreateFuncMsg *pCreateFuncMsg = (SCreateFuncMsg *)pCmd->payload; - - pCmd->msgType = TDMT_MND_CREATE_FUNCTION; - - pCmd->payloadLen = sizeof(SCreateFuncMsg) + htonl(pCreateFuncMsg->codeLen); - - return TSDB_CODE_SUCCESS; -} - - -int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateDnodeReq); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateDnodeReq *pCreate = (SCreateDnodeReq *)pCmd->payload; - - SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); - strncpy(pCreate->ep, t0->z, t0->n); - - pCmd->msgType = TDMT_MND_CREATE_DNODE; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateAcctReq); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateAcctReq *pAlterMsg = (SCreateAcctReq *)pCmd->payload; - - SStrToken *pName = &pInfo->pMiscInfo->user.user; - SStrToken *pPwd = &pInfo->pMiscInfo->user.passwd; - - strncpy(pAlterMsg->user, pName->z, pName->n); - strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); - - SCreateAcctInfo *pAcctOpt = &pInfo->pMiscInfo->acctOpt; - - pAlterMsg->cfg.maxUsers = htonl(pAcctOpt->maxUsers); - pAlterMsg->cfg.maxDbs = htonl(pAcctOpt->maxDbs); - pAlterMsg->cfg.maxTimeSeries = htonl(pAcctOpt->maxTimeSeries); - pAlterMsg->cfg.maxStreams = htonl(pAcctOpt->maxStreams); - pAlterMsg->cfg.maxPointsPerSecond = htonl(pAcctOpt->maxPointsPerSecond); - pAlterMsg->cfg.maxStorage = htobe64(pAcctOpt->maxStorage); - pAlterMsg->cfg.maxQueryTime = htobe64(pAcctOpt->maxQueryTime); - pAlterMsg->cfg.maxConnections = htonl(pAcctOpt->maxConnections); - - if (pAcctOpt->stat.n == 0) { - pAlterMsg->cfg.accessState = -1; - } else { - if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { - pAlterMsg->cfg.accessState = TSDB_VN_READ_ACCCESS; - } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { - pAlterMsg->cfg.accessState = TSDB_VN_WRITE_ACCCESS; - } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { - pAlterMsg->cfg.accessState = TSDB_VN_ALL_ACCCESS; - } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { - pAlterMsg->cfg.accessState = 0; - } - } - - pCmd->msgType = TDMT_MND_CREATE_ACCT; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateUserReq); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateUserReq *pAlterMsg = (SCreateUserReq *)pCmd->payload; - - SUserInfo *pUser = &pInfo->pMiscInfo->user; - strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n); - pAlterMsg->flag = (int8_t)pUser->type; - - if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - pAlterMsg->privilege = (char)pCmd->count; - } else if (pUser->type == TSDB_ALTER_USER_PASSWD) { - strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); - } else { // create user password info - strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); - } - - if (pUser->type == TSDB_ALTER_USER_PASSWD || pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - pCmd->msgType = TDMT_MND_ALTER_USER; - } else { - pCmd->msgType = TDMT_MND_CREATE_USER; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SMCfgDnodeReq); - pCmd->msgType = TDMT_MND_CONFIG_DNODE; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SDropDbMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pDropDbMsg->db); - assert(code == TSDB_CODE_SUCCESS && pTableMetaInfo->name.type == TSDB_DB_NAME_T); - - pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; - - pCmd->msgType = (pInfo->pMiscInfo->dbType == TSDB_DB_TYPE_DEFAULT) ? TDMT_MND_DROP_DB : TSDB_MSG_TYPE_DROP_TP; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - pCmd->msgType = TDMT_MND_DROP_FUNCTION; - - pCmd->payloadLen = sizeof(SDropFuncMsg); - - return TSDB_CODE_SUCCESS; -} - - -int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMDropTableMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCMDropTableMsg *pDropTableMsg = (SCMDropTableMsg*)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pDropTableMsg->name); - - pDropTableMsg->supertable = (pInfo->pMiscInfo->tableType == TSDB_SUPER_TABLE)? 1:0; - pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; - pCmd->msgType = TDMT_MND_DROP_TABLE; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char dnodeEp[TSDB_EP_LEN] = {0}; - tstrncpy(dnodeEp, pCmd->payload, TSDB_EP_LEN); - - pCmd->payloadLen = sizeof(SDropDnodeReq); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SDropDnodeReq * pDrop = (SDropDnodeReq *)pCmd->payload; - tstrncpy(pDrop->ep, dnodeEp, tListLen(pDrop->ep)); - pCmd->msgType = TDMT_MND_DROP_DNODE; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropUserAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char user[TSDB_USER_LEN] = {0}; - tstrncpy(user, pCmd->payload, TSDB_USER_LEN); - - pCmd->payloadLen = sizeof(SDropUserReq); - pCmd->msgType = (pInfo->type == TSDB_SQL_DROP_USER)? TDMT_MND_DROP_USER:TDMT_MND_DROP_ACCT; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SDropUserReq *pDropMsg = (SDropUserReq *)pCmd->payload; - tstrncpy(pDropMsg->user, user, tListLen(user)); - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SUseDbMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pUseDbMsg->db); - pCmd->msgType = TDMT_MND_USE_DB; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildSyncDbReplicaMsg(SSqlObj* pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SSyncDbMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SSyncDbMsg *pSyncMsg = (SSyncDbMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pSyncMsg->db); - pCmd->msgType = TDMT_MND_SYNC_DB; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TDMT_MND_SHOW; - pCmd->payloadLen = sizeof(SShowReq) + 100; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt; - SShowReq *pShowMsg = (SShowReq *)pCmd->payload; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - if (pShowInfo->showType == TSDB_MGMT_TABLE_FUNCTION) { - pShowMsg->type = pShowInfo->showType; - pShowMsg->payloadLen = 0; - pCmd->payloadLen = sizeof(SShowReq); - - return TSDB_CODE_SUCCESS; - } - - if (tNameIsEmpty(&pTableMetaInfo->name)) { - char *p = cloneCurrentDBName(pSql); - tstrncpy(pShowMsg->db, p, sizeof(pShowMsg->db)); - tfree(p); - } else { - tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db); - } - - pShowMsg->type = pShowInfo->showType; - - if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { - SStrToken *pPattern = &pShowInfo->pattern; - if (pPattern->type > 0) { // only show tables support wildcard query - strncpy(pShowMsg->payload, pPattern->z, pPattern->n); - pShowMsg->payloadLen = htons(pPattern->n); - } - } else { - SStrToken *pEpAddr = &pShowInfo->prefix; - assert(pEpAddr->n > 0 && pEpAddr->type > 0); - - strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); - pShowMsg->payloadLen = htons(pEpAddr->n); - } - - pCmd->payloadLen = sizeof(SShowReq) + htons(pShowMsg->payloadLen); - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SKillQueryReq); - - switch (pCmd->command) { - case TSDB_SQL_KILL_QUERY: - pCmd->msgType = TDMT_MND_KILL_QUERY; - break; - case TSDB_SQL_KILL_CONNECTION: - pCmd->msgType = TDMT_MND_KILL_CONN; - break; - case TSDB_SQL_KILL_STREAM: - pCmd->msgType = TSDB_MSG_TYPE_KILL_STREAM; - break; - } - return TSDB_CODE_SUCCESS; -} - -int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &(pSql->cmd); - int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg) + sizeof(SCreateTableMsg); - - SCreateTableSql *pCreateTableInfo = pInfo->pCreateTableInfo; - if (pCreateTableInfo->type == TSQL_CREATE_TABLE_FROM_STABLE) { - int32_t numOfTables = (int32_t)taosArrayGetSize(pInfo->pCreateTableInfo->childTableInfo); - size += numOfTables * (sizeof(SCreateTableMsg) + TSDB_MAX_TAGS_LEN); - } else { - size += sizeof(SSchema) * (pCmd->numOfCols + pCmd->count); - } - - if (pCreateTableInfo->pSelect != NULL) { - size += (pCreateTableInfo->pSelect->sqlstr.n + 1); - } - - return size + TSDB_EXTRA_PAYLOAD_SIZE; -} - -int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - int msgLen = 0; - int size = 0; - SSchema *pSchema; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - // Reallocate the payload size - size = tscEstimateCreateTableMsgLength(pSql, pInfo); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - tscError("0x%"PRIx64" failed to malloc for create table msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCMCreateTableMsg *pCreateTableMsg = (SCMCreateTableMsg *)pCmd->payload; - - SCreateTableMsg* pCreateMsg = (SCreateTableMsg*)((char*) pCreateTableMsg + sizeof(SCMCreateTableMsg)); - char* pMsg = NULL; - - int8_t type = pInfo->pCreateTableInfo->type; - if (type == TSQL_CREATE_TABLE_FROM_STABLE) { // create by using super table, tags value - SArray* list = pInfo->pCreateTableInfo->childTableInfo; - - int32_t numOfTables = (int32_t) taosArrayGetSize(list); - pCreateTableMsg->numOfTables = htonl(numOfTables); - - pMsg = (char*) pCreateMsg; - for(int32_t i = 0; i < numOfTables; ++i) { - SCreateTableMsg* pCreate = (SCreateTableMsg*) pMsg; - - pCreate->numOfColumns = htons(pCmd->numOfCols); - pCreate->numOfTags = htons(pCmd->count); - pMsg += sizeof(SCreateTableMsg); - - SCreatedTableInfo* p = taosArrayGet(list, i); - strcpy(pCreate->tableName, p->fullname); - pCreate->igExists = (p->igExist)? 1 : 0; - - // use dbinfo from table id without modifying current db info - pMsg = serializeTagData(&p->tagdata, pMsg); - - int32_t len = (int32_t)(pMsg - (char*) pCreate); - pCreate->len = htonl(len); - } - } else { // create (super) table - pCreateTableMsg->numOfTables = htonl(1); // only one table will be created - - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateMsg->tableName); - assert(code == 0); - - SCreateTableSql *pCreateTable = pInfo->pCreateTableInfo; - - pCreateMsg->igExists = pCreateTable->existCheck ? 1 : 0; - pCreateMsg->numOfColumns = htons(pCmd->numOfCols); - pCreateMsg->numOfTags = htons(pCmd->count); - - pCreateMsg->sqlLen = 0; - pMsg = (char *)pCreateMsg->schema; - - pSchema = (SSchema *)pCreateMsg->schema; - - for (int i = 0; i < pCmd->numOfCols + pCmd->count; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - pSchema->type = pField->type; - strcpy(pSchema->name, pField->name); - pSchema->bytes = htons(pField->bytes); - - pSchema++; - } - - pMsg = (char *)pSchema; - if (type == TSQL_CREATE_STREAM) { // check if it is a stream sql - SSqlNode *pQuerySql = pInfo->pCreateTableInfo->pSelect; - - strncpy(pMsg, pQuerySql->sqlstr.z, pQuerySql->sqlstr.n + 1); - pCreateMsg->sqlLen = htons(pQuerySql->sqlstr.n + 1); - pMsg += pQuerySql->sqlstr.n + 1; - } - } - - tscFieldInfoClear(&pQueryInfo->fieldsInfo); - - msgLen = (int32_t)(pMsg - (char*)pCreateTableMsg); - pCreateTableMsg->contLen = htonl(msgLen); - pCmd->payloadLen = msgLen; - pCmd->msgType = TDMT_MND_CREATE_TABLE; - - assert(msgLen + minMsgSize() <= size); - return TSDB_CODE_SUCCESS; -} - -int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) { - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE; -} - -int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - char *pMsg; - int msgLen = 0; - - SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SAlterTableInfo *pAlterInfo = pInfo->pAlterInfo; - int size = tscEstimateAlterTableMsgLength(pCmd); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - tscError("0x%"PRIx64" failed to malloc for alter table msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload; - - tNameExtractFullName(&pTableMetaInfo->name, pAlterTableMsg->tableFname); - pAlterTableMsg->type = htons(pAlterInfo->type); - - pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo)); - SSchema *pSchema = pAlterTableMsg->schema; - for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - pSchema->type = pField->type; - strcpy(pSchema->name, pField->name); - pSchema->bytes = htons(pField->bytes); - pSchema++; - } - - pMsg = (char *)pSchema; - pAlterTableMsg->tagValLen = htonl(pAlterInfo->tagData.dataLen); - if (pAlterInfo->tagData.dataLen > 0) { - memcpy(pMsg, pAlterInfo->tagData.data, pAlterInfo->tagData.dataLen); - } - pMsg += pAlterInfo->tagData.dataLen; - - msgLen = (int32_t)(pMsg - (char*)pAlterTableMsg); - - pCmd->payloadLen = msgLen; - pCmd->msgType = TDMT_VND_ALTER_TABLE; - - assert(msgLen + minMsgSize() <= size); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { - SSqlCmd* pCmd = &pSql->cmd; - pCmd->msgType = TDMT_VND_UPDATE_TAG_VAL; - - SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; - pCmd->payloadLen = htonl(pUpdateMsg->head.contLen); - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - - SNewVgroupInfo vgroupInfo = {.vgId = -1}; - taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); - assert(vgroupInfo.vgId > 0); - - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - - return TSDB_CODE_SUCCESS; -} - -int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SAlterDbMsg); - pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TDMT_MND_ALTER_DB : TSDB_MSG_TYPE_ALTER_TP; - - SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload; - pAlterDbMsg->dbType = -1; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pAlterDbMsg->db); - - return TSDB_CODE_SUCCESS; -} -int tscBuildCompactMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - if (pInfo->list == NULL || taosArrayGetSize(pInfo->list) <= 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - STscObj *pObj = pSql->pTscObj; - SSqlCmd *pCmd = &pSql->cmd; - SArray *pList = pInfo->list; - int32_t size = (int32_t)taosArrayGetSize(pList); - - int32_t *result = malloc(sizeof(int32_t) * size); - if (result == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < size; i++) { - tSqlExprItem* pSub = taosArrayGet(pList, i); - tVariant* pVar = &pSub->pNode->value; - if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) { - result[i] = (int32_t)(pVar->i64); - } else { - free(result); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - int count = removeDupVgid(result, size); - pCmd->payloadLen = sizeof(SCompactMsg) + count * sizeof(int32_t); - pCmd->msgType = TSDB_MSG_TYPE_COMPACT_VNODE; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - free(result); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - SCompactMsg *pCompactMsg = (SCompactMsg *)pCmd->payload; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - if (tNameIsEmpty(&pTableMetaInfo->name)) { - pthread_mutex_lock(&pObj->mutex); - tstrncpy(pCompactMsg->db, pObj->db, sizeof(pCompactMsg->db)); - pthread_mutex_unlock(&pObj->mutex); - } else { - tNameGetFullDbName(&pTableMetaInfo->name, pCompactMsg->db); - } - - pCompactMsg->numOfVgroup = htons(count); - for (int32_t i = 0; i < count; i++) { - pCompactMsg->vgid[i] = htons(result[i]); - } - free(result); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TDMT_MND_SHOW_RETRIEVE; - pCmd->payloadLen = sizeof(SRetrieveTableReq); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - SRetrieveTableReq *pRetrieveMsg = (SRetrieveTableReq*)pCmd->payload; - pRetrieveMsg->qId = htobe64(pSql->res.qId); - pRetrieveMsg->free = htons(pQueryInfo->type); - - return TSDB_CODE_SUCCESS; -} - -/* - * this function can only be called once. - * by using pRes->rspType to denote its status - * - * if pRes->rspType is 1, no more result - */ -static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - pRes->code = TSDB_CODE_SUCCESS; - if (pRes->rspType == 0) { - pRes->numOfRows = numOfRes; - pRes->row = 0; - pRes->rspType = 1; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { - return pRes->code; - } - - tscSetResRawPtr(pRes, pQueryInfo); - } else { - tscResetForNextRetrieve(pRes); - } - - uint8_t code = pSql->res.code; - if (pSql->fp) { - if (code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); - } else { - tscAsyncResultOnError(pSql); - } - } - - return code; -} - -int tscProcessDescribeTableRsp(SSqlObj *pSql) { - SSqlCmd * pCmd = &pSql->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - int32_t numOfRes = tinfo.numOfColumns + tinfo.numOfTags; - return tscLocalResultCommonBuilder(pSql, numOfRes); -} - -int tscProcessLocalRetrieveRsp(SSqlObj *pSql) { - int32_t numOfRes = 1; - pSql->res.completed = true; - return tscLocalResultCommonBuilder(pSql, numOfRes); -} - -int tscProcessRetrieveGlobalMergeRsp(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - SSqlCmd* pCmd = &pSql->cmd; - - int32_t code = pRes->code; - if (pRes->code != TSDB_CODE_SUCCESS) { - tscAsyncResultOnError(pSql); - return code; - } - - if (pRes->pMerger == NULL) { // no result from subquery, so abort here directly. - (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); - return code; - } - - // global aggregation may be the upstream for parent query - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - if (pQueryInfo->pQInfo == NULL) { - STableGroupInfo tableGroupInfo = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES),}; - tableGroupInfo.map = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - - STableKeyInfo tableKeyInfo = {.pTable = NULL, .lastKey = INT64_MIN}; - - SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); - taosArrayPush(group, &tableKeyInfo); - taosArrayPush(tableGroupInfo.pGroupList, &group); - - tscDebug("0x%"PRIx64" create QInfo 0x%"PRIx64" to execute query processing", pSql->self, pSql->self); - pQueryInfo->pQInfo = createQInfoFromQueryNode(pQueryInfo, &tableGroupInfo, NULL, NULL, pRes->pMerger, MERGE_STAGE, pSql->self); - } - - uint64_t localQueryId = pSql->self; - qTableQuery(pQueryInfo->pQInfo, &localQueryId); - convertQueryResult(pRes, pQueryInfo, pSql->self, true); - - code = pRes->code; - if (pRes->code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); - } else { - tscAsyncResultOnError(pSql); - } - - return code; -} - -int tscProcessEmptyResultRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder(pSql, 0); } - -int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - STscObj *pObj = pSql->pTscObj; - SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TDMT_MND_CONNECT; - pCmd->payloadLen = sizeof(SConnectReq); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SConnectReq *pConnect = (SConnectReq*)pCmd->payload; - - // TODO refactor full_name - char *db; // ugly code to move the space - - pthread_mutex_lock(&pObj->mutex); - db = strstr(pObj->db, TS_PATH_DELIMITER); - - db = (db == NULL) ? pObj->db : db + 1; - tstrncpy(pConnect->db, db, sizeof(pConnect->db)); - pthread_mutex_unlock(&pObj->mutex); - - tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion)); - tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion)); - - pConnect->pid = htonl(taosGetPId()); - taosGetCurrentAPPName(pConnect->appName, NULL); - - return TSDB_CODE_SUCCESS; -} - -/** - * multi table meta req pkg format: - * |SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... - * 4B - **/ -int tscBuildMultiTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - pCmd->msgType = TDMT_VND_TABLES_META; - assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize); - - tscDebug("0x%"PRIx64" build load multi-tablemeta msg completed, numOfTables:%d, msg size:%d", pSql->self, pCmd->count, - pCmd->payloadLen); - - return pCmd->payloadLen; -} - -int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char* pMsg = pCmd->payload; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg; - pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables); - pMsg += sizeof(SSTableVgroupMsg); - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, i); - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pMsg); - assert(code == TSDB_CODE_SUCCESS); - - pMsg += TSDB_TABLE_FNAME_LEN; - } - - pCmd->msgType = TSDB_MSG_TYPE_STABLE_VGROUP; - pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildRetrieveFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char *pMsg = pCmd->payload; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - int32_t numOfFuncs = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo); - - SRetrieveFuncMsg *pRetrieveFuncMsg = (SRetrieveFuncMsg *)pMsg; - pRetrieveFuncMsg->num = htonl(numOfFuncs); - - pMsg += sizeof(SRetrieveFuncMsg); - for(int32_t i = 0; i < numOfFuncs; ++i) { - SUdfInfo* pUdf = taosArrayGet(pQueryInfo->pUdfInfo, i); - STR_TO_NET_VARSTR(pMsg, pUdf->name); - pMsg += varDataNetTLen(pMsg); - } - - pCmd->msgType = TSDB_MSG_TYPE_RETRIEVE_FUNC; - pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; - - pthread_mutex_lock(&pObj->mutex); - - int32_t numOfQueries = 2; - SSqlObj *tpSql = pObj->sqlList; - while (tpSql) { - tpSql = tpSql->next; - numOfQueries++; - } - - int32_t numOfStreams = 2; - SSqlStream *pStream = pObj->streamList; - while (pStream) { - pStream = pStream->next; - numOfStreams++; - } - - int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatReq) + 100; - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - pthread_mutex_unlock(&pObj->mutex); - tscError("0x%"PRIx64" failed to create heartbeat msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - // TODO the expired hb and client can not be identified by server till now. - SHeartBeatReq *pHeartbeat = (SHeartBeatReq *)pCmd->payload; - tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer)); - - pHeartbeat->numOfQueries = numOfQueries; - pHeartbeat->numOfStreams = numOfStreams; - - pHeartbeat->pid = htonl(taosGetPId()); - taosGetCurrentAPPName(pHeartbeat->appName, NULL); - - int msgLen = tscBuildQueryStreamDesc(pHeartbeat, pObj); - - pthread_mutex_unlock(&pObj->mutex); - - pCmd->payloadLen = msgLen; - pCmd->msgType = TDMT_MND_HEARTBEAT; - - return TSDB_CODE_SUCCESS; -} - -static int32_t tableMetaMsgConvert(STableMetaMsg* pMetaMsg) { - pMetaMsg->tid = htonl(pMetaMsg->tid); - pMetaMsg->sversion = htons(pMetaMsg->sversion); - pMetaMsg->tversion = htons(pMetaMsg->tversion); - pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); - - pMetaMsg->uid = htobe64(pMetaMsg->uid); - pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - - if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && - (pMetaMsg->tid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { - tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, - pMetaMsg->tid, pMetaMsg->tableFname); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (pMetaMsg->numOfTags > TSDB_MAX_TAGS) { - tscError("invalid numOfTags:%d", pMetaMsg->numOfTags); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (pMetaMsg->numOfColumns > TSDB_MAX_COLUMNS || pMetaMsg->numOfColumns <= 0) { - tscError("invalid numOfColumns:%d", pMetaMsg->numOfColumns); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - for (int i = 0; i < pMetaMsg->vgroup.numOfEps; ++i) { - pMetaMsg->vgroup.epAddr[i].port = htons(pMetaMsg->vgroup.epAddr[i].port); - } - - SSchema* pSchema = pMetaMsg->schema; - - int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags; - for (int i = 0; i < numOfTotalCols; ++i) { - pSchema->bytes = htons(pSchema->bytes); - pSchema->colId = htons(pSchema->colId); - - if (pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - assert(i == 0); - } - - pSchema++; - } - - return TSDB_CODE_SUCCESS; -} - -// update the vgroupInfo if needed -static void doUpdateVgroupInfo(int32_t vgId, SVgroupMsg *pVgroupMsg) { - assert(vgId > 0); - - SNewVgroupInfo vgroupInfo = {.inUse = -1}; - taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); - - // vgroup info exists, compare with it - if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, pVgroupMsg)) || (vgroupInfo.inUse < 0)) { - vgroupInfo = createNewVgroupInfo(pVgroupMsg); - taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); - tscDebug("add/update new VgroupInfo, vgId:%d, total cached:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); - } -} - -static void doAddTableMetaToLocalBuf(STableMeta* pTableMeta, STableMetaMsg* pMetaMsg, bool updateSTable) { - if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - // add or update the corresponding super table meta data info - int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); - - // The super tableMeta already exists, create it according to tableMeta and add it to hash map - if (updateSTable) { - STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); - uint32_t size = tscGetTableMetaSize(pSupTableMeta); - int32_t code = taosHashPut(tscTableMetaMap, pTableMeta->sTableName, len, pSupTableMeta, size); - assert(code == TSDB_CODE_SUCCESS); - - tfree(pSupTableMeta); - } - - CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); - taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), cMeta, sizeof(CChildTableMeta)); - tfree(cMeta); - } else { - uint32_t s = tscGetTableMetaSize(pTableMeta); - taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), pTableMeta, s); - } -} - -int tscProcessTableMetaRsp(SSqlObj *pSql) { - STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; - int32_t code = tableMetaMsgConvert(pMetaMsg); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - assert(pTableMetaInfo->pTableMeta == NULL); - - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - if (pTableMeta == NULL){ - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { - tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); - tfree(pTableMeta); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - assert(strncmp(pMetaMsg->tableFname, name, tListLen(pMetaMsg->tableFname)) == 0); - - doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, true); - if (pTableMeta->tableType != TSDB_SUPER_TABLE) { - doUpdateVgroupInfo(pTableMeta->vgId, &pMetaMsg->vgroup); - } - - tscDebug("0x%"PRIx64" recv table meta, uid:%" PRIu64 ", tid:%d, name:%s, numOfCols:%d, numOfTags:%d", pSql->self, - pTableMeta->id.uid, pTableMeta->id.tid, tNameGetTableName(&pTableMetaInfo->name), pTableMeta->tableInfo.numOfColumns, - pTableMeta->tableInfo.numOfTags); - - free(pTableMeta); - return TSDB_CODE_SUCCESS; -} - -static SArray* createVgroupIdListFromMsg(char* pMsg, SHashObj* pSet, char* name, int32_t* size, uint64_t id) { - SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; - - pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); - *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); - - SArray* vgroupIdList = taosArrayInit(pVgroupMsg->numOfVgroups, sizeof(int32_t)); - - if (pVgroupMsg->numOfVgroups <= 0) { - tscDebug("0x%" PRIx64 " empty vgroup id list, no corresponding tables for stable:%s", id, name); - } else { - // just init, no need to lock - for (int32_t j = 0; j < pVgroupMsg->numOfVgroups; ++j) { - SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; - vmsg->vgId = htonl(vmsg->vgId); - for (int32_t k = 0; k < vmsg->numOfEps; ++k) { - vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); - } - - taosArrayPush(vgroupIdList, &vmsg->vgId); - - if (taosHashGet(pSet, &vmsg->vgId, sizeof(vmsg->vgId)) == NULL) { - taosHashPut(pSet, &vmsg->vgId, sizeof(vmsg->vgId), "", 0); - doUpdateVgroupInfo(vmsg->vgId, vmsg); - } - } - } - - return vgroupIdList; -} - -static SVgroupsInfo* createVgroupInfoFromMsg(char* pMsg, int32_t* size, uint64_t id) { - SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; - pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); - - *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); - - size_t vgroupsz = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); - SVgroupsInfo *pVgroupInfo = calloc(1, vgroupsz); - assert(pVgroupInfo != NULL); - - pVgroupInfo->numOfVgroups = pVgroupMsg->numOfVgroups; - if (pVgroupInfo->numOfVgroups <= 0) { - tscDebug("0x%" PRIx64 " empty vgroup info, no corresponding tables for stable", id); - } else { - for (int32_t j = 0; j < pVgroupInfo->numOfVgroups; ++j) { - // just init, no need to lock - SVgroupMsg *pVgroup = &pVgroupInfo->vgroups[j]; - - SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; - vmsg->vgId = htonl(vmsg->vgId); - for (int32_t k = 0; k < vmsg->numOfEps; ++k) { - vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); - } - - pVgroup->numOfEps = vmsg->numOfEps; - pVgroup->vgId = vmsg->vgId; - for (int32_t k = 0; k < vmsg->numOfEps; ++k) { - pVgroup->epAddr[k].port = vmsg->epAddr[k].port; - tstrncpy(pVgroup->epAddr[k].fqdn, vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); -// pVgroup->epAddr[k].fqdn = strndup(vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); - } - - doUpdateVgroupInfo(pVgroup->vgId, vmsg); - } - } - - return pVgroupInfo; -} - -int tscProcessRetrieveFuncRsp(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SUdfFuncMsg* pFuncMsg = (SUdfFuncMsg *)pSql->res.pRsp; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - pFuncMsg->num = htonl(pFuncMsg->num); - assert(pFuncMsg->num == taosArrayGetSize(pQueryInfo->pUdfInfo)); - - char* pMsg = pFuncMsg->content; - for(int32_t i = 0; i < pFuncMsg->num; ++i) { - SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; - - for(int32_t j = 0; j < pFuncMsg->num; ++j) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); - if (strcmp(pUdfInfo->name, pFunc->name) != 0) { - continue; - } - - if (pUdfInfo->content) { - continue; - } - - pUdfInfo->resBytes = htons(pFunc->resBytes); - pUdfInfo->resType = pFunc->resType; - pUdfInfo->funcType = htonl(pFunc->funcType); - pUdfInfo->contLen = htonl(pFunc->len); - pUdfInfo->bufSize = htonl(pFunc->bufSize); - - pUdfInfo->content = malloc(pUdfInfo->contLen); - memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); - - pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; - } - } - - // master sqlObj locates in param - SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); - if(parent == NULL) { - return pSql->res.code; - } - - SQueryInfo* parQueryInfo = tscGetQueryInfo(&parent->cmd); - - assert(parent->signature == parent && (int64_t)pSql->param == parent->self); - taosArrayDestroy(parQueryInfo->pUdfInfo); - - parQueryInfo->pUdfInfo = pQueryInfo->pUdfInfo; // assigned to parent sql obj. - pQueryInfo->pUdfInfo = NULL; - return TSDB_CODE_SUCCESS; -} - -int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { - char *rsp = pSql->res.pRsp; - - SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp; - pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables); - pMultiMeta->numOfVgroup = htonl(pMultiMeta->numOfVgroup); - pMultiMeta->numOfUdf = htonl(pMultiMeta->numOfUdf); - - rsp += sizeof(SMultiTableMeta); - - SSqlObj* pParentSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); - if(pParentSql == NULL) { - return pSql->res.code; - } - - SSqlCmd *pParentCmd = &pParentSql->cmd; - SHashObj *pSet = taosHashInit(pMultiMeta->numOfVgroup, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - - char* buf = NULL; - char* pMsg = pMultiMeta->meta; - - // decompresss the message payload - if (pMultiMeta->compressed) { - buf = malloc(pMultiMeta->rawLen - sizeof(SMultiTableMeta)); - int32_t len = tsDecompressString(pMultiMeta->meta, pMultiMeta->contLen - sizeof(SMultiTableMeta), 1, - buf, pMultiMeta->rawLen - sizeof(SMultiTableMeta), ONE_STAGE_COMP, NULL, 0); - assert(len == pMultiMeta->rawLen - sizeof(SMultiTableMeta)); - - pMsg = buf; - } - - if (pParentCmd->pTableMetaMap == NULL) { - pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - } - - for (int32_t i = 0; i < pMultiMeta->numOfTables; i++) { - STableMetaMsg *pMetaMsg = (STableMetaMsg *)pMsg; - int32_t code = tableMetaMsgConvert(pMetaMsg); - if (code != TSDB_CODE_SUCCESS) { - taosHashCleanup(pSet); - taosReleaseRef(tscObjRef, pParentSql->self); - - tfree(buf); - return code; - } - - bool freeMeta = false; - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { - tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, pMetaMsg->tableFname); - tfree(pTableMeta); - taosHashCleanup(pSet); - taosReleaseRef(tscObjRef, pParentSql->self); - - tfree(buf); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (pMultiMeta->metaClone == 1 || pTableMeta->tableType == TSDB_SUPER_TABLE) { - STableMetaVgroupInfo p = {.pTableMeta = pTableMeta,}; - size_t keyLen = strnlen(pMetaMsg->tableFname, TSDB_TABLE_FNAME_LEN); - void* t = taosHashGet(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen); - assert(t == NULL); - - taosHashPut(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen, &p, sizeof(STableMetaVgroupInfo)); - } else { - freeMeta = true; - } - - // for each super table, only update meta information once - bool updateStableMeta = false; - if (pTableMeta->tableType == TSDB_CHILD_TABLE && taosHashGet(pSet, &pMetaMsg->suid, sizeof(pMetaMsg->suid)) == NULL) { - updateStableMeta = true; - taosHashPut(pSet, &pTableMeta->suid, sizeof(pMetaMsg->suid), "", 0); - } - - // create the tableMeta and add it into the TableMeta map - doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, updateStableMeta); - - // for each vgroup, only update the information once. - int64_t vgId = pMetaMsg->vgroup.vgId; - if (pTableMeta->tableType != TSDB_SUPER_TABLE && taosHashGet(pSet, &vgId, sizeof(vgId)) == NULL) { - doUpdateVgroupInfo((int32_t) vgId, &pMetaMsg->vgroup); - taosHashPut(pSet, &vgId, sizeof(vgId), "", 0); - } - - pMsg += pMetaMsg->contLen; - if (freeMeta) { - tfree(pTableMeta); - } - } - - for(int32_t i = 0; i < pMultiMeta->numOfVgroup; ++i) { - char fname[TSDB_TABLE_FNAME_LEN] = {0}; - tstrncpy(fname, pMsg, TSDB_TABLE_FNAME_LEN); - size_t len = strnlen(fname, TSDB_TABLE_FNAME_LEN); - - pMsg += TSDB_TABLE_FNAME_LEN; - - STableMetaVgroupInfo* p = taosHashGet(pParentCmd->pTableMetaMap, fname, len); - assert(p != NULL); - - int32_t size = 0; - if (p->vgroupIdList!= NULL) { - taosArrayDestroy(p->vgroupIdList); - } - - p->vgroupIdList = createVgroupIdListFromMsg(pMsg, pSet, fname, &size, pSql->self); - - int32_t numOfVgId = (int32_t) taosArrayGetSize(p->vgroupIdList); - int32_t s = sizeof(tFilePage) + numOfVgId * sizeof(int32_t); - - tFilePage* idList = calloc(1, s); - idList->num = numOfVgId; - memcpy(idList->data, TARRAY_GET_START(p->vgroupIdList), numOfVgId * sizeof(int32_t)); - - void* idListInst = taosCachePut(tscVgroupListBuf, fname, len, idList, s, 5000); - taosCacheRelease(tscVgroupListBuf, (void*) &idListInst, false); - - tfree(idList); - pMsg += size; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pParentCmd); - if (pMultiMeta->numOfUdf > 0) { - assert(pQueryInfo->pUdfInfo != NULL); - } - - for(int32_t i = 0; i < pMultiMeta->numOfUdf; ++i) { - SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; - - for(int32_t j = 0; j < pMultiMeta->numOfUdf; ++j) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); - if (strcmp(pUdfInfo->name, pFunc->name) != 0) { - continue; - } - - if (pUdfInfo->content) { - continue; - } - - pUdfInfo->resBytes = htons(pFunc->resBytes); - pUdfInfo->resType = pFunc->resType; - pUdfInfo->funcType = htonl(pFunc->funcType); - pUdfInfo->contLen = htonl(pFunc->len); - pUdfInfo->bufSize = htonl(pFunc->bufSize); - - pUdfInfo->content = malloc(pUdfInfo->contLen); - memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); - - pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; - } - } - - pSql->res.code = TSDB_CODE_SUCCESS; - pSql->res.numOfTotal = pMultiMeta->numOfTables; - tscDebug("0x%"PRIx64" load multi-tableMeta from mnode, numOfTables:%d", pSql->self, pMultiMeta->numOfTables); - - taosHashCleanup(pSet); - taosReleaseRef(tscObjRef, pParentSql->self); - - tfree(buf); - return TSDB_CODE_SUCCESS; -} - -int tscProcessSTableVgroupRsp(SSqlObj *pSql) { - // master sqlObj locates in param - SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); - if(parent == NULL) { - return pSql->res.code; - } - - assert(parent->signature == parent && (int64_t)pSql->param == parent->self); - - SSqlRes* pRes = &pSql->res; - - // NOTE: the order of several table must be preserved. - SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp; - pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables); - char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg); - - SSqlCmd* pCmd = &parent->cmd; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - char fName[TSDB_TABLE_FNAME_LEN] = {0}; - for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) { - char* name = pMsg; - pMsg += TSDB_TABLE_FNAME_LEN; - - STableMetaInfo *pInfo = NULL; - for(int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { - STableMetaInfo *pInfo1 = tscGetTableMetaInfoFromCmd(pCmd, j); - memset(fName, 0, tListLen(fName)); - - tNameExtractFullName(&pInfo1->name, fName); - if (strcmp(name, fName) != 0) { - continue; - } - - pInfo = pInfo1; - break; - } - - if (!pInfo){ - continue; - } - int32_t size = 0; - pInfo->vgroupList = createVgroupInfoFromMsg(pMsg, &size, pSql->self); - pMsg += size; - } - - taosReleaseRef(tscObjRef, parent->self); - return pSql->res.code; -} - -int tscProcessShowRsp(SSqlObj *pSql) { - STableMetaMsg *pMetaMsg; - SShowRsp * pShow; - SSchema * pSchema; - - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - pShow = (SShowRsp *)pRes->pRsp; - pShow->qhandle = htobe64(pShow->qhandle); - pRes->qId = pShow->qhandle; - - tscResetForNextRetrieve(pRes); - pMetaMsg = &(pShow->tableMeta); - - pMetaMsg->numOfColumns = ntohs(pMetaMsg->numOfColumns); - - pSchema = pMetaMsg->schema; - pMetaMsg->tid = ntohs(pMetaMsg->tid); - for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { - pSchema->bytes = htons(pSchema->bytes); - pSchema++; - } - - tfree(pTableMetaInfo->pTableMeta); - pTableMetaInfo->pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - - SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - if (pQueryInfo->colList == NULL) { - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - } - - SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; - - SColumnIndex index = {0}; - pSchema = pMetaMsg->schema; - - uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; - for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i, ++pSchema) { - index.columnIndex = i; - tscColumnListInsert(pQueryInfo->colList, i, uid, pSchema); - - TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); - SInternalField* pInfo = tscFieldInfoAppend(pFieldInfo, &f); - - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, - pTableSchema[i].type, pTableSchema[i].bytes, getNewResColId(pCmd), pTableSchema[i].bytes, false); - } - - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - tscFieldInfoUpdateOffset(pQueryInfo); - return 0; -} - -static void createHbObj(STscObj* pObj) { - if (pObj->hbrid != 0) { - return; - } - - SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (NULL == pSql) return; - - pSql->fp = tscProcessHeartBeatRsp; - - SQueryInfo *pQueryInfo = tscGetQueryInfoS(&pSql->cmd); - if (pQueryInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tfree(pSql); - return; - } - - pQueryInfo->command = TSDB_SQL_HB; - - pSql->cmd.command = pQueryInfo->command; - if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tfree(pSql); - return; - } - - pSql->param = pObj; - pSql->pTscObj = pObj; - pSql->signature = pSql; - - registerSqlObj(pSql); - tscDebug("0x%"PRIx64" HB is allocated, pObj:%p", pSql->self, pObj); - - pObj->hbrid = pSql->self; -} - -int tscProcessConnectRsp(SSqlObj *pSql) { - STscObj *pObj = pSql->pTscObj; - SSqlRes *pRes = &pSql->res; - - char temp[TSDB_TABLE_FNAME_LEN * 2] = {0}; - - SConnectRsp *pConnect = (SConnectRsp *)pRes->pRsp; - tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId)); // copy acctId from response - - pthread_mutex_lock(&pObj->mutex); - int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); - - assert(len <= sizeof(pObj->db)); - tstrncpy(pObj->db, temp, sizeof(pObj->db)); - pthread_mutex_unlock(&pObj->mutex); - - if (pConnect->epSet.numOfEps > 0) { - tscEpSetHtons(&pConnect->epSet); - tscUpdateMgmtEpSet(pSql, &pConnect->epSet); - - for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { - tscDebug("0x%"PRIx64" epSet.fqdn[%d]: %s, pObj:%p", pSql->self, i, pConnect->epSet.fqdn[i], pObj); - } - } - - strcpy(pObj->sversion, pConnect->serverVersion); - pObj->writeAuth = pConnect->writeAuth; - pObj->superAuth = pConnect->superAuth; - pObj->connId = htonl(pConnect->connId); - - createHbObj(pObj); - - //launch a timer to send heartbeat to maintain the connection and send status to mnode - taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pObj->rid, tscTmr, &pObj->pTimer); - - return 0; -} - -int tscProcessUseDbRsp(SSqlObj *pSql) { - STscObj * pObj = pSql->pTscObj; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - - pthread_mutex_lock(&pObj->mutex); - int ret = tNameExtractFullName(&pTableMetaInfo->name, pObj->db); - pthread_mutex_unlock(&pObj->mutex); - - return ret; -} - -//todo only invalid the buffered data that belongs to dropped databases -int tscProcessDropDbRsp(SSqlObj *pSql) { - //TODO LOCK DB WHEN MODIFY IT - //pSql->pTscObj->db[0] = 0; - - taosHashClear(tscTableMetaMap); - taosHashClear(tscVgroupMap); - taosCacheEmpty(tscVgroupListBuf); - return 0; -} - -int tscProcessDropTableRsp(SSqlObj *pSql) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - tscRemoveCachedTableMeta(pTableMetaInfo, pSql->self); - tfree(pTableMetaInfo->pTableMeta); - return 0; -} - -int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - - tscDebug("0x%"PRIx64" remove tableMeta in hashMap after alter-table: %s", pSql->self, name); - - bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - tfree(pTableMetaInfo->pTableMeta); - - if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta - if (pSql->res.pRsp == NULL) { - tscDebug("0x%"PRIx64" unexpected resp from mnode, super table: %s failed to update super table meta ", pSql->self, name); - return 0; - } - return tscProcessTableMetaRsp(pSql); - } - - return 0; -} - -int tscProcessAlterDbMsgRsp(SSqlObj *pSql) { - UNUSED(pSql); - return 0; -} -int tscProcessCompactRsp(SSqlObj *pSql) { - UNUSED(pSql); - return TSDB_CODE_SUCCESS; -} - -int tscProcessShowCreateRsp(SSqlObj *pSql) { - return tscLocalResultCommonBuilder(pSql, 1); -} - -int tscProcessQueryRsp(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - - SQueryTableRsp *pQueryAttr = (SQueryTableRsp *)pRes->pRsp; - pQueryAttr->qId = htobe64(pQueryAttr->qId); - - pRes->qId = pQueryAttr->qId; - pRes->data = NULL; - - tscResetForNextRetrieve(pRes); - tscDebug("0x%"PRIx64" query rsp received, qId:0x%"PRIx64, pSql->self, pRes->qId); - return 0; -} - -static void decompressQueryColData(SSqlObj *pSql, SSqlRes *pRes, SQueryInfo* pQueryInfo, char **data, int8_t compressed, int32_t compLen) { - int32_t decompLen = 0; - int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - int32_t *compSizes; - char *pData = *data; - compSizes = (int32_t *)(pData + compLen); - - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); - int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); - char *outputBuf = tcalloc(pRes->numOfRows, (pField->bytes + offset)); - - char *p = outputBuf; - int32_t bufOffset; - for (int32_t i = 0; i < numOfCols; ++i) { - SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); - bufOffset = pInfo->field.bytes * pRes->numOfRows; - - int32_t flen = (*(tDataTypes[pInfo->field.type].decompFunc))(pData, htonl(compSizes[i]), pRes->numOfRows, p, bufOffset, - compressed, NULL, 0); - - p += flen; - decompLen +=flen; - pData += htonl(compSizes[i]); - } - - /* Resize rsp as decompressed data will occupy more space */ - pRes->rspLen = pRes->rspLen - (compLen + numOfCols * sizeof(int32_t)) + decompLen; - char *new_rsp = (char *)realloc(pRes->pRsp, pRes->rspLen); - if (new_rsp == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return; - } else { - pRes->pRsp = new_rsp; - *data = ((SRetrieveTableRsp *)pRes->pRsp)->data; - pData = *data + compLen + numOfCols * sizeof(int32_t); - } - - tscDebug("0x%"PRIx64" decompress col data, compressed size:%d, decompressed size:%d", - pSql->self, (int32_t)(compLen + numOfCols * sizeof(int32_t)), decompLen); - - int32_t tailLen = pRes->rspLen - sizeof(SRetrieveTableRsp) - decompLen; - memmove(*data + decompLen, pData, tailLen); - memmove(*data, outputBuf, decompLen); - - tfree(outputBuf); -} - -int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - assert(pRes->rspLen >= sizeof(SRetrieveTableRsp)); - - SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *)pRes->pRsp; - if (pRetrieve == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; - } - - pRes->numOfRows = htonl(pRetrieve->numOfRows); - pRes->precision = htons(pRetrieve->precision); - pRes->offset = htobe64(pRetrieve->offset); - pRes->useconds = htobe64(pRetrieve->useconds); - pRes->completed = (pRetrieve->completed == 1); - pRes->data = pRetrieve->data; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { - return pRes->code; - } - - //Decompress col data if compressed from server - if (pRetrieve->compressed) { - int32_t compLen = htonl(pRetrieve->compLen); - decompressQueryColData(pSql, pRes, pQueryInfo, &pRes->data, pRetrieve->compressed, compLen); - } - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if ((pCmd->command == TSDB_SQL_RETRIEVE_MNODE) || - ((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) || - (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE))) { - tscSetResRawPtr(pRes, pQueryInfo); - } - - if (pSql->pSubscription != NULL) { - int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); - int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); - - char* p = pRes->data + (pField->bytes + offset) * pRes->numOfRows; - - int32_t numOfTables = htonl(*(int32_t*)p); - p += sizeof(int32_t); - for (int i = 0; i < numOfTables; i++) { - int64_t uid = htobe64(*(int64_t*)p); - p += sizeof(int64_t); - p += sizeof(int32_t); // skip tid - TSKEY key = htobe64(*(TSKEY*)p); - p += sizeof(TSKEY); - tscUpdateSubscriptionProgress(pSql->pSubscription, uid, key); - } - } - - pRes->row = 0; - tscDebug("0x%"PRIx64" numOfRows:%d, offset:%" PRId64 ", complete:%d, qId:0x%"PRIx64, pSql->self, pRes->numOfRows, pRes->offset, - pRes->completed, pRes->qId); - - return 0; -} - -void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); - -static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool autocreate) { - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - if (NULL == pNew) { - tscError("0x%"PRIx64" malloc failed for new sqlobj to get table meta", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_META; - - tscAddQueryInfo(&pNew->cmd); - - SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { - tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); - - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - STableMetaInfo *pNewTableMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); - assert(pNewQueryInfo->numOfTables == 1); - - tNameAssign(&pNewTableMetaInfo->name, &pTableMetaInfo->name); - - registerSqlObj(pNew); - - pNew->fp = tscTableMetaCallBack; - pNew->param = (void *)pSql->self; - - tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get tableMeta, auto create:%d, metaRid from %"PRId64" to %"PRId64, - pSql->self, pNew->self, autocreate, pSql->metaRid, pNew->self); - pSql->metaRid = pNew->self; - - { - STableInfoMsg *pInfoMsg = (STableInfoMsg *)pNew->cmd.payload; - int32_t code = tNameExtractFullName(&pNewTableMetaInfo->name, pInfoMsg->tableFname); - if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - pInfoMsg->createFlag = htons(autocreate? 1 : 0); - char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); - - // tag data exists - if (autocreate && pSql->cmd.insertParam.tagData.dataLen != 0) { - pMsg = serializeTagData(&pSql->cmd.insertParam.tagData, pMsg); - } - - pNew->cmd.payloadLen = (int32_t)(pMsg - (char*)pInfoMsg); - pNew->cmd.msgType = TDMT_VND_TABLE_META; - } - - int32_t code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated - } - - return code; -} - -int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone) { - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - if (NULL == pNew) { - tscError("0x%"PRIx64" failed to allocate sqlobj to get multiple table meta", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_MULTI_META; - - int32_t numOfTable = (int32_t) taosArrayGetSize(pNameList); - int32_t numOfVgroupList = (int32_t) taosArrayGetSize(pVgroupNameList); - int32_t numOfUdf = pUdfList ? (int32_t)taosArrayGetSize(pUdfList) : 0; - - int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + TSDB_FUNC_NAME_LEN * numOfUdf + sizeof(SMultiTableInfoMsg); - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, size)) { - tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SMultiTableInfoMsg* pInfo = (SMultiTableInfoMsg*) pNew->cmd.payload; - pInfo->metaClone = metaClone? 1:0; - pInfo->numOfTables = htonl((uint32_t) taosArrayGetSize(pNameList)); - pInfo->numOfVgroups = htonl((uint32_t) taosArrayGetSize(pVgroupNameList)); - pInfo->numOfUdfs = htonl(numOfUdf); - - char* start = pInfo->tableNames; - int32_t len = 0; - for(int32_t i = 0; i < numOfTable; ++i) { - char* name = taosArrayGetP(pNameList, i); - if (i < numOfTable - 1 || numOfVgroupList > 0 || numOfUdf > 0) { - len = sprintf(start, "%s,", name); - } else { - len = sprintf(start, "%s", name); - } - - start += len; - } - - for(int32_t i = 0; i < numOfVgroupList; ++i) { - char* name = taosArrayGetP(pVgroupNameList, i); - if (i < numOfVgroupList - 1 || numOfUdf > 0) { - len = sprintf(start, "%s,", name); - } else { - len = sprintf(start, "%s", name); - } - - start += len; - } - - for(int32_t i = 0; i < numOfUdf; ++i) { - SUdfInfo * u = taosArrayGet(pUdfList, i); - if (i < numOfUdf - 1) { - len = sprintf(start, "%s,", u->name); - } else { - len = sprintf(start, "%s", u->name); - } - - start += len; - } - - pNew->cmd.payloadLen = (int32_t) ((start - pInfo->tableNames) + sizeof(SMultiTableInfoMsg)); - pNew->cmd.msgType = TDMT_VND_TABLES_META; - - registerSqlObj(pNew); - tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, udf:%d, msg size:%d", pSql->self, - pNew->self, numOfTable, numOfVgroupList, numOfUdf, pNew->cmd.payloadLen); - - pNew->fp = fp; - pNew->param = (void *)pSql->self; - - tscDebug("0x%"PRIx64" metaRid from 0x%" PRIx64 " to 0x%" PRIx64 , pSql->self, pSql->metaRid, pNew->self); - - pSql->metaRid = pNew->self; - int32_t code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated - } - - return code; -} - -int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) { - assert(tIsValidName(&pTableMetaInfo->name)); - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - - size_t len = strlen(name); - // just make runtime happy - if (pTableMetaInfo->tableMetaCapacity != 0 && pTableMetaInfo->pTableMeta != NULL) { - memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity); - } - - if (NULL == taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity)) { - tfree(pTableMetaInfo->pTableMeta); - } - - STableMeta* pMeta = pTableMetaInfo->pTableMeta; - STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf); - - if (pMeta && pMeta->id.uid > 0) { - // in case of child table, here only get the - if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity, (STableMeta **)(&pSTMeta)); - pSql->pBuf = (void *)(pSTMeta); - if (code != TSDB_CODE_SUCCESS) { - return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); - } - } - - tscDebug("0x%"PRIx64 " %s retrieve tableMeta from cache, numOfCols:%d, numOfTags:%d", pSql->self, name, pMeta->tableInfo.numOfColumns, pMeta->tableInfo.numOfTags); - return TSDB_CODE_SUCCESS; - } - - if (onlyLocal) { - return TSDB_CODE_TSC_NO_META_CACHED; - } - - return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); -} - -int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { - return tscGetTableMetaImpl(pSql, pTableMetaInfo, false, false); -} - -int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists, bool onlyLocal) { - return tscGetTableMetaImpl(pSql, pTableMetaInfo, createIfNotExists, onlyLocal); -} - -int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) { - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - if (NULL == pNew) { - tscError("%p malloc failed for new sqlobj to get user-defined functions", pSql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_RETRIEVE_FUNC; - - if (tscAddQueryInfo(&pNew->cmd) != TSDB_CODE_SUCCESS) { - tscError("%p malloc failed for new queryinfo", pSql); - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); - - pNewQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(SUdfInfo)); - for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { - SUdfInfo info = {0}; - SUdfInfo* p1 = taosArrayGet(pQueryInfo->pUdfInfo, i); - info = *p1; - info.name = strdup(p1->name); - taosArrayPush(pNewQueryInfo->pUdfInfo, &info); - } - - pNew->cmd.active = pNewQueryInfo; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { - tscError("%p malloc failed for payload to get table meta", pSql); - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - tscDebug("%p new pSqlObj:%p to retrieve udf", pSql, pNew); - registerSqlObj(pNew); - - pNew->fp = tscTableMetaCallBack; - pNew->param = (void *)pSql->self; - - tscDebug("%p metaRid from %" PRId64 " to %" PRId64 , pSql, pSql->metaRid, pNew->self); - - pSql->metaRid = pNew->self; - - int32_t code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated - } - - return code; -} - -static void freeElem(void* p) { - tfree(*(char**)p); -} - -/** - * retrieve table meta from mnode, and then update the local table meta hashmap. - * @param pSql sql object - * @param tableIndex table index - * @return status code - */ -int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { - SSqlCmd* pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, name); - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" failed to generate the table full name", pSql->self); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pTableMeta) { - tscDebug("0x%"PRIx64" update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRIu64, pSql->self, name, - tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); - } - - - // remove stored tableMeta info in hash table - tscResetSqlCmd(pCmd, true, pSql->self); - - SArray* pNameList = taosArrayInit(1, POINTER_BYTES); - SArray* vgroupList = taosArrayInit(1, POINTER_BYTES); - - char* n = strdup(name); - taosArrayPush(pNameList, &n); - code = getMultiTableMetaFromMnode(pSql, pNameList, vgroupList, NULL, tscTableMetaCallBack, true); - taosArrayDestroyEx(pNameList, freeElem); - taosArrayDestroyEx(vgroupList, freeElem); - - return code; -} - -static bool allVgroupInfoRetrieved(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - if (pTableMetaInfo->vgroupList == NULL) { - return false; - } - } - - // all super tables vgroupinfo are retrieved, no need to retrieve vgroup info anymore - return true; -} - -int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) { - int32_t code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - if (allVgroupInfoRetrieved(pQueryInfo)) { - return TSDB_CODE_SUCCESS; - } - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - - pNew->cmd.command = TSDB_SQL_STABLEVGROUP; - - // TODO TEST IT - SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); - if (pNewQueryInfo == NULL) { - tscFreeSqlObj(pNew); - return code; - } - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); - STableMeta* pTableMeta = tscTableMetaDup(pMInfo->pTableMeta); - tscAddTableMetaInfo(pNewQueryInfo, &pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); - } - - if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pNew); - return code; - } - - pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; - registerSqlObj(pNew); - - tscDebug("0x%"PRIx64" svgroupRid from %" PRId64 " to %" PRId64 , pSql->self, pSql->svgroupRid, pNew->self); - - pSql->svgroupRid = pNew->self; - tscDebug("0x%"PRIx64" new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql->self, pNew, pNewQueryInfo->numOfTables); - - pNew->fp = tscTableMetaCallBack; - pNew->param = (void *)pSql->self; - code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; - } - - return code; -} - -void tscInitMsgsFp() { - tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; - tscBuildMsg[TSDB_SQL_INSERT] = tscBuildSubmitMsg; - tscBuildMsg[TSDB_SQL_FETCH] = tscBuildFetchMsg; - - tscBuildMsg[TSDB_SQL_CREATE_DB] = tscBuildCreateDbMsg; - tscBuildMsg[TSDB_SQL_CREATE_USER] = tscBuildUserMsg; - tscBuildMsg[TSDB_SQL_CREATE_FUNCTION] = tscBuildCreateFuncMsg; - - tscBuildMsg[TSDB_SQL_CREATE_ACCT] = tscBuildAcctMsg; - tscBuildMsg[TSDB_SQL_ALTER_ACCT] = tscBuildAcctMsg; - - tscBuildMsg[TSDB_SQL_CREATE_TABLE] = tscBuildCreateTableMsg; - tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserAcctMsg; - tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropUserAcctMsg; - tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg; - tscBuildMsg[TSDB_SQL_DROP_FUNCTION] = tscBuildDropFuncMsg; - tscBuildMsg[TSDB_SQL_SYNC_DB_REPLICA] = tscBuildSyncDbReplicaMsg; - tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg; - tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildUserMsg; - tscBuildMsg[TSDB_SQL_CREATE_DNODE] = tscBuildCreateDnodeMsg; - tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; - tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; - tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; - tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; - tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; - tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; - - tscBuildMsg[TSDB_SQL_CONNECT] = tscBuildConnectMsg; - tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; - tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg; - tscBuildMsg[TSDB_SQL_RETRIEVE_FUNC] = tscBuildRetrieveFuncMsg; - - tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; - tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; - tscBuildMsg[TSDB_SQL_RETRIEVE_MNODE] = tscBuildRetrieveFromMgmtMsg; - tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; - tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; - tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; - - tscProcessMsgRsp[TSDB_SQL_SELECT] = tscProcessQueryRsp; - tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromNode; - - tscProcessMsgRsp[TSDB_SQL_DROP_DB] = tscProcessDropDbRsp; - tscProcessMsgRsp[TSDB_SQL_DROP_TABLE] = tscProcessDropTableRsp; - tscProcessMsgRsp[TSDB_SQL_CONNECT] = tscProcessConnectRsp; - tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; - tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp; - tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp; - tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiTableMetaRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; - - tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE_MNODE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. - tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; - - tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; - tscProcessMsgRsp[TSDB_SQL_CURRENT_USER] = tscProcessLocalRetrieveRsp; - tscProcessMsgRsp[TSDB_SQL_SERV_VERSION] = tscProcessLocalRetrieveRsp; - tscProcessMsgRsp[TSDB_SQL_CLI_VERSION] = tscProcessLocalRetrieveRsp; - tscProcessMsgRsp[TSDB_SQL_SERV_STATUS] = tscProcessLocalRetrieveRsp; - - tscProcessMsgRsp[TSDB_SQL_RETRIEVE_EMPTY_RESULT] = tscProcessEmptyResultRsp; - - tscProcessMsgRsp[TSDB_SQL_RETRIEVE_GLOBALMERGE] = tscProcessRetrieveGlobalMergeRsp; - - tscProcessMsgRsp[TSDB_SQL_ALTER_TABLE] = tscProcessAlterTableMsgRsp; - tscProcessMsgRsp[TSDB_SQL_ALTER_DB] = tscProcessAlterDbMsgRsp; - tscProcessMsgRsp[TSDB_SQL_COMPACT_VNODE] = tscProcessCompactRsp; - - tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_TABLE] = tscProcessShowCreateRsp; - tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_STABLE] = tscProcessShowCreateRsp; - tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; - - tscKeepConn[TSDB_SQL_SHOW] = 1; - tscKeepConn[TSDB_SQL_RETRIEVE_MNODE] = 1; - tscKeepConn[TSDB_SQL_SELECT] = 1; - tscKeepConn[TSDB_SQL_FETCH] = 1; - tscKeepConn[TSDB_SQL_HB] = 1; -} - diff --git a/2.0/src/client/src/tscSql.c b/2.0/src/client/src/tscSql.c deleted file mode 100644 index e84bc393a518647e4157e48b2d3aa8935f7bdd25..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscSql.c +++ /dev/null @@ -1,1021 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "hash.h" -#include "os.h" -#include "texpr.h" -#include "tkey.h" -#include "tcache.h" -#include "tnote.h" -#include "trpc.h" -#include "tscLog.h" -#include "tscSubquery.h" -#include "tscUtil.h" -#include "tsclient.h" -#include "ttokendef.h" -#include "tutil.h" -#include "ttimer.h" -#include "tscProfile.h" - -static bool validImpl(const char* str, size_t maxsize) { - if (str == NULL) { - return false; - } - - size_t len = strlen(str); - if (len <= 0 || len > maxsize) { - return false; - } - - return true; -} - -static bool validUserName(const char* user) { - return validImpl(user, TSDB_USER_LEN - 1); -} - -static bool validPassword(const char* passwd) { - return validImpl(passwd, TSDB_KEY_LEN - 1); -} - -static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *auth, const char *db, - void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos) { - if (taos_init()) { - return NULL; - } - - if (!validUserName(user)) { - terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH; - return NULL; - } - SRpcCorEpSet corMgmtEpSet; - - char secretEncrypt[32] = {0}; - int secretEncryptLen = 0; - if (auth == NULL) { - if (!validPassword(pass)) { - terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH; - return NULL; - } - taosEncryptPass((uint8_t *)pass, strlen(pass), secretEncrypt); - } else { - int outlen = 0; - int len = (int)strlen(auth); - char *base64 = (char *)base64_decode(auth, len, &outlen); - if (base64 == NULL || outlen == 0) { - tscError("invalid auth info:%s", auth); - free(base64); - terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH; - return NULL; - } else { - memcpy(secretEncrypt, base64, outlen); - free(base64); - } - secretEncryptLen = outlen; - } - - if (ip) { - if (tscSetMgmtEpSetFromCfg(ip, NULL, &corMgmtEpSet) < 0) return NULL; - if (port) corMgmtEpSet.epSet.port[0] = port; - } else { - if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond, &corMgmtEpSet) < 0) return NULL; - } - char rpcKey[512] = {0}; - snprintf(rpcKey, sizeof(rpcKey), "%s:%s:%s:%d", user, pass, ip, port); - - void *pRpcObj = NULL; - if (tscAcquireRpc(rpcKey, user, secretEncrypt, &pRpcObj) != 0) { - terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; - return NULL; - } - - STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); - if (NULL == pObj) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscReleaseRpc(pRpcObj); - return NULL; - } - - pObj->tscCorMgmtEpSet = malloc(sizeof(SRpcCorEpSet)); - if (pObj->tscCorMgmtEpSet == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscReleaseRpc(pRpcObj); - free(pObj); - return NULL; - } - memcpy(pObj->tscCorMgmtEpSet, &corMgmtEpSet, sizeof(corMgmtEpSet)); - - pObj->signature = pObj; - pObj->pRpcObj = (SRpcObj *)pRpcObj; - tstrncpy(pObj->user, user, sizeof(pObj->user)); - secretEncryptLen = TMIN(secretEncryptLen, sizeof(pObj->pass)); - memcpy(pObj->pass, secretEncrypt, secretEncryptLen); - - if (db) { - int32_t len = (int32_t)strlen(db); - /* db name is too long */ - if (len >= TSDB_DB_NAME_LEN) { - terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH; - tscReleaseRpc(pRpcObj); - free(pObj); - return NULL; - } - - char tmp[TSDB_DB_NAME_LEN] = {0}; - tstrncpy(tmp, db, sizeof(tmp)); - - strdequote(tmp); - strtolower(pObj->db, tmp); - } - - pthread_mutex_init(&pObj->mutex, NULL); - - SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (NULL == pSql) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscReleaseRpc(pRpcObj); - free(pObj); - return NULL; - } - - pSql->pTscObj = pObj; - pSql->signature = pSql; - pSql->maxRetry = TSDB_MAX_REPLICA; - pSql->fp = fp; - pSql->param = param; - pSql->cmd.command = TSDB_SQL_CONNECT; - - tsem_init(&pSql->rspSem, 0, 0); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscReleaseRpc(pRpcObj); - free(pSql); - free(pObj); - return NULL; - } - - if (taos != NULL) { - *taos = pObj; - } - pObj->rid = taosAddRef(tscRefId, pObj); - registerSqlObj(pSql); - - return pSql; -} - -static void syncConnCallback(void *param, TAOS_RES *tres, int code) { - SSqlObj *pSql = (SSqlObj *) tres; - assert(pSql != NULL); - - tsem_post(&pSql->rspSem); -} - -TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, - uint16_t port) { - STscObj *pObj = NULL; - SSqlObj *pSql = taosConnectImpl(ip, user, pass, auth, db, syncConnCallback, NULL, (void **)&pObj); - if (pSql != NULL) { - pSql->fp = syncConnCallback; - pSql->param = pSql; - - tscBuildAndSendRequest(pSql, NULL); - tsem_wait(&pSql->rspSem); - - if (pSql->res.code != TSDB_CODE_SUCCESS) { - terrno = pSql->res.code; - if (terrno ==TSDB_CODE_RPC_FQDN_ERROR) { - printf("taos connect failed, reason: %s\n\n", taos_errstr(pSql)); - } else { - printf("taos connect failed, reason: %s.\n\n", tstrerror(terrno)); - } - taos_free_result(pSql); - taos_close(pObj); - return NULL; - } - - tscDebug("%p DB connection is opening, rpcObj: %p, dnodeConn:%p", pObj, pObj->pRpcObj, pObj->pRpcObj->pDnodeConn); - taos_free_result(pSql); - - // version compare only requires the first 3 segments of the version string - int code = taosCheckVersion(version, taos_get_server_info(pObj), 3); - if (code != 0) { - terrno = code; - taos_close(pObj); - return NULL; - } else { - return pObj; - } - } - - return NULL; -} - -TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - tscDebug("try to create a connection to %s:%u, user:%s db:%s", ip, port != 0 ? port : tsServerPort , user, db); - if (user == NULL) user = TSDB_DEFAULT_USER; - if (pass == NULL) pass = TSDB_DEFAULT_PASS; - - return taos_connect_internal(ip, user, pass, NULL, db, port); -} - -TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) { - tscDebug("try to create a connection to %s:%u by auth, user:%s db:%s", ip, port, user, db); - if (user == NULL) user = TSDB_DEFAULT_USER; - if (auth == NULL) return NULL; - - return taos_connect_internal(ip, user, NULL, auth, db, port); -} - -TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t userLen, const char *pass, - uint8_t passLen, const char *db, uint8_t dbLen, uint16_t port) { - char ipBuf[TSDB_EP_LEN] = {0}; - char userBuf[TSDB_USER_LEN] = {0}; - char passBuf[TSDB_KEY_LEN] = {0}; - char dbBuf[TSDB_DB_NAME_LEN] = {0}; - strncpy(ipBuf, ip, TMIN(TSDB_EP_LEN - 1, ipLen)); - strncpy(userBuf, user, TMIN(TSDB_USER_LEN - 1, userLen)); - strncpy(passBuf, pass, TMIN(TSDB_KEY_LEN - 1, passLen)); - strncpy(dbBuf, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen)); - return taos_connect(ipBuf, userBuf, passBuf, dbBuf, port); -} - -static void asyncConnCallback(void *param, TAOS_RES *tres, int code) { - SSqlObj *pSql = (SSqlObj *) tres; - assert(pSql != NULL); - - pSql->fetchFp(pSql->param, tres, code); -} - -TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), - void *param, TAOS **taos) { - STscObj *pObj = NULL; - SSqlObj *pSql = taosConnectImpl(ip, user, pass, NULL, db, asyncConnCallback, param, (void **)&pObj); - if (pSql == NULL) { - return NULL; - } - - if (taos) *taos = pObj; - - pSql->fetchFp = fp; - pSql->res.code = tscBuildAndSendRequest(pSql, NULL); - tscDebug("%p DB async connection is opening", taos); - return pObj; -} - -void taos_close(TAOS *taos) { - STscObj *pObj = (STscObj *)taos; - - if (pObj == NULL) { - tscDebug("(null) try to free tscObj and close dnodeConn"); - return; - } - - tscDebug("%p try to free tscObj", pObj); - if (pObj->signature != pObj) { - tscDebug("%p already closed or invalid tscObj", pObj); - return; - } - - if (RID_VALID(pObj->hbrid)) { - SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); - if (pHb != NULL) { - if (RID_VALID(pHb->rpcRid)) { // wait for rsp from dnode - rpcCancelRequest(pHb->rpcRid); - pHb->rpcRid = -1; - } - - tscDebug("0x%"PRIx64" HB is freed", pHb->self); - taosReleaseRef(tscObjRef, pHb->self); -#ifdef __APPLE__ - // to satisfy later tsem_destroy in taos_free_result - tsem_init(&pHb->rspSem, 0, 0); -#endif // __APPLE__ - taos_free_result(pHb); - } - } - - tscDebug("%p all sqlObj are freed, free tscObj", pObj); - taosRemoveRef(tscRefId, pObj->rid); -} - -void waitForQueryRsp(void *param, TAOS_RES *tres, int code) { - assert(tres != NULL); - - SSqlObj *pSql = (SSqlObj *) tres; - tsem_post(&pSql->rspSem); -} - -static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) { - SSqlObj* pSql = (SSqlObj*) tres; - tsem_post(&pSql->rspSem); -} - -TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, int64_t* res) { - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - if (sqlLen > (uint32_t)tsMaxSQLStringLen) { - tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); - terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - return NULL; - } - - nPrintTsc("%s", sqlstr); - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - tscError("failed to malloc sqlObj"); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - tsem_init(&pSql->rspSem, 0, 0); - doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); - - if (res != NULL) { - atomic_store_64(res, pSql->self); - } - - tsem_wait(&pSql->rspSem); - return pSql; -} - -TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) { - return taos_query_c(taos, sqlstr, (uint32_t)strlen(sqlstr), NULL); -} - -TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res) { - return taos_query_c(taos, sqlstr, (uint32_t) strlen(sqlstr), res); -} - -int taos_result_precision(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - if (pSql == NULL || pSql->signature != pSql) return 0; - - return pSql->res.precision; -} - -int taos_num_rows(TAOS_RES *res) { return 0; } - -int taos_num_fields(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - if (pSql == NULL || pSql->signature != pSql) return 0; - - int32_t num = 0; - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - if (pQueryInfo == NULL) { - return num; - } - - while(pQueryInfo->pDownstream != NULL) { - pQueryInfo = pQueryInfo->pDownstream; - } - - size_t numOfCols = tscNumOfFields(pQueryInfo); - for(int32_t i = 0; i < numOfCols; ++i) { - SInternalField* pInfo = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); - if (pInfo->visible) { - num++; - } - } - - return num; -} - -int taos_field_count(TAOS_RES *tres) { - SSqlObj* pSql = (SSqlObj*) tres; - if (pSql == NULL || pSql->signature != pSql) return 0; - - return taos_num_fields(pSql); -} - -int taos_affected_rows(TAOS_RES *tres) { - SSqlObj* pSql = (SSqlObj*) tres; - if (pSql == NULL || pSql->signature != pSql) return 0; - - return pSql->res.numOfRows; -} - -TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - SSqlRes *pRes = &pSql->res; - if (pSql == NULL || pSql->signature != pSql) return 0; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - if (pQueryInfo == NULL) { - return NULL; - } - - size_t numOfCols = tscNumOfFields(pQueryInfo); - if (numOfCols == 0) { - return NULL; - } - - SFieldInfo *pFieldInfo = &pQueryInfo->fieldsInfo; - - if (pRes->final == NULL) { - TAOS_FIELD* f = calloc(pFieldInfo->numOfOutput, sizeof(TAOS_FIELD)); - - int32_t j = 0; - for(int32_t i = 0; i < pFieldInfo->numOfOutput; ++i) { - SInternalField* pField = tscFieldInfoGetInternalField(pFieldInfo, i); - if (pField->visible) { - f[j] = pField->field; - - // revise the length for binary and nchar fields - if (f[j].type == TSDB_DATA_TYPE_BINARY) { - f[j].bytes -= VARSTR_HEADER_SIZE; - } else if (f[j].type == TSDB_DATA_TYPE_NCHAR) { - f[j].bytes = (f[j].bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - } - - j += 1; - } - } - - pRes->final = f; - } - - return pRes->final; -} - -static bool needToFetchNewBlock(SSqlObj* pSql) { - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - return (pRes->completed != true || hasMoreVnodesToTry(pSql) || hasMoreClauseToTry(pSql)) && - (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || - pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE || - pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE || - pCmd->command == TSDB_SQL_FETCH || - pCmd->command == TSDB_SQL_SHOW || - pCmd->command == TSDB_SQL_SHOW_CREATE_TABLE || - pCmd->command == TSDB_SQL_SHOW_CREATE_STABLE || - pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE || - pCmd->command == TSDB_SQL_SELECT || - pCmd->command == TSDB_SQL_DESCRIBE_TABLE || - pCmd->command == TSDB_SQL_SERV_STATUS || - pCmd->command == TSDB_SQL_CURRENT_DB || - pCmd->command == TSDB_SQL_SERV_VERSION || - pCmd->command == TSDB_SQL_CLI_VERSION || - pCmd->command == TSDB_SQL_CURRENT_USER); -} - -TAOS_ROW taos_fetch_row(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - if (pSql == NULL || pSql->signature != pSql) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pRes->qId == 0 || - pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED || - pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || - pCmd->command == TSDB_SQL_INSERT) { - return NULL; - } - - // set the sql object owner - tscSetSqlOwner(pSql); - - // current data set are exhausted, fetch more result from node - if (pRes->row >= pRes->numOfRows && needToFetchNewBlock(pSql)) { - taos_fetch_rows_a(res, waitForRetrieveRsp, pSql->pTscObj); - tsem_wait(&pSql->rspSem); - } - - void* data = doSetResultRowData(pSql); - - tscClearSqlOwner(pSql); - return data; -} - -int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { - SSqlObj *pSql = (SSqlObj *)res; - if (pSql == NULL || pSql->signature != pSql) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return 0; - } - - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pRes->qId == 0 || - pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED || - pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || - pCmd->command == TSDB_SQL_INSERT) { - return 0; - } - - tscResetForNextRetrieve(pRes); - - // set the sql object owner - tscSetSqlOwner(pSql); - - // current data set are exhausted, fetch more data from node - if (needToFetchNewBlock(pSql)) { - taos_fetch_rows_a(res, waitForRetrieveRsp, pSql->pTscObj); - tsem_wait(&pSql->rspSem); - } - - *rows = pRes->urow; - - tscClearSqlOwner(pSql); - return pRes->numOfRows; -} - -int taos_select_db(TAOS *taos, const char *db) { - char sql[256] = {0}; - - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - snprintf(sql, tListLen(sql), "use %s", db); - SSqlObj* pSql = taos_query(taos, sql); - int32_t code = pSql->res.code; - taos_free_result(pSql); - - return code; -} - -// send free message to vnode to free qhandle and corresponding resources in vnode -static bool tscKillQueryInDnode(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - if (pRes == NULL || pRes->qId == 0) { - return true; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - if ((pQueryInfo == NULL) || pQueryInfo->globalMerge) { - return true; - } - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - tscRemoveFromSqlList(pSql); - - int32_t cmd = pCmd->command; - if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) && - (cmd == TSDB_SQL_SELECT || - cmd == TSDB_SQL_SHOW || - cmd == TSDB_SQL_RETRIEVE_MNODE || - cmd == TSDB_SQL_FETCH)) { - pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; - tscDebug("0x%"PRIx64" send msg to dnode to free qhandle ASAP before free sqlObj, command:%s", pSql->self, sqlCmd[pCmd->command]); - - tscBuildAndSendRequest(pSql, NULL); - return false; - } - - return true; -} - -void taos_free_result(TAOS_RES *res) { - SSqlObj* pSql = (SSqlObj*) res; - if (pSql == NULL || pSql->signature != pSql) { - tscError("0x%"PRIx64" already released sqlObj", pSql ? pSql->self : -1); - return; - } - - bool freeNow = tscKillQueryInDnode(pSql); - if (freeNow) { - tscDebug("0x%"PRIx64" free sqlObj in cache", pSql->self); - taosReleaseRef(tscObjRef, pSql->self); - } -} - -int taos_errno(TAOS_RES *tres) { - SSqlObj *pSql = (SSqlObj *) tres; - if (pSql == NULL || pSql->signature != pSql) { - return terrno; - } - - return pSql->res.code; -} - -/* - * In case of invalid sql/sql syntax error, additional information is attached to explain - * why the sql is invalid - */ -static bool hasAdditionalErrorInfo(int32_t code, SSqlCmd *pCmd) { - if (code != TSDB_CODE_TSC_INVALID_OPERATION - && code != TSDB_CODE_TSC_SQL_SYNTAX_ERROR) { - return false; - } - - size_t len = strlen(pCmd->payload); - - char *z = NULL; - if (len > 0) { - z = strstr(pCmd->payload, "invalid operation"); - if (z == NULL) { - z = strstr(pCmd->payload, "syntax error"); - } - } - return z != NULL; -} - -// todo should not be used in async model -char *taos_errstr(TAOS_RES *tres) { - SSqlObj *pSql = (SSqlObj *) tres; - - if (pSql == NULL || pSql->signature != pSql) { - return (char*) tstrerror(terrno); - } - - if (hasAdditionalErrorInfo(pSql->res.code, &pSql->cmd) || pSql->res.code == TSDB_CODE_RPC_FQDN_ERROR) { - return pSql->cmd.payload; - } else { - return (char*)tstrerror(pSql->res.code); - } -} - -void taos_config(int debug, char *log_path) { - uDebugFlag = debug; - tstrncpy(tsLogDir, log_path, TSDB_FILENAME_LEN); -} - -char *taos_get_server_info(TAOS *taos) { - STscObj *pObj = (STscObj *)taos; - - if (pObj == NULL) return NULL; - - return pObj->sversion; -} - -int* taos_fetch_lengths(TAOS_RES *res) { - SSqlObj* pSql = (SSqlObj* ) res; - if (pSql == NULL || pSql->signature != pSql) { - return NULL; - } - - return pSql->res.length; -} - -char *taos_get_client_info() { return version; } - -static void tscKillSTableQuery(SSqlObj *pSql) { - SSqlCmd* pCmd = &pSql->cmd; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - if (!pQueryInfo->globalMerge) { - return; - } - - // set the master sqlObj flag to cancel query - pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - - tscLockByThread(&pSql->squeryLock); - - for (int i = 0; i < pSql->subState.numOfSub; ++i) { - // NOTE: pSub may have been released already here - SSqlObj *pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - SSqlObj* pSubObj = pSub; - - pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - if (pSubObj->rpcRid > 0) { - rpcCancelRequest(pSubObj->rpcRid); - pSubObj->rpcRid = -1; - } - - tscAsyncResultOnError(pSubObj); - // taosRelekaseRef(tscObjRef, pSubObj->self); - } - - if (pSql->subState.numOfSub <= 0) { - tscAsyncResultOnError(pSql); - } - - tscUnlockByThread(&pSql->squeryLock); - - tscDebug("0x%"PRIx64" super table query cancelled", pSql->self); -} - -void taos_stop_query(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - if (pSql == NULL || pSql->signature != pSql) { - return; - } - - tscDebug("0x%"PRIx64" start to cancel query", pSql->self); - SSqlCmd *pCmd = &pSql->cmd; - - // set the error code for master pSqlObj firstly - pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - if (pQueryInfo->globalMerge) { - assert(pSql->rpcRid <= 0); - tscKillSTableQuery(pSql); - } else { - if (pSql->cmd.command < TSDB_SQL_LOCAL) { - /* - * There is multi-thread problem here, since pSql->pRpcCtx may have been - * reset and freed in the processMsgFromServer function, and causes the invalid - * write problem for rpcCancelRequest. - */ - if (pSql->rpcRid > 0) { - rpcCancelRequest(pSql->rpcRid); - pSql->rpcRid = -1; - } - - tscAsyncResultOnError(pSql); - } - } - - tscDebug("0x%"PRIx64" query is cancelled", pSql->self); -} - -bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { - SSqlObj *pSql = (SSqlObj *)res; - if (pSql == NULL || pSql->signature != pSql) { - return true; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - if (pQueryInfo == NULL) { - return true; - } - - SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, col); - if (col < 0 || col >= tscNumOfFields(pQueryInfo) || row < 0 || row > pSql->res.numOfRows) { - return true; - } - - return isNull(((char*) pSql->res.urow[col]) + row * pInfo->field.bytes, pInfo->field.type); -} - -int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { - int len = 0; - - for (int i = 0; i < num_fields; ++i) { - if (i > 0) { - str[len++] = ' '; - } - - if (row[i] == NULL) { - len += sprintf(str + len, "%s", TSDB_DATA_NULL_STR); - continue; - } - - switch (fields[i].type) { - case TSDB_DATA_TYPE_TINYINT: - len += sprintf(str + len, "%d", *((int8_t *)row[i])); - break; - - case TSDB_DATA_TYPE_UTINYINT: - len += sprintf(str + len, "%u", *((uint8_t *)row[i])); - break; - - case TSDB_DATA_TYPE_SMALLINT: - len += sprintf(str + len, "%d", *((int16_t *)row[i])); - break; - - case TSDB_DATA_TYPE_USMALLINT: - len += sprintf(str + len, "%u", *((uint16_t *)row[i])); - break; - - case TSDB_DATA_TYPE_INT: - len += sprintf(str + len, "%d", *((int32_t *)row[i])); - break; - - case TSDB_DATA_TYPE_UINT: - len += sprintf(str + len, "%u", *((uint32_t *)row[i])); - break; - - case TSDB_DATA_TYPE_BIGINT: - len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); - break; - - case TSDB_DATA_TYPE_UBIGINT: - len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i])); - break; - - case TSDB_DATA_TYPE_FLOAT: { - float fv = 0; - fv = GET_FLOAT_VAL(row[i]); - len += sprintf(str + len, "%f", fv); - } break; - - case TSDB_DATA_TYPE_DOUBLE: { - double dv = 0; - dv = GET_DOUBLE_VAL(row[i]); - len += sprintf(str + len, "%lf", dv); - } break; - - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - int32_t charLen = varDataLen((char*)row[i] - VARSTR_HEADER_SIZE); - if (fields[i].type == TSDB_DATA_TYPE_BINARY) { - assert(charLen <= fields[i].bytes && charLen >= 0); - } else { - assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0); - } - - memcpy(str + len, row[i], charLen); - len += charLen; - } break; - - case TSDB_DATA_TYPE_TIMESTAMP: - len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); - break; - - case TSDB_DATA_TYPE_BOOL: - len += sprintf(str + len, "%d", *((int8_t *)row[i])); - default: - break; - } - } - - return len; -} - -static void asyncCallback(void *param, TAOS_RES *tres, int code) { - assert(param != NULL); - SSqlObj *pSql = ((SSqlObj *)param); - pSql->res.code = code; - tsem_post(&pSql->rspSem); -} - -int taos_validate_sql(TAOS *taos, const char *sql) { - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - - pSql->pTscObj = taos; - pSql->signature = pSql; - SSqlCmd *pCmd = &pSql->cmd; - - pCmd->resColumnId = TSDB_RES_COL_ID; - - tscDebug("0x%"PRIx64" Valid SQL: %s pObj:%p", pSql->self, sql, pObj); - - int32_t sqlLen = (int32_t)strlen(sql); - if (sqlLen > tsMaxSQLStringLen) { - tscError("0x%"PRIx64" sql too long", pSql->self); - tfree(pSql); - return TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - } - - char* sqlstr = realloc(pSql->sqlstr, sqlLen + 1); - if(sqlstr == NULL && pSql->sqlstr) free(pSql->sqlstr); - pSql->sqlstr = sqlstr; - if (pSql->sqlstr == NULL) { - tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self); - tfree(pSql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - strtolower(pSql->sqlstr, sql); - -// pCmd->curSql = NULL; - if (NULL != pCmd->insertParam.pTableBlockHashList) { - taosHashCleanup(pCmd->insertParam.pTableBlockHashList); - pCmd->insertParam.pTableBlockHashList = NULL; - } - - pSql->fp = asyncCallback; - pSql->fetchFp = asyncCallback; - pSql->param = pSql; - - registerSqlObj(pSql); - int code = tsParseSql(pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - tsem_wait(&pSql->rspSem); - code = pSql->res.code; - } - - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" invalid SQL result:%d, %s pObj:%p", pSql->self, code, taos_errstr(pSql), pObj); - } - - taos_free_result(pSql); - return code; -} - -void loadMultiTableMetaCallback(void *param, TAOS_RES *res, int code) { - SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)param); - if (pSql == NULL) { - return; - } - - taosReleaseRef(tscObjRef, pSql->self); - pSql->res.code = code; - tsem_post(&pSql->rspSem); -} - -static void freeElem(void* p) { - tfree(*(char**)p); -} - -int taos_load_table_info(TAOS *taos, const char *tableNameList) { - const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024; // 12MB list - - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - int32_t length = (int32_t)strlen(tableNameList); - if (length == 0) { - return TSDB_CODE_SUCCESS; - } - - if (length > MAX_TABLE_NAME_LENGTH) { - tscError("tableNameList too long, length:%d, maximum allowed:%d", length, MAX_TABLE_NAME_LENGTH); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - char *str = calloc(1, length + 1); - if (str == NULL) { - tscError("failed to allocate sql string buffer, size:%d", length); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - strtolower(str, tableNameList); - SArray* plist = taosArrayInit(4, POINTER_BYTES); - if (plist == NULL) { - tfree(str); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SArray* vgroupList = taosArrayInit(4, POINTER_BYTES); - if (vgroupList == NULL) { - taosArrayDestroy(plist); - tfree(str); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - tscAllocPayload(&pSql->cmd, 1024); - - pSql->pTscObj = taos; - pSql->signature = pSql; - - int32_t code = (uint8_t) tscTransferTableNameList(pSql, str, length, plist); - free(str); - - if (code != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pSql); - taosArrayDestroyEx(plist, freeElem); - taosArrayDestroyEx(vgroupList, freeElem); - return code; - } - - pSql->cmd.pTableMetaMap = taosHashInit(taosArrayGetSize(plist), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - registerSqlObj(pSql); - tscDebug("0x%"PRIx64" load multiple table meta, tableNameList: %s pObj:%p", pSql->self, tableNameList, pObj); - - code = getMultiTableMetaFromMnode(pSql, plist, vgroupList, NULL, loadMultiTableMetaCallback, false); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - code = TSDB_CODE_SUCCESS; - } - - taosArrayDestroyEx(plist, freeElem); - taosArrayDestroyEx(vgroupList, freeElem); - - if (code != TSDB_CODE_SUCCESS) { - tscFreeRegisteredSqlObj(pSql); - return code; - } - - tsem_wait(&pSql->rspSem); - tscFreeRegisteredSqlObj(pSql); - return code; -} diff --git a/2.0/src/client/src/tscStream.c b/2.0/src/client/src/tscStream.c deleted file mode 100644 index c8a305e804d5fcd8db1e0f7af919022d012ac63e..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscStream.c +++ /dev/null @@ -1,776 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tmsg.h" -#include "tscLog.h" -#include "tscUtil.h" -#include "tsched.h" -#include "tcache.h" -#include "tsclient.h" -#include "ttimer.h" -#include "tutil.h" - -#include "tscProfile.h" -#include "tscSubquery.h" - -static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOfRows); -static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOfRows); -static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql); -static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer); - -static int64_t getDelayValueAfterTimewindowClosed(SSqlStream* pStream, int64_t launchDelay) { - return taosGetTimestamp(pStream->precision) + launchDelay - pStream->stime - 1; -} - -static bool isProjectStream(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - SExprInfo *pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId != TSDB_FUNC_PRJ) { - return false; - } - } - - return true; -} - -static int64_t tscGetRetryDelayTime(SSqlStream* pStream, int64_t slidingTime, int16_t prec) { - float retryRangeFactor = 0.3f; - int64_t retryDelta = (int64_t)(tsRetryStreamCompDelay * retryRangeFactor); - retryDelta = ((rand() % retryDelta) + tsRetryStreamCompDelay) * 1000L; - - if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') { - // change to ms - slidingTime = convertTimePrecision(slidingTime, pStream->precision, TSDB_TIME_PRECISION_MILLI); - - if (slidingTime < retryDelta) { - return slidingTime; - } - } - - return retryDelta; -} - -static void setRetryInfo(SSqlStream* pStream, int32_t code) { - SSqlObj* pSql = pStream->pSql; - - pSql->res.code = code; - int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision); - tscDebug("0x%"PRIx64" stream:%p, get table Meta failed, retry in %" PRId64 "ms", pSql->self, pStream, retryDelayTime); - tscSetRetryTimer(pStream, pSql, retryDelayTime); -} - -static void doLaunchQuery(void* param, TAOS_RES* tres, int32_t code) { - SSqlStream *pStream = (SSqlStream *)param; - assert(pStream->pSql == tres); - - SSqlObj* pSql = (SSqlObj*) tres; - - pSql->fp = doLaunchQuery; - pSql->fetchFp = doLaunchQuery; - pSql->res.completed = false; - - if (code != TSDB_CODE_SUCCESS) { - setRetryInfo(pStream, code); - return; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - code = tscGetTableMeta(pSql, pTableMetaInfo); - if (code == 0 && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - code = tscGetSTableVgroupInfo(pSql, pQueryInfo); - } - - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - return; - } - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) && (pTableMetaInfo->pVgroupTables == NULL) && (pTableMetaInfo->vgroupList == NULL || pTableMetaInfo->vgroupList->numOfVgroups <= 0)) { - tscDebug("0x%"PRIx64" empty vgroup list", pSql->self); - pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); - code = TSDB_CODE_TSC_APP_ERROR; - } - - // failed to get table Meta or vgroup list, retry in 10sec. - if (code == TSDB_CODE_SUCCESS) { - tscTansformFuncForSTableQuery(pQueryInfo); - - tscDebug("0x%"PRIx64" stream:%p, start stream query on:%s QueryInfo->skey=%"PRId64" ekey=%"PRId64" ", pSql->self, pStream, tNameGetTableName(&pTableMetaInfo->name), pQueryInfo->window.skey, pQueryInfo->window.ekey); - - pQueryInfo->command = TSDB_SQL_SELECT; - - pSql->fp = tscProcessStreamQueryCallback; - pSql->fetchFp = tscProcessStreamQueryCallback; - executeQuery(pSql, pQueryInfo); - tscIncStreamExecutionCount(pStream); - } else { - setRetryInfo(pStream, code); - } -} - -static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { - SSqlStream *pStream = (SSqlStream *)pMsg->ahandle; - doLaunchQuery(pStream, pStream->pSql, 0); -} - -static void tscProcessStreamTimer(void *handle, void *tmrId) { - SSqlStream *pStream = (SSqlStream *)handle; - if (pStream == NULL || pStream->pTimer != tmrId) { - return; - } - - pStream->pTimer = NULL; - - pStream->numOfRes = 0; // reset the numOfRes. - SSqlObj *pSql = pStream->pSql; - - // pSql == NULL maybe killStream already called - if(pSql == NULL) { - return ; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - tscDebug("0x%"PRIx64" add into timer", pSql->self); - - if (pStream->isProject) { - /* - * pQueryInfo->window.ekey, which is the start time, does not change in case of - * repeat first execution, once the first execution failed. - */ - pQueryInfo->window.skey = pStream->stime; // start time - - pQueryInfo->window.ekey = taosGetTimestamp(pStream->precision); // end time - if (pQueryInfo->window.ekey > pStream->etime) { - pQueryInfo->window.ekey = pStream->etime; - } - } else { - pQueryInfo->window.skey = pStream->stime; - int64_t etime = taosGetTimestamp(pStream->precision); - // delay to wait all data in last time window - etime -= convertTimePrecision(tsMaxStreamComputDelay, TSDB_TIME_PRECISION_MILLI, pStream->precision); - if (etime > pStream->etime) { - etime = pStream->etime; - } else if (pStream->interval.intervalUnit != 'y' && pStream->interval.intervalUnit != 'n') { - if(pStream->stime == INT64_MIN) { - etime = taosTimeTruncate(etime, &pStream->interval, pStream->precision); - } else { - etime = pStream->stime + (etime - pStream->stime) / pStream->interval.interval * pStream->interval.interval; - } - } else { - etime = taosTimeTruncate(etime, &pStream->interval, pStream->precision); - } - pQueryInfo->window.ekey = etime; - if (pQueryInfo->window.skey >= pQueryInfo->window.ekey) { - int64_t timer = pStream->interval.sliding; - if (pStream->interval.intervalUnit == 'y' || pStream->interval.intervalUnit == 'n') { - timer = 86400 * 1000l; - } else { - timer = convertTimePrecision(timer, pStream->precision, TSDB_TIME_PRECISION_MILLI); - } - tscSetRetryTimer(pStream, pSql, timer); - return; - } - } - - // launch stream computing in a new thread - SSchedMsg schedMsg = {0}; - schedMsg.fp = tscProcessStreamLaunchQuery; - schedMsg.ahandle = pStream; - schedMsg.thandle = (void *)1; - schedMsg.msg = NULL; - taosScheduleTask(tscQhandle, &schedMsg); -} - -static void cbParseSql(void* param, TAOS_RES* res, int code); - -static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOfRows) { - SSqlStream *pStream = (SSqlStream *)param; - if (tres == NULL || numOfRows < 0) { - int64_t retryDelay = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision); - tscError("0x%"PRIx64" stream:%p, query data failed, code:0x%08x, retry in %" PRId64 "ms", pStream->pSql->self, - pStream, numOfRows, retryDelay); - - SSqlObj* pSql = pStream->pSql; - - tscFreeSqlResult(pSql); - tscFreeSubobj(pSql); - tfree(pSql->pSubs); - pSql->subState.numOfSub = 0; - - int32_t code = tsParseSql(pSql, true); - if (code == TSDB_CODE_SUCCESS) { - cbParseSql(pStream, pSql, code); - } else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - tscDebug("0x%"PRIx64" CQ taso_open_stream IN Process", pSql->self); - } else { - tscError("0x%"PRIx64" open stream failed, code:%s", pSql->self, tstrerror(code)); - taosReleaseRef(tscObjRef, pSql->self); - free(pStream); - } - -// tscSetRetryTimer(pStream, pStream->pSql, retryDelay); -// return; - } - - taos_fetch_rows_a(tres, tscProcessStreamRetrieveResult, param); -} - -// no need to be called as this is alreay done in the query -static void tscStreamFillTimeGap(SSqlStream* pStream, TSKEY ts) { -#if 0 - SSqlObj * pSql = pStream->pSql; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - if (pQueryInfo->fillType != TSDB_FILL_SET_VALUE && pQueryInfo->fillType != TSDB_FILL_NULL) { - return; - } - - SSqlRes *pRes = &pSql->res; - /* failed to retrieve any result in this retrieve */ - pSql->res.numOfRows = 1; - void *row[TSDB_MAX_COLUMNS] = {0}; - char tmpRes[TSDB_MAX_BYTES_PER_ROW] = {0}; - void *oldPtr = pSql->res.data; - pSql->res.data = tmpRes; - int32_t rowNum = 0; - - while (pStream->stime + pStream->slidingTime < ts) { - pStream->stime += pStream->slidingTime; - *(TSKEY*)row[0] = pStream->stime; - for (int32_t i = 1; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i); - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - assignVal(pSql->res.data + offset, (char *)(&pQueryInfo->fillVal[i]), pField->bytes, pField->type); - row[i] = pSql->res.data + offset; - } - (*pStream->fp)(pStream->param, pSql, row); - ++rowNum; - } - - if (rowNum > 0) { - tscDebug("0x%"PRIx64" stream:%p %d rows padded", pSql, pStream, rowNum); - } - - pRes->numOfRows = 0; - pRes->data = oldPtr; -#endif -} - -static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOfRows) { - SSqlStream * pStream = (SSqlStream *)param; - SSqlObj * pSql = (SSqlObj *)res; - - if (pSql == NULL || numOfRows < 0) { - int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision); - tscError("stream:%p, retrieve data failed, code:0x%08x, retry in %" PRId64 " ms", pStream, numOfRows, retryDelayTime); - - tscSetRetryTimer(pStream, pStream->pSql, retryDelayTime); - return; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMetaInfo *pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - - if (numOfRows > 0) { // when reaching here the first execution of stream computing is successful. - for(int32_t i = 0; i < numOfRows; ++i) { - TAOS_ROW row = taos_fetch_row(res); - if (row != NULL) { - tscDebug("0x%"PRIx64" stream:%p fetch result", pSql->self, pStream); - tscStreamFillTimeGap(pStream, *(TSKEY*)row[0]); - pStream->stime = *(TSKEY *)row[0]; - // user callback function - (*pStream->fp)(pStream->param, res, row); - pStream->numOfRes++; - } - } - - if (!pStream->isProject) { - pStream->stime = taosTimeAdd(pStream->stime, pStream->interval.sliding, pStream->interval.slidingUnit, pStream->precision); - } - // actually only one row is returned. this following is not necessary - taos_fetch_rows_a(res, tscProcessStreamRetrieveResult, pStream); - } else { // numOfRows == 0, all data has been retrieved - pStream->useconds += pSql->res.useconds; - if (pStream->numOfRes == 0) { - if (pStream->isProject) { - /* no resuls in the query range, retry */ - // todo set retry dynamic time - int32_t retry = tsProjectExecInterval; - tscError("0x%"PRIx64" stream:%p, retrieve no data, code:0x%08x, retry in %" PRId32 "ms", pSql->self, pStream, numOfRows, retry); - - tscSetRetryTimer(pStream, pStream->pSql, retry); - return; - } - } else if (pStream->isProject) { - pStream->stime += 1; - } - - tscDebug("0x%"PRIx64" stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql->self, pStream, tNameGetTableName(&pTableMetaInfo->name), - pStream->numOfRes); - - tfree(pTableMetaInfo->pTableMeta); - if (pQueryInfo->pQInfo != NULL) { - qDestroyQueryInfo(pQueryInfo->pQInfo); - pQueryInfo->pQInfo = NULL; - } - - tscFreeSqlResult(pSql); - tscFreeSubobj(pSql); - tfree(pSql->pSubs); - pSql->subState.numOfSub = 0; - pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); - tscSetNextLaunchTimer(pStream, pSql); - } -} - -static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer) { - int64_t delay = getDelayValueAfterTimewindowClosed(pStream, timer); - - if (pStream->isProject) { - int64_t now = taosGetTimestamp(pStream->precision); - int64_t etime = now > pStream->etime ? pStream->etime : now; - int64_t maxRetent = tsMaxRetentWindow * 1000; - if(pStream->precision == TSDB_TIME_PRECISION_MICRO) { - maxRetent *= 1000; - } - - if (pStream->etime < now && now - pStream->etime > maxRetent) { - /* - * current time window will be closed, since it too early to exceed the maxRetentWindow value - */ - tscDebug("0x%"PRIx64" stream:%p, etime:%" PRId64 " is too old, exceeds the max retention time window:%" PRId64 ", stop the stream", - pStream->pSql->self, pStream, pStream->stime, pStream->etime); - // TODO : How to terminate stream here - if (pStream->callback) { - // Callback function from upper level - pStream->callback(pStream->param); - } - taos_close_stream(pStream); - return; - } - - tscDebug("0x%"PRIx64" stream:%p, next start at %" PRId64 "(ts window ekey), in %" PRId64 " ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql->self, pStream, - now + timer, timer, delay, pStream->stime, etime); - } else { - tscDebug("0x%"PRIx64" stream:%p, next start at %" PRId64 " - %" PRId64 " end, in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql->self, pStream, - pStream->stime, pStream->etime, timer, delay, pStream->stime - pStream->interval.interval, pStream->stime - 1); - } - - pSql->cmd.command = TSDB_SQL_SELECT; - - // start timer for next computing - taosTmrReset(tscProcessStreamTimer, (int32_t)timer, pStream, tscTmr, &pStream->pTimer); -} - -static int64_t getLaunchTimeDelay(const SSqlStream* pStream) { - int64_t maxDelay = convertTimePrecision(tsMaxStreamComputDelay, TSDB_TIME_PRECISION_MILLI, pStream->precision); - - int64_t delayDelta = maxDelay; - if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') { - delayDelta = (int64_t)(pStream->interval.sliding * tsStreamComputDelayRatio); - if (delayDelta > maxDelay) { - delayDelta = maxDelay; - } - int64_t remainTimeWindow = pStream->interval.sliding - delayDelta; - if (maxDelay > remainTimeWindow) { - maxDelay = (int64_t)(remainTimeWindow / 1.5f); - } - } - - int64_t currentDelay = (rand() % maxDelay); // a random number - currentDelay += delayDelta; - if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') { - assert(currentDelay < pStream->interval.sliding); - } - - return currentDelay; -} - - -static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) { - int64_t timer = 0; - - if (pStream->isProject) { - /* - * for project query, no mater fetch data successfully or not, next launch will issue - * more than the sliding time window - */ - timer = pStream->interval.sliding; - if (pStream->stime > pStream->etime) { - tscDebug("0x%"PRIx64" stream:%p, stime:%" PRId64 " is larger than end time: %" PRId64 ", stop the stream", - pStream->pSql->self, pStream, pStream->stime, pStream->etime); - // TODO : How to terminate stream here - if (pStream->callback) { - // Callback function from upper level - pStream->callback(pStream->param); - } - taos_close_stream(pStream); - return; - } - } else { - int64_t stime = taosTimeTruncate(pStream->stime - 1, &pStream->interval, pStream->precision); - if (stime >= pStream->etime) { - tscDebug("0x%"PRIx64" stream:%p, stime:%" PRId64 " is larger than end time: %" PRId64 ", stop the stream", pStream->pSql->self, pStream, - pStream->stime, pStream->etime); - // TODO : How to terminate stream here - if (pStream->callback) { - // Callback function from upper level - pStream->callback(pStream->param); - } - taos_close_stream(pStream); - return; - } - - if (pStream->stime > 0) { - timer = pStream->stime - taosGetTimestamp(pStream->precision); - if (timer < 0) { - timer = 0; - } - } - } - - timer += getLaunchTimeDelay(pStream); - - timer = convertTimePrecision(timer, pStream->precision, TSDB_TIME_PRECISION_MILLI); - - tscSetRetryTimer(pStream, pSql, timer); -} - -static int32_t tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { - int64_t minIntervalTime = - convertTimePrecision(tsMinIntervalTime, TSDB_TIME_PRECISION_MILLI, pStream->precision); - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - if (!pStream->isProject && pQueryInfo->interval.interval == 0) { - sprintf(pSql->cmd.payload, "the interval value is 0"); - return -1; - } - - if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit!= 'y' && pQueryInfo->interval.interval < minIntervalTime) { - tscWarn("0x%"PRIx64" stream:%p, original sample interval:%" PRId64 " too small, reset to:%" PRId64, pSql->self, pStream, - (int64_t)pQueryInfo->interval.interval, minIntervalTime); - pQueryInfo->interval.interval = minIntervalTime; - } - - pStream->interval.intervalUnit = pQueryInfo->interval.intervalUnit; - pStream->interval.interval = pQueryInfo->interval.interval; // it shall be derived from sql string - - if (pQueryInfo->interval.sliding <= 0) { - pQueryInfo->interval.sliding = pQueryInfo->interval.interval; - pQueryInfo->interval.slidingUnit = pQueryInfo->interval.intervalUnit; - } - - int64_t minSlidingTime = - convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, pStream->precision); - - if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit!= 'y' && pQueryInfo->interval.sliding < minSlidingTime) { - tscWarn("0x%"PRIx64" stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64, pSql->self, pStream, - pQueryInfo->interval.sliding, minSlidingTime); - - pQueryInfo->interval.sliding = minSlidingTime; - } - - if (pQueryInfo->interval.sliding > pQueryInfo->interval.interval) { - tscWarn("0x%"PRIx64" stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64, pSql->self, pStream, - pQueryInfo->interval.sliding, pQueryInfo->interval.interval); - - pQueryInfo->interval.sliding = pQueryInfo->interval.interval; - } - - pStream->interval.slidingUnit = pQueryInfo->interval.slidingUnit; - pStream->interval.sliding = pQueryInfo->interval.sliding; - - if (pStream->isProject) { - pQueryInfo->interval.interval = 0; // clear the interval value to avoid the force time window split by query processor - pQueryInfo->interval.sliding = 0; - } - - return TSDB_CODE_SUCCESS; -} - -static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, int64_t stime) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - if (pStream->isProject) { - // no data in table, flush all data till now to destination meter, 10sec delay - pStream->interval.interval = tsProjectExecInterval; - pStream->interval.sliding = tsProjectExecInterval; - - if (stime != INT64_MIN) { // first projection start from the latest event timestamp - assert(stime >= pQueryInfo->window.skey); - stime += 1; // exclude the last records from table - } else { - stime = pQueryInfo->window.skey; - } - } else { // timewindow based aggregation stream - if (stime == INT64_MIN) { // no data in meter till now - if (pQueryInfo->window.skey != INT64_MIN) { - stime = pQueryInfo->window.skey; - } else { - return stime; - } - - stime = taosTimeTruncate(stime, &pStream->interval, pStream->precision); - } else { - int64_t newStime = taosTimeTruncate(stime, &pStream->interval, pStream->precision); - if (newStime != stime) { - tscWarn("0x%"PRIx64" stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64, pSql->self, pStream, stime, newStime); - stime = newStime; - } - } - } - - return stime; -} - -static int64_t tscGetLaunchTimestamp(const SSqlStream *pStream) { - int64_t timer = 0, now = taosGetTimestamp(pStream->precision); - if (pStream->stime > now) { - timer = pStream->stime - now; - } - - int64_t startDelay = convertTimePrecision(tsStreamCompStartDelay, TSDB_TIME_PRECISION_MILLI, pStream->precision); - - timer += getLaunchTimeDelay(pStream); - timer += startDelay; - - return convertTimePrecision(timer, pStream->precision, TSDB_TIME_PRECISION_MILLI); -} - -static void tscCreateStream(void *param, TAOS_RES *res, int code) { - SSqlStream* pStream = (SSqlStream*)param; - SSqlObj* pSql = pStream->pSql; - SSqlCmd* pCmd = &pSql->cmd; - - if (code != TSDB_CODE_SUCCESS) { - pSql->res.code = code; - tscError("0x%"PRIx64" open stream failed, sql:%s, reason:%s, code:%s", pSql->self, pSql->sqlstr, pCmd->payload, tstrerror(code)); - pStream->fp(pStream->param, NULL, NULL); - return; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - pStream->isProject = isProjectStream(pQueryInfo); - pStream->precision = tinfo.precision; - - pStream->ctime = taosGetTimestamp(pStream->precision); - pStream->etime = pQueryInfo->window.ekey; - - if (tscSetSlidingWindowInfo(pSql, pStream) != TSDB_CODE_SUCCESS) { - pSql->res.code = code; - - tscError("0x%"PRIx64" stream %p open failed, since the interval value is incorrect", pSql->self, pStream); - pStream->fp(pStream->param, NULL, NULL); - return; - } - - pStream->stime = tscGetStreamStartTimestamp(pSql, pStream, pStream->stime); - - // set stime with ltime if ltime > stime - const char* dstTable = pStream->dstTable? pStream->dstTable: ""; - tscDebug("0x%"PRIx64" CQ table %s ltime is %"PRId64, pSql->self, dstTable, pStream->ltime); - - if(pStream->ltime != INT64_MIN && pStream->ltime > pStream->stime) { - tscWarn("0x%"PRIx64" CQ set stream %s stime=%"PRId64" replace with ltime=%"PRId64" if ltime > 0", pSql->self, dstTable, pStream->stime, pStream->ltime); - pStream->stime = pStream->ltime; - } - - int64_t starttime = tscGetLaunchTimestamp(pStream); - pCmd->command = TSDB_SQL_SELECT; - - tscAddIntoStreamList(pStream); - taosTmrReset(tscProcessStreamTimer, (int32_t)starttime, pStream, tscTmr, &pStream->pTimer); - - tscDebug("0x%"PRIx64" stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql->self, - pStream, tNameGetTableName(&pTableMetaInfo->name), pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr); -} - -void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable) { - pStream->dstTable = dstTable; -} - -// fetchFp call back -void fetchFpStreamLastRow(void* param ,TAOS_RES* res, int num) { - SSqlStream* pStream = (SSqlStream*)param; - SSqlObj* pSql = res; - - // get row data set to ltime - tscSetSqlOwner(pSql); - TAOS_ROW row = doSetResultRowData(pSql); - if( row && row[0] ) { - pStream->ltime = *((int64_t*)row[0]); - const char* dstTable = pStream->dstTable? pStream->dstTable: ""; - tscDebug(" CQ stream table=%s last row time=%"PRId64" .", dstTable, pStream->ltime); - } - tscClearSqlOwner(pSql); - - // no condition call - tscCreateStream(param, pStream->pSql, TSDB_CODE_SUCCESS); - taos_free_result(res); -} - -// fp callback -void fpStreamLastRow(void* param ,TAOS_RES* res, int code) { - // check result successful - if (code != TSDB_CODE_SUCCESS) { - tscCreateStream(param, res, TSDB_CODE_SUCCESS); - taos_free_result(res); - return ; - } - - // asynchronous fetch last row data - taos_fetch_rows_a(res, fetchFpStreamLastRow, param); -} - -void cbParseSql(void* param, TAOS_RES* res, int code) { - // check result successful - SSqlStream* pStream = (SSqlStream*)param; - SSqlObj* pSql = pStream->pSql; - SSqlCmd* pCmd = &pSql->cmd; - if (code != TSDB_CODE_SUCCESS) { - pSql->res.code = code; - tscDebug("0x%"PRIx64" open stream parse sql failed, sql:%s, reason:%s, code:%s", pSql->self, pSql->sqlstr, pCmd->payload, tstrerror(code)); - pStream->fp(pStream->param, NULL, NULL); - return; - } - - // check dstTable valid - if(pStream->dstTable == NULL || strlen(pStream->dstTable) == 0) { - tscDebug(" cbParseSql dstTable is empty."); - tscCreateStream(param, res, code); - return ; - } - - // query stream last row time async - char sql[128] = ""; - sprintf(sql, "select last_row(*) from %s;", pStream->dstTable); - taos_query_a(pSql->pTscObj, sql, fpStreamLastRow, param); -} - -TAOS_STREAM *taos_open_stream_withname(TAOS *taos, const char* dstTable, const char *sqlstr, void (*fp)(void *, TAOS_RES *, TAOS_ROW), - int64_t stime, void *param, void (*callback)(void *), void* cqhandle) { - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) return NULL; - - if(fp == NULL){ - tscError(" taos_open_stream api fp param must not NULL."); - return NULL; - } - - SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - return NULL; - } - - pSql->signature = pSql; - pSql->pTscObj = pObj; - - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - SSqlStream *pStream = (SSqlStream *)calloc(1, sizeof(SSqlStream)); - if (pStream == NULL) { - tscError("0x%"PRIx64" open stream failed, sql:%s, reason:%s, code:0x%08x", pSql->self, sqlstr, pCmd->payload, pRes->code); - tscFreeSqlObj(pSql); - return NULL; - } - - pStream->ltime = INT64_MIN; - pStream->stime = stime; - pStream->fp = fp; - pStream->callback = callback; - pStream->param = param; - pStream->pSql = pSql; - pStream->cqhandle = cqhandle; - tscSetStreamDestTable(pStream, dstTable); - - pSql->pStream = pStream; - pSql->param = pStream; - pSql->maxRetry = TSDB_MAX_REPLICA; - - pSql->sqlstr = calloc(1, strlen(sqlstr) + 1); - if (pSql->sqlstr == NULL) { - tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self); - tscFreeSqlObj(pSql); - free(pStream); - return NULL; - } - - strtolower(pSql->sqlstr, sqlstr); - pSql->fp = tscCreateStream; - pSql->fetchFp = tscCreateStream; - pSql->cmd.resColumnId = TSDB_RES_COL_ID; - - tsem_init(&pSql->rspSem, 0, 0); - registerSqlObj(pSql); - - tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); - - pSql->fp = cbParseSql; - pSql->fetchFp = cbParseSql; - registerSqlObj(pSql); - - int32_t code = tsParseSql(pSql, true); - if (code == TSDB_CODE_SUCCESS) { - cbParseSql(pStream, pSql, code); - } else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - tscDebug("0x%"PRIx64" CQ taso_open_stream IN Process", pSql->self); - } else { - tscError("0x%"PRIx64" open stream failed, sql:%s, code:%s", pSql->self, sqlstr, tstrerror(code)); - taosReleaseRef(tscObjRef, pSql->self); - free(pStream); - return NULL; - } - - return pStream; -} - -TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *, TAOS_RES *, TAOS_ROW), - int64_t stime, void *param, void (*callback)(void *)) { - return taos_open_stream_withname(taos, "", sqlstr, fp, stime, param, callback, NULL); -} - -void taos_close_stream(TAOS_STREAM *handle) { - SSqlStream *pStream = (SSqlStream *)handle; - - SSqlObj *pSql = (SSqlObj *)atomic_exchange_ptr(&pStream->pSql, 0); - if (pSql == NULL) { - return; - } - - /* - * stream may be closed twice, 1. drop dst table, 2. kill stream - * Here, we need a check before release memory - */ - if (pSql->signature == pSql) { - tscRemoveFromStreamList(pStream, pSql); - - taosTmrStopA(&(pStream->pTimer)); - - tscDebug("0x%"PRIx64" stream:%p is closed", pSql->self, pStream); - // notify CQ to release the pStream object - pStream->fp(pStream->param, NULL, NULL); - pStream->pSql = NULL; - - taos_free_result(pSql); - tfree(pStream); - } -} diff --git a/2.0/src/client/src/tscSub.c b/2.0/src/client/src/tscSub.c deleted file mode 100644 index 6464224c5e7549f4016a108be66138c437eda36d..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscSub.c +++ /dev/null @@ -1,631 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "../../../include/client/taos.h" -#include "os.h" -#include "tcache.h" -#include "trpc.h" -#include "tscLog.h" -#include "tscProfile.h" -#include "tscUtil.h" -#include "tsclient.h" -#include "tsocket.h" -#include "ttimer.h" -#include "tutil.h" - -typedef struct SSubscriptionProgress { - int64_t uid; - TSKEY key; -} SSubscriptionProgress; - -typedef struct SSub { - void * signature; - char topic[32]; - tsem_t sem; - int64_t lastSyncTime; - int64_t lastConsumeTime; - TAOS * taos; - void * pTimer; - SSqlObj * pSql; - int interval; - TAOS_SUBSCRIBE_CALLBACK fp; - void * param; - SArray* progress; -} SSub; - - -static int tscCompareSubscriptionProgress(const void* a, const void* b) { - const SSubscriptionProgress* x = (const SSubscriptionProgress*)a; - const SSubscriptionProgress* y = (const SSubscriptionProgress*)b; - if (x->uid > y->uid) return 1; - if (x->uid < y->uid) return -1; - return 0; -} - -TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) { - if (sub == NULL) { - return dflt; - } - SSub* pSub = (SSub*)sub; - - SSubscriptionProgress target = {.uid = uid, .key = 0}; - SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ); - if (p == NULL) { - return dflt; - } - return p->key; -} - -void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) { - if( sub == NULL) { - return; - } - - SSub* pSub = (SSub*)sub; - - SSubscriptionProgress target = {.uid = uid, .key = ts}; - SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ); - if (p != NULL) { - p->key = ts; - tscDebug("subscribe:%s, uid:%"PRIu64" update sub start ts:%"PRId64, pSub->topic, p->uid, p->key); - } -} - - -static void asyncCallback(void *param, TAOS_RES *tres, int code) { - assert(param != NULL); - SSub *pSub = ((SSub *)param); - pSub->pSql->res.code = code; - tsem_post(&pSub->sem); -} - - -static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* sql) { - int code = TSDB_CODE_SUCCESS, line = __LINE__; - SSqlObj* pSql = NULL; - - SSub* pSub = calloc(1, sizeof(SSub)); - if (pSub == NULL) { - line = __LINE__; - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto fail; - } - pSub->signature = pSub; - if (tsem_init(&pSub->sem, 0, 0) == -1) { - line = __LINE__; - code = TAOS_SYSTEM_ERROR(errno); - goto fail; - } - - tstrncpy(pSub->topic, topic, sizeof(pSub->topic)); - pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress)); - if (pSub->progress == NULL) { - line = __LINE__; - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto fail; - } - - pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - line = __LINE__; - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto fail; - } - - pSql->signature = pSql; - pSql->pTscObj = pObj; - pSql->pSubscription = pSub; - pSub->pSql = pSql; - - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - if (tsem_init(&pSql->rspSem, 0, 0) == -1) { - line = __LINE__; - code = TAOS_SYSTEM_ERROR(errno); - goto fail; - } - - pSql->param = pSub; - pSql->maxRetry = TSDB_MAX_REPLICA; - pSql->fp = asyncCallback; - pSql->fetchFp = asyncCallback; - pSql->sqlstr = strdup(sql); - if (pSql->sqlstr == NULL) { - line = __LINE__; - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto fail; - } - - strtolower(pSql->sqlstr, pSql->sqlstr); - pRes->qId = 0; - pRes->numOfRows = 1; - pCmd->resColumnId = TSDB_RES_COL_ID; - - code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); - if (code != TSDB_CODE_SUCCESS) { - line = __LINE__; - goto fail; - } - - registerSqlObj(pSql); - - code = tsParseSql(pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - tsem_wait(&pSub->sem); - code = pSql->res.code; - } - - if (code != TSDB_CODE_SUCCESS) { - line = __LINE__; - goto fail; - } - - if (pSql->cmd.command != TSDB_SQL_SELECT && pSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - line = __LINE__; - code = TSDB_CODE_TSC_INVALID_OPERATION; - goto fail; - } - - return pSub; - -fail: - tscError("tscCreateSubscription failed at line %d, reason: %s", line, tstrerror(code)); - if (pSql != NULL) { - if (pSql->self != 0) { - taosReleaseRef(tscObjRef, pSql->self); - } else { - tscFreeSqlObj(pSql); - } - - pSql = NULL; - } - - if (pSub != NULL) { - taosArrayDestroy(pSub->progress); - tsem_destroy(&pSub->sem); - free(pSub); - pSub = NULL; - } - - terrno = code; - return NULL; -} - - -static void tscProcessSubscriptionTimer(void *handle, void *tmrId) { - SSub *pSub = (SSub *)handle; - if (pSub == NULL || pSub->pTimer != tmrId) return; - - TAOS_RES* res = taos_consume(pSub); - if (res != NULL) { - pSub->fp(pSub, res, pSub->param, 0); - } - - taosTmrReset(tscProcessSubscriptionTimer, pSub->interval, pSub, tscTmr, &pSub->pTimer); -} - -//TODO refactor: extract table list name not simply from the sql -static SArray* getTableList( SSqlObj* pSql ) { - const char* p = strstr( pSql->sqlstr, " from " ); - assert(p != NULL); // we are sure this is a 'select' statement - char* sql = alloca(strlen(p) + 32); - sprintf(sql, "select tbid(tbname)%s", p); - - SSqlObj* pNew = taos_query(pSql->pTscObj, sql); - if (pNew == NULL) { - tscError("0x%"PRIx64" failed to retrieve table id: cannot create new sql object.", pSql->self); - return NULL; - - } else if (taos_errno(pNew) != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" failed to retrieve table id,error: %s", pSql->self, tstrerror(taos_errno(pNew))); - return NULL; - } - - TAOS_ROW row; - SArray* result = taosArrayInit( 128, sizeof(STidTags) ); - while ((row = taos_fetch_row(pNew))) { - STidTags tags; - memcpy(&tags, row[0], sizeof(tags)); - taosArrayPush(result, &tags); - } - - taos_free_result(pNew); - - return result; -} - -static int32_t compareTidTag(const void* p1, const void* p2) { - const STidTags* t1 = (const STidTags*)p1; - const STidTags* t2 = (const STidTags*)p2; - - if (t1->vgId != t2->vgId) { - return (t1->vgId > t2->vgId) ? 1 : -1; - } - if (t1->tid != t2->tid) { - return (t1->tid > t2->tid) ? 1 : -1; - } - return 0; -} - - -static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { - SSqlObj* pSql = pSub->pSql; - - SSqlCmd* pCmd = &pSql->cmd; - - TSDB_QUERY_CLEAR_TYPE(tscGetQueryInfo(pCmd)->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = 0}; - SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ); - if (p == NULL) { - taosArrayClear(pSub->progress); - taosArrayPush(pSub->progress, &target); - } - - pSub->lastSyncTime = taosGetTimestampMs(); - return 1; - } - - SArray* tables = getTableList(pSql); - if (tables == NULL) { - pSub->lastSyncTime = 0; //force to get table list next time - return 0; - } - size_t numOfTables = taosArrayGetSize(tables); - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - SArray* progress = taosArrayInit(numOfTables, sizeof(SSubscriptionProgress)); - for( size_t i = 0; i < numOfTables; i++ ) { - STidTags* tt = taosArrayGet( tables, i ); - SSubscriptionProgress p = { .uid = tt->uid }; - p.key = tscGetSubscriptionProgress(pSub, tt->uid, pQueryInfo->window.skey); - taosArrayPush(progress, &p); - } - taosArraySort(progress, tscCompareSubscriptionProgress); - - taosArrayDestroy(pSub->progress); - pSub->progress = progress; - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - taosArraySort( tables, compareTidTag ); - tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); - tscBuildVgroupTableInfo(pSql, pTableMetaInfo, tables); - } - taosArrayDestroy(tables); - - if (pTableMetaInfo->pVgroupTables && taosArrayGetSize(pTableMetaInfo->pVgroupTables) > 0) { - TSDB_QUERY_SET_TYPE(tscGetQueryInfo(pCmd)->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); - } - - pSub->lastSyncTime = taosGetTimestampMs(); - return 1; -} - - -static int tscLoadSubscriptionProgress(SSub* pSub) { - char buf[TSDB_MAX_SQL_LEN]; - sprintf(buf, "%s/subscribe/%s", tsDataDir, pSub->topic); - - FILE* fp = fopen(buf, "rb"); - if (fp == NULL) { - tscDebug("subscription progress file does not exist: %s", pSub->topic); - return 1; - } - - if (fgets(buf, sizeof(buf), fp) == NULL) { - tscDebug("invalid subscription progress file: %s", pSub->topic); - fclose(fp); - return 0; - } - - for (int i = 0; i < sizeof(buf); i++) { - if (buf[i] == 0) - break; - if (buf[i] == '\r' || buf[i] == '\n') { - buf[i] = 0; - break; - } - } - if (strcmp(buf, pSub->pSql->sqlstr) != 0) { - tscDebug("subscription sql statement mismatch: %s", pSub->topic); - fclose(fp); - return 0; - } - - SArray* progress = pSub->progress; - taosArrayClear(progress); - while( 1 ) { - if (fgets(buf, sizeof(buf), fp) == NULL) { - fclose(fp); - return 0; - } - SSubscriptionProgress p; - sscanf(buf, "%" SCNd64 ":%" SCNd64, &p.uid, &p.key); - taosArrayPush(progress, &p); - } - - fclose(fp); - - taosArraySort(progress, tscCompareSubscriptionProgress); - tscDebug("subscription progress loaded, %" PRIzu " tables: %s", taosArrayGetSize(progress), pSub->topic); - return 1; -} - -void tscSaveSubscriptionProgress(void* sub) { - SSub* pSub = (SSub*)sub; - - char path[256]; - sprintf(path, "%s/subscribe", tsDataDir); - if (taosMkDir(path) != 0) { - tscError("failed to create subscribe dir: %s", path); - } - - sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic); - FILE* fp = fopen(path, "wb+"); - if (fp == NULL) { - tscError("failed to create progress file for subscription: %s", pSub->topic); - return; - } - - fputs(pSub->pSql->sqlstr, fp); - fprintf(fp, "\n"); - for(size_t i = 0; i < taosArrayGetSize(pSub->progress); i++) { - SSubscriptionProgress* p = taosArrayGet(pSub->progress, i); - fprintf(fp, "%" PRId64 ":%" PRId64 "\n", p->uid, p->key); - } - - fclose(fp); -} - -TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) { - STscObj* pObj = (STscObj*)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - tscError("connection disconnected"); - return NULL; - } - - SSub* pSub = tscCreateSubscription(pObj, topic, sql); - if (pSub == NULL) { - return NULL; - } - pSub->taos = taos; - - if (restart) { - tscDebug("restart subscription: %s", topic); - } else { - tscLoadSubscriptionProgress(pSub); - } - - if (pSub->pSql->cmd.command == TSDB_SQL_SELECT) { - if (!tscUpdateSubscription(pObj, pSub)) { - taos_unsubscribe(pSub, 1); - return NULL; - } - } - - pSub->interval = interval; - if (fp != NULL) { - tscDebug("asynchronize subscription, create new timer: %s", topic); - pSub->fp = fp; - pSub->param = param; - taosTmrReset(tscProcessSubscriptionTimer, interval, pSub, tscTmr, &pSub->pTimer); - } - - return pSub; -} - -SSqlObj* recreateSqlObj(SSub* pSub) { - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - return NULL; - } - - pSql->signature = pSql; - pSql->pTscObj = pSub->taos; - - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - if (tsem_init(&pSql->rspSem, 0, 0) == -1) { - tscFreeSqlObj(pSql); - return NULL; - } - - pSql->param = pSub; - pSql->maxRetry = TSDB_MAX_REPLICA; - pSql->fp = asyncCallback; - pSql->fetchFp = asyncCallback; - pSql->sqlstr = strdup(pSub->pSql->sqlstr); - if (pSql->sqlstr == NULL) { - tscFreeSqlObj(pSql); - return NULL; - } - - pRes->qId = 0; - pRes->numOfRows = 1; - - int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); - if (code != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pSql); - return NULL; - } - - registerSqlObj(pSql); - - code = tsParseSql(pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - tsem_wait(&pSub->sem); - code = pSql->res.code; - } - - if (code != TSDB_CODE_SUCCESS) { - taosReleaseRef(tscObjRef, pSql->self); - return NULL; - } - - if (pSql->cmd.command != TSDB_SQL_SELECT) { - taosReleaseRef(tscObjRef, pSql->self); - return NULL; - } - - return pSql; -} - -TAOS_RES *taos_consume(TAOS_SUB *tsub) { - SSub *pSub = (SSub *)tsub; - if (pSub == NULL) return NULL; - - if (pSub->pTimer == NULL) { - int64_t duration = taosGetTimestampMs() - pSub->lastConsumeTime; - if (duration < (int64_t)(pSub->interval)) { - tscDebug("subscription consume too frequently, blocking..."); - taosMsleep(pSub->interval - (int32_t)duration); - } - } - - if (pSub->pSql->cmd.command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { //may reach here when retireve stable vgroup failed - SSqlObj* pSql = recreateSqlObj(pSub); - if (pSql == NULL) { - return NULL; - } - - if (pSub->pSql->self != 0) { - taosReleaseRef(tscObjRef, pSub->pSql->self); - } else { - tscFreeSqlObj(pSub->pSql); - } - - pSub->pSql = pSql; - pSql->pSubscription = pSub; - pSub->lastSyncTime = 0; - - // no table list now, force to update it - tscDebug("begin table synchronization"); - if (!tscUpdateSubscription(pSub->taos, pSub)) return NULL; - tscDebug("table synchronization completed"); - } - - tscSaveSubscriptionProgress(pSub); - - SSqlObj *pSql = pSub->pSql; - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single table subscription - - size_t size = taosArrayGetSize(pSub->progress); - TSKEY s = INT64_MAX; - for(int32_t i = 0; i < size; ++i) { - TSKEY k = ((SSubscriptionProgress*)taosArrayGet(pSub->progress, i))->key; - if (s > k) { - s = k; - } - } - - pQueryInfo->window.skey = s; - tscDebug("subscribe:%s set next round subscribe skey:%"PRId64, pSub->topic, pQueryInfo->window.skey); - } - - size_t size = taosArrayGetSize(pSub->progress) * sizeof(STableIdInfo); - size += sizeof(SQueryTableMsg) + 4096; - int code = tscAllocPayload(&pSql->cmd, (int)size); - if (code != TSDB_CODE_SUCCESS) { - tscError("failed to alloc payload"); - return NULL; - } - - for (int retry = 0; retry < 3; retry++) { - tscRemoveFromSqlList(pSql); - - if (taosGetTimestampMs() - pSub->lastSyncTime > 10 * 60 * 1000) { - tscDebug("begin table synchronization"); - if (!tscUpdateSubscription(pSub->taos, pSub)) return NULL; - tscDebug("table synchronization completed"); - } - - uint32_t type = pQueryInfo->type; - tscFreeSqlResult(pSql); - pRes->numOfRows = 1; - pRes->qId = 0; - pSql->cmd.command = TSDB_SQL_SELECT; - pQueryInfo->type = type; - - pTableMetaInfo->vgroupIndex = 0; - - pSql->fp = asyncCallback; - pSql->fetchFp = asyncCallback; - pSql->param = pSub; - - pSql->cmd.active = pQueryInfo; - executeQuery(pSql, pQueryInfo); - - tsem_wait(&pSub->sem); - - if (pRes->code != TSDB_CODE_SUCCESS) { - continue; - } - // meter was removed, make sync time zero, so that next retry will - // do synchronization first - pSub->lastSyncTime = 0; - break; - } - - if (pRes->code != TSDB_CODE_SUCCESS) { - tscError("failed to query data: %s", tstrerror(pRes->code)); - tscRemoveFromSqlList(pSql); - return NULL; - } - - pSub->lastConsumeTime = taosGetTimestampMs(); - return pSql; -} - -void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { - SSub *pSub = (SSub *)tsub; - if (pSub == NULL || pSub->signature != pSub) return; - - if (pSub->pTimer != NULL) { - taosTmrStop(pSub->pTimer); - } - - if (keepProgress) { - if (pSub->progress != NULL) { - tscSaveSubscriptionProgress(pSub); - } - } else { - char path[256]; - sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic); - if (remove(path) != 0) { - tscError("failed to remove progress file, topic = %s, error = %s", pSub->topic, strerror(errno)); - } - } - - if (pSub->pSql != NULL) { - if (pSub->pSql->self != 0) { - taosReleaseRef(tscObjRef, pSub->pSql->self); - } else { - tscFreeSqlObj(pSub->pSql); - } - } - - taosArrayDestroy(pSub->progress); - tsem_destroy(&pSub->sem); - memset(pSub, 0, sizeof(*pSub)); - free(pSub); -} diff --git a/2.0/src/client/src/tscSubquery.c b/2.0/src/client/src/tscSubquery.c deleted file mode 100644 index 8ed7eb08e67141d94744a9a7c7ec27d55b1452f8..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscSubquery.c +++ /dev/null @@ -1,3737 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#define _GNU_SOURCE - -#include "os.h" -#include "texpr.h" - -#include "tsched.h" -#include "qTsbuf.h" -#include "tcompare.h" -#include "tscLog.h" -#include "tscSubquery.h" -#include "qTableMeta.h" -#include "tsclient.h" -#include "qUdf.h" -#include "qUtil.h" -#include "qPlan.h" - -typedef struct SInsertSupporter { - SSqlObj* pSql; - int32_t index; -} SInsertSupporter; - -static void freeJoinSubqueryObj(SSqlObj* pSql); -//static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql); - -static int32_t tsCompare(int32_t order, int64_t left, int64_t right) { - if (left == right) { - return 0; - } - - if (order == TSDB_ORDER_ASC) { - return left < right? -1:1; - } else { - return left > right? -1:1; - } -} - -static void skipRemainValue(STSBuf* pTSBuf, tVariant* tag1) { - STSElem el1 = tsBufGetElem(pTSBuf); - - int32_t res = tVariantCompare(el1.tag, tag1); - if (res != 0) { // it is a record with new tag - return; - } - - while (tsBufNextPos(pTSBuf)) { - el1 = tsBufGetElem(pTSBuf); - - res = tVariantCompare(el1.tag, tag1); - if (res != 0) { // it is a record with new tag - return; - } - } -} - -static void subquerySetState(SSqlObj *pSql, SSubqueryState *subState, int idx, int8_t state) { - assert(idx < subState->numOfSub && subState->states != NULL); - tscDebug("subquery:0x%"PRIx64",%d state set to %d", pSql->self, idx, state); - - pthread_mutex_lock(&subState->mutex); - subState->states[idx] = state; - pthread_mutex_unlock(&subState->mutex); -} - -static bool allSubqueryDone(SSqlObj *pParentSql) { - bool done = true; - SSubqueryState *subState = &pParentSql->subState; - - //lock in caller - tscDebug("0x%"PRIx64" total subqueries: %d", pParentSql->self, subState->numOfSub); - for (int i = 0; i < subState->numOfSub; i++) { - SSqlObj* pSub = pParentSql->pSubs[i]; - if (0 == subState->states[i]) { - tscDebug("0x%"PRIx64" subquery:0x%"PRIx64", index: %d NOT finished yet", pParentSql->self, pSub->self, i); - done = false; - break; - } else { - if (pSub != NULL) { - tscDebug("0x%"PRIx64" subquery:0x%"PRIx64", index: %d finished", pParentSql->self, pSub->self, i); - } else { - tscDebug("0x%"PRIx64" subquery:%p, index: %d finished", pParentSql->self, pSub, i); - } - } - } - - return done; -} - -bool subAndCheckDone(SSqlObj *pSql, SSqlObj *pParentSql, int idx) { - SSubqueryState *subState = &pParentSql->subState; - assert(idx < subState->numOfSub); - - pthread_mutex_lock(&subState->mutex); - - tscDebug("0x%"PRIx64" subquery:0x%"PRIx64", index:%d state set to 1", pParentSql->self, pSql->self, idx); - subState->states[idx] = 1; - - bool done = allSubqueryDone(pParentSql); - if (!done) { - tscDebug("0x%"PRIx64" sub:%p,%d completed, total:%d", pParentSql->self, pSql, idx, pParentSql->subState.numOfSub); - } - pthread_mutex_unlock(&subState->mutex); - return done; -} - - - -static int64_t doTSBlockIntersect(SSqlObj* pSql, STimeWindow * win) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - win->skey = INT64_MAX; - win->ekey = INT64_MIN; - - SLimitVal* pLimit = &pQueryInfo->limit; - int32_t order = pQueryInfo->order.order; - int32_t joinNum = pSql->subState.numOfSub; - SMergeTsCtx ctxlist[TSDB_MAX_JOIN_TABLE_NUM] = {{0}}; - SMergeTsCtx* ctxStack[TSDB_MAX_JOIN_TABLE_NUM] = {0}; - int32_t slot = 0; - size_t tableNum = 0; - int16_t* tableMIdx = 0; - int32_t equalNum = 0; - int32_t stackidx = 0; - SMergeTsCtx* ctx = NULL; - SMergeTsCtx* pctx = NULL; - SMergeTsCtx* mainCtx = NULL; - STSElem cur; - STSElem prev; - SArray* tsCond = NULL; - int32_t mergeDone = 0; - - for (int32_t i = 0; i < joinNum; ++i) { - STSBuf* output = tsBufCreate(true, pQueryInfo->order.order); - SQueryInfo* pSubQueryInfo = tscGetQueryInfo(&pSql->pSubs[i]->cmd); - - pSubQueryInfo->tsBuf = output; - - SJoinSupporter* pSupporter = pSql->pSubs[i]->param; - - if (pSupporter->pTSBuf == NULL) { - tscDebug("0x%"PRIx64" at least one ts-comp is empty, 0 for secondary query after ts blocks intersecting", pSql->self); - return 0; - } - - tsBufResetPos(pSupporter->pTSBuf); - - if (!tsBufNextPos(pSupporter->pTSBuf)) { - tscDebug("0x%"PRIx64" input1 is empty, 0 for secondary query after ts blocks intersecting", pSql->self); - return 0; - } - - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" table idx:%d, input group number:%d", pSql->self, - pSql->pSubs[i]->self, i, pSupporter->pTSBuf->numOfGroups); - - ctxlist[i].p = pSupporter; - ctxlist[i].res = output; - } - - TSKEY st = taosGetTimestampUs(); - - for (int16_t tidx = 0; tidx < joinNum; tidx++) { - pctx = &ctxlist[tidx]; - if (pctx->compared) { - continue; - } - - assert(pctx->numOfInput == 0); - - tsCond = pQueryInfo->tagCond.joinInfo.joinTables[tidx]->tsJoin; - - tableNum = taosArrayGetSize(tsCond); - assert(tableNum >= 2); - - for (int32_t i = 0; i < tableNum; ++i) { - tableMIdx = taosArrayGet(tsCond, i); - SMergeTsCtx* tctx = &ctxlist[*tableMIdx]; - tctx->compared = 1; - } - - tableMIdx = taosArrayGet(tsCond, 0); - pctx = &ctxlist[*tableMIdx]; - - mainCtx = pctx; - - while (1) { - pctx = mainCtx; - - prev = tsBufGetElem(pctx->p->pTSBuf); - - ctxStack[stackidx++] = pctx; - - if (!tsBufIsValidElem(&prev)) { - break; - } - - tVariant tag = {0}; - tVariantAssign(&tag, prev.tag); - - int32_t skipped = 0; - - for (int32_t i = 1; i < tableNum; ++i) { - SMergeTsCtx* tctx = &ctxlist[i]; - - // find the data in supporter2 with the same tag value - STSElem e2 = tsBufFindElemStartPosByTag(tctx->p->pTSBuf, &tag); - - if (!tsBufIsValidElem(&e2)) { - skipRemainValue(pctx->p->pTSBuf, &tag); - skipped = 1; - break; - } - } - - if (skipped) { - slot = 0; - stackidx = 0; - continue; - } - - tableMIdx = taosArrayGet(tsCond, ++slot); - equalNum = 1; - - while (1) { - ctx = &ctxlist[*tableMIdx]; - - prev = tsBufGetElem(pctx->p->pTSBuf); - cur = tsBufGetElem(ctx->p->pTSBuf); - - // data with current are exhausted - if (!tsBufIsValidElem(&prev) || tVariantCompare(prev.tag, &tag) != 0) { - break; - } - - if (!tsBufIsValidElem(&cur) || tVariantCompare(cur.tag, &tag) != 0) { // ignore all records with the same tag - break; - } - - ctxStack[stackidx++] = ctx; - - int32_t ret = tsCompare(order, prev.ts, cur.ts); - if (ret == 0) { - if (++equalNum < tableNum) { - pctx = ctx; - - if (++slot >= tableNum) { - slot = 0; - } - - tableMIdx = taosArrayGet(tsCond, slot); - continue; - } - - assert(stackidx == tableNum); - - if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { - if (win->skey > prev.ts) { - win->skey = prev.ts; - } - - if (win->ekey < prev.ts) { - win->ekey = prev.ts; - } - - for (int32_t i = 0; i < stackidx; ++i) { - SMergeTsCtx* tctx = ctxStack[i]; - prev = tsBufGetElem(tctx->p->pTSBuf); - - tsBufAppend(tctx->res, prev.id, prev.tag, (const char*)&prev.ts, sizeof(prev.ts)); - } - } else { - pLimit->offset -= 1;//offset apply to projection? - } - - for (int32_t i = 0; i < stackidx; ++i) { - SMergeTsCtx* tctx = ctxStack[i]; - - if (!tsBufNextPos(tctx->p->pTSBuf) && tctx == mainCtx) { - mergeDone = 1; - } - tctx->numOfInput++; - } - - if (mergeDone) { - break; - } - - stackidx = 0; - equalNum = 1; - - ctxStack[stackidx++] = pctx; - } else if (ret > 0) { - if (!tsBufNextPos(ctx->p->pTSBuf) && ctx == mainCtx) { - mergeDone = 1; - break; - } - - ctx->numOfInput++; - stackidx--; - } else { - stackidx--; - - for (int32_t i = 0; i < stackidx; ++i) { - SMergeTsCtx* tctx = ctxStack[i]; - - if (!tsBufNextPos(tctx->p->pTSBuf) && tctx == mainCtx) { - mergeDone = 1; - } - tctx->numOfInput++; - } - - if (mergeDone) { - break; - } - - stackidx = 0; - equalNum = 1; - - ctxStack[stackidx++] = pctx; - } - - } - - if (mergeDone) { - break; - } - - slot = 0; - stackidx = 0; - - skipRemainValue(mainCtx->p->pTSBuf, &tag); - } - - stackidx = 0; - slot = 0; - mergeDone = 0; - } - - /* - * failed to set the correct ts order yet in two cases: - * 1. only one element - * 2. only one element for each tag. - */ - if (ctxlist[0].res->tsOrder == -1) { - for (int32_t i = 0; i < joinNum; ++i) { - ctxlist[i].res->tsOrder = TSDB_ORDER_ASC; - } - } - - for (int32_t i = 0; i < joinNum; ++i) { - tsBufFlush(ctxlist[i].res); - - tsBufDestroy(ctxlist[i].p->pTSBuf); - ctxlist[i].p->pTSBuf = NULL; - } - - TSKEY et = taosGetTimestampUs(); - - for (int32_t i = 0; i < joinNum; ++i) { - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" tblidx:%d, input:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks " - "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elapsed time:%" PRId64 " us", - pSql->self, pSql->pSubs[i]->self, i, ctxlist[i].numOfInput, ctxlist[i].res->numOfTotal, ctxlist[i].res->numOfGroups, win->skey, win->ekey, - tsBufGetNumOfGroup(ctxlist[i].res), et - st); - } - - return ctxlist[0].res->numOfTotal; -} - - -// todo handle failed to create sub query -SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index) { - SJoinSupporter* pSupporter = calloc(1, sizeof(SJoinSupporter)); - if (pSupporter == NULL) { - return NULL; - } - - pSupporter->pObj = pSql; - - pSupporter->subqueryIndex = index; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - memcpy(&pSupporter->interval, &pQueryInfo->interval, sizeof(pSupporter->interval)); - pSupporter->limit = pQueryInfo->limit; - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, index); - pSupporter->uid = pTableMetaInfo->pTableMeta->id.uid; - assert (pSupporter->uid != 0); - - taosGetTmpfilePath("join-", pSupporter->path); - - // do NOT create file here to reduce crash generated file left issue - pSupporter->f = NULL; - - return pSupporter; -} - -static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) { - if (pSupporter == NULL) { - return; - } - - if (pSupporter->exprList != NULL) { - tscExprDestroy(pSupporter->exprList); - pSupporter->exprList = NULL; - } - - if (pSupporter->colList != NULL) { - tscColumnListDestroy(pSupporter->colList); - } - -// tscFieldInfoClear(&pSupporter->fieldsInfo); - if (pSupporter->fieldsInfo.internalField != NULL) { - taosArrayDestroy(pSupporter->fieldsInfo.internalField); - } - if (pSupporter->pTSBuf != NULL) { - tsBufDestroy(pSupporter->pTSBuf); - pSupporter->pTSBuf = NULL; - } - - unlink(pSupporter->path); - - if (pSupporter->f != NULL) { - fclose(pSupporter->f); - pSupporter->f = NULL; - } - - if (pSupporter->pVgroupTables != NULL) { - //taosArrayDestroy(pSupporter->pVgroupTables); - tscFreeVgroupTableInfo(pSupporter->pVgroupTables); - pSupporter->pVgroupTables = NULL; - } - - tfree(pSupporter->pIdTagList); - tscTagCondRelease(&pSupporter->tagCond); - free(pSupporter); -} - -static void filterVgroupTables(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { - int32_t num = 0; - int32_t* list = NULL; - tsBufGetGroupIdList(pQueryInfo->tsBuf, &num, &list); - - // The virtual node, of which all tables are disqualified after the timestamp intersection, - // is removed to avoid next stage query. - // TODO: If tables from some vnodes are not qualified for next stage query, discard them. - for (int32_t k = 0; k < taosArrayGetSize(pVgroupTables);) { - SVgroupTableInfo* p = taosArrayGet(pVgroupTables, k); - - bool found = false; - for (int32_t f = 0; f < num; ++f) { - if (p->vgInfo.vgId == list[f]) { - found = true; - break; - } - } - - if (!found) { - tscRemoveVgroupTableGroup(pVgroupTables, k); - } else { - k++; - } - } - - assert(taosArrayGetSize(pVgroupTables) > 0); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); - - tfree(list); -} - -static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { - int32_t num = 0; - int32_t* list = NULL; - tsBufGetGroupIdList(pQueryInfo->tsBuf, &num, &list); - - size_t numOfGroups = taosArrayGetSize(pVgroupTables); - - SArray* pNew = taosArrayInit(num, sizeof(SVgroupTableInfo)); - - SVgroupTableInfo info; - for (int32_t i = 0; i < num; ++i) { - int32_t vnodeId = list[i]; - - for (int32_t j = 0; j < numOfGroups; ++j) { - SVgroupTableInfo* p1 = taosArrayGet(pVgroupTables, j); - if (p1->vgInfo.vgId == vnodeId) { - tscVgroupTableCopy(&info, p1); - break; - } - } - - taosArrayPush(pNew, &info); - } - - tfree(list); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); - - return pNew; -} - -/* - * launch secondary stage query to fetch the result that contains timestamp in set - */ -static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { - int32_t numOfSub = 0; - SJoinSupporter* pSupporter = NULL; - - //If the columns are not involved in the final select clause, the corresponding query will not be issued. - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - pSupporter = pSql->pSubs[i]->param; - if (taosArrayGetSize(pSupporter->exprList) > 0) { - ++numOfSub; - } - } - - assert(numOfSub > 0); - - // scan all subquery, if one sub query has only ts, ignore it - tscDebug("0x%"PRIx64" start to launch secondary subqueries, %d out of %d needs to query", pSql->self, numOfSub, pSql->subState.numOfSub); - - bool success = true; - - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj *pPrevSub = pSql->pSubs[i]; - pSql->pSubs[i] = NULL; - - pSupporter = pPrevSub->param; - - if (taosArrayGetSize(pSupporter->exprList) == 0) { - tscDebug("0x%"PRIx64" subIndex: %d, no need to launch query, ignore it", pSql->self, i); - - tscDestroyJoinSupporter(pSupporter); - taos_free_result(pPrevSub); - - pSql->pSubs[i] = NULL; - continue; - } - - SQueryInfo *pSubQueryInfo = tscGetQueryInfo(&pPrevSub->cmd); - STSBuf *pTsBuf = pSubQueryInfo->tsBuf; - pSubQueryInfo->tsBuf = NULL; - - // free result for async object will also free sqlObj - assert(tscNumOfExprs(pSubQueryInfo) == 1); // ts_comp query only requires one result columns - taos_free_result(pPrevSub); - - SSqlObj *pNew = createSubqueryObj(pSql, (int16_t) i, tscJoinQueryCallback, pSupporter, TSDB_SQL_SELECT, NULL); - if (pNew == NULL) { - tscDestroyJoinSupporter(pSupporter); - success = false; - break; - } - - tscClearSubqueryInfo(&pNew->cmd); - pSql->pSubs[i] = pNew; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pNew->cmd); - pQueryInfo->tsBuf = pTsBuf; // transfer the ownership of timestamp comp-z data to the new created object - - // set the second stage sub query for join process - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE); - memcpy(&pQueryInfo->interval, &pSupporter->interval, sizeof(pQueryInfo->interval)); - - tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); - - pQueryInfo->colList = pSupporter->colList; - pQueryInfo->exprList = pSupporter->exprList; - pQueryInfo->fieldsInfo = pSupporter->fieldsInfo; - pQueryInfo->groupbyExpr = pSupporter->groupInfo; - pQueryInfo->pUpstream = taosArrayInit(4, sizeof(POINTER_BYTES)); - - assert(pNew->subState.numOfSub == 0 && pQueryInfo->numOfTables == 1); - - tscFieldInfoUpdateOffset(pQueryInfo); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - pTableMetaInfo->pVgroupTables = pSupporter->pVgroupTables; - - pSupporter->exprList = NULL; - pSupporter->colList = NULL; - pSupporter->pVgroupTables = NULL; - memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo)); - memset(&pSupporter->groupInfo, 0, sizeof(SGroupbyExpr)); - - /* - * When handling the projection query, the offset value will be modified for table-table join, which is changed - * during the timestamp intersection. - */ - pSupporter->limit = pQueryInfo->limit; - SColumnIndex index = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - SSchema* s = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, 0); - - SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); - int16_t funcId = pExpr->base.functionId; - - // add the invisible timestamp column - if ((pExpr->base.colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) || - (funcId != TSDB_FUNC_TS && funcId != TSDB_FUNC_TS_DUMMY && funcId != TSDB_FUNC_PRJ)) { - - int16_t functionId = tscIsProjectionQuery(pQueryInfo)? TSDB_FUNC_PRJ : TSDB_FUNC_TS; - - tscAddFuncInSelectClause(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL, getNewResColId(&pNew->cmd)); - tscPrintSelNodeList(pNew, 0); - tscFieldInfoUpdateOffset(pQueryInfo); - - pExpr = tscExprGet(pQueryInfo, 0); - } - - // set the join condition tag column info, todo extract method - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - assert(pQueryInfo->tagCond.joinInfo.hasJoin); - int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); - - // set the tag column id for executor to extract correct tag value - tVariant* pVariant = &pExpr->base.param[0]; - - pVariant->i64 = colId; - pVariant->nType = TSDB_DATA_TYPE_BIGINT; - pVariant->nLen = sizeof(int64_t); - - pExpr->base.numOfParams = 1; - } - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - assert(pTableMetaInfo->pVgroupTables != NULL); - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - SArray* p = buildVgroupTableByResult(pQueryInfo, pTableMetaInfo->pVgroupTables); - tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); - pTableMetaInfo->pVgroupTables = p; - } else { - filterVgroupTables(pQueryInfo, pTableMetaInfo->pVgroupTables); - } - } - - subquerySetState(pNew, &pSql->subState, i, 0); - - size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); - tscDebug("0x%"PRIx64" subquery:0x%"PRIx64" tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", - pSql->self, pNew->self, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, taosArrayGetSize(pQueryInfo->exprList), - numOfCols, pQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); - } - - //prepare the subqueries object failed, abort - if (!success) { - pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("0x%"PRIx64" failed to prepare subqueries objs for secondary phase query, numOfSub:%d, code:%d", pSql->self, - pSql->subState.numOfSub, pSql->res.code); - freeJoinSubqueryObj(pSql); - - return pSql->res.code; - } - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - if (pSql->pSubs[i] == NULL) { - continue; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->pSubs[i]->cmd); - executeQuery(pSql->pSubs[i], pQueryInfo); - } - - return TSDB_CODE_SUCCESS; -} - -void freeJoinSubqueryObj(SSqlObj* pSql) { - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - SJoinSupporter* p = pSub->param; - tscDestroyJoinSupporter(p); - - taos_free_result(pSub); - pSql->pSubs[i] = NULL; - } - - if (pSql->subState.states) { - pthread_mutex_destroy(&pSql->subState.mutex); - } - - tfree(pSql->subState.states); - pSql->subState.numOfSub = 0; -} - -static int32_t quitAllSubquery(SSqlObj* pSqlSub, SSqlObj* pSqlObj, SJoinSupporter* pSupporter) { - if (subAndCheckDone(pSqlSub, pSqlObj, pSupporter->subqueryIndex)) { - tscError("0x%"PRIx64" all subquery return and query failed, global code:%s", pSqlObj->self, tstrerror(pSqlObj->res.code)); - freeJoinSubqueryObj(pSqlObj); - return 0; - } - - return 1; - //tscDestroyJoinSupporter(pSupporter); -} - -// update the query time range according to the join results on timestamp -static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) { - assert(pQueryInfo->window.skey <= win->skey && pQueryInfo->window.ekey >= win->ekey); - pQueryInfo->window = *win; -} - -int32_t tagValCompar(const void* p1, const void* p2) { - const STidTags* t1 = (const STidTags*) varDataVal(p1); - const STidTags* t2 = (const STidTags*) varDataVal(p2); - - __compar_fn_t func = getComparFunc(t1->padding, 0); - - return func(t1->tag, t2->tag); -} - -int32_t tidTagsCompar(const void* p1, const void* p2) { - const STidTags* t1 = (const STidTags*) (p1); - const STidTags* t2 = (const STidTags*) (p2); - - if (t1->vgId != t2->vgId) { - return (t1->vgId > t2->vgId) ? 1 : -1; - } - - return tagValCompar(p1, p2); -} - -void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) { - SArray* result = taosArrayInit(4, sizeof(SVgroupTableInfo)); - SArray* vgTables = NULL; - STidTags* prev = NULL; - - size_t numOfTables = taosArrayGetSize(tables); - for (size_t i = 0; i < numOfTables; i++) { - STidTags* tt = taosArrayGet(tables, i); - - if (prev == NULL || tt->vgId != prev->vgId) { - SVgroupsInfo* pvg = pTableMetaInfo->vgroupList; - - SVgroupTableInfo info = {{0}}; - for (int32_t m = 0; m < pvg->numOfVgroups; ++m) { - if (tt->vgId == pvg->vgroups[m].vgId) { - memcpy(&info.vgInfo, &pvg->vgroups[m], sizeof(info.vgInfo)); - break; - } - } - - assert(info.vgInfo.numOfEps != 0); - - vgTables = taosArrayInit(4, sizeof(STableIdInfo)); - info.itemList = vgTables; - - if (taosArrayGetSize(result) > 0) { - SVgroupTableInfo* prevGroup = taosArrayGet(result, taosArrayGetSize(result) - 1); - tscDebug("0x%"PRIx64" vgId:%d, tables:%"PRIzu, pSql->self, prevGroup->vgInfo.vgId, taosArrayGetSize(prevGroup->itemList)); - } - - taosArrayPush(result, &info); - } - - STableIdInfo item = {.uid = tt->uid, .tid = tt->tid, .key = INT64_MIN}; - taosArrayPush(vgTables, &item); - - tscTrace("0x%"PRIx64" tid:%d, uid:%"PRIu64",vgId:%d added", pSql->self, tt->tid, tt->uid, tt->vgId); - prev = tt; - } - - pTableMetaInfo->vgroupIndex = 0; - - if (taosArrayGetSize(result) <= 0) { - pTableMetaInfo->pVgroupTables = NULL; - taosArrayDestroy(result); - } else { - pTableMetaInfo->pVgroupTables = result; - - SVgroupTableInfo* g = taosArrayGet(result, taosArrayGetSize(result) - 1); - tscDebug("0x%"PRIx64" vgId:%d, tables:%"PRIzu, pSql->self, g->vgInfo.vgId, taosArrayGetSize(g->itemList)); - } -} - -static void issueTsCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) { - SSqlCmd* pCmd = &pSql->cmd; - tscClearSubqueryInfo(pCmd); - tscFreeSqlResult(pSql); - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - assert(pQueryInfo->numOfTables == 1); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STimeWindow window = pQueryInfo->window; - tscInitQueryInfo(pQueryInfo); - - pQueryInfo->colCond = pSupporter->colCond; - pQueryInfo->window = window; - TSDB_QUERY_CLEAR_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); - - pCmd->command = TSDB_SQL_SELECT; - pSql->fp = tscJoinQueryCallback; - - SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; - - SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL, getNewResColId(pCmd)); - - // set the tags value for ts_comp function - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - SExprInfo *pExpr = tscExprGet(pQueryInfo, 0); - int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->id.uid); - pExpr->base.param[0].i64 = tagColId; - pExpr->base.param[0].nLen = sizeof(int64_t); - pExpr->base.param[0].nType = TSDB_DATA_TYPE_BIGINT; - pExpr->base.numOfParams = 1; - } - - // add the filter tag column - if (pSupporter->colList != NULL) { - size_t s = taosArrayGetSize(pSupporter->colList); - - for (int32_t i = 0; i < s; ++i) { - SColumn *pCol = taosArrayGetP(pSupporter->colList, i); - - if (pCol->info.flist.numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. - SColumn *p = tscColumnClone(pCol); - taosArrayPush(pQueryInfo->colList, &p); - } - } - } - - size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); - - tscDebug( - "0x%"PRIx64" subquery:0x%"PRIx64" tableIndex:%d, vgroupIndex:%d, numOfVgroups:%d, type:%d, ts_comp query to retrieve timestamps, " - "numOfExpr:%" PRIzu ", colList:%" PRIzu ", numOfOutputFields:%d, name:%s", - pParent->self, pSql->self, 0, pTableMetaInfo->vgroupIndex, pTableMetaInfo->vgroupList->numOfVgroups, pQueryInfo->type, - tscNumOfExprs(pQueryInfo), numOfCols, pQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); - - tscBuildAndSendRequest(pSql, NULL); -} - -static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSqlObj* pPSqlObj) { - for(int32_t i = 1; i < p1->num; ++i) { - STidTags* prev = (STidTags*) varDataVal(p1->pIdTagList + (i - 1) * p1->tagSize); - STidTags* p = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); - assert(prev->vgId >= 1 && p->vgId >= 1); - - if (doCompare(prev->tag, p->tag, pColSchema->type, pColSchema->bytes) == 0) { - tscError("0x%"PRIx64" join tags have same value for different table, free all sub SqlObj and quit", pPSqlObj->self); - pPSqlObj->res.code = TSDB_CODE_QRY_DUP_JOIN_KEY; - return false; - } - } - - return true; -} - -static void setTidTagType(SJoinSupporter* p, uint8_t type) { - for (int32_t i = 0; i < p->num; ++i) { - STidTags * tag = (STidTags*) varDataVal(p->pIdTagList + i * p->tagSize); - tag->padding = type; - } -} - -static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray* resList) { - int16_t joinNum = pParentSql->subState.numOfSub; - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); - SJoinSupporter* p0 = pParentSql->pSubs[0]->param; - SMergeCtx ctxlist[TSDB_MAX_JOIN_TABLE_NUM] = {{0}}; - SMergeCtx* ctxStack[TSDB_MAX_JOIN_TABLE_NUM] = {0}; - - // int16_t for padding - int32_t size = p0->tagSize - sizeof(int16_t); - - SSchema* pColSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); - - tscDebug("0x%"PRIx64" all subquery retrieve complete, do tags match", pParentSql->self); - - for (int32_t i = 0; i < joinNum; i++) { - SJoinSupporter* p = pParentSql->pSubs[i]->param; - - setTidTagType(p, pColSchema->type); - - ctxlist[i].p = p; - ctxlist[i].res = taosArrayInit(p->num, size); - - tscDebug("Join %d - num:%d", i, p->num); - - // sort according to the tag valu - if (p->pIdTagList != NULL) { - qsort(p->pIdTagList, p->num, p->tagSize, tagValCompar); - } - - if (!checkForDuplicateTagVal(pColSchema, p, pParentSql)) { - for (int32_t j = 0; j <= i; j++) { - taosArrayDestroy(ctxlist[j].res); - } - return TSDB_CODE_QRY_DUP_JOIN_KEY; - } - } - - int32_t slot = 0; - size_t tableNum = 0; - int16_t* tableMIdx = 0; - int32_t equalNum = 0; - int32_t stackidx = 0; - int32_t mergeDone = 0; - SMergeCtx* ctx = NULL; - SMergeCtx* pctx = NULL; - STidTags* cur = NULL; - STidTags* prev = NULL; - SArray* tagCond = NULL; - - for (int16_t tidx = 0; tidx < joinNum; tidx++) { - pctx = &ctxlist[tidx]; - if (pctx->compared) { - continue; - } - - assert(pctx->idx == 0 && taosArrayGetSize(pctx->res) == 0); - - tagCond = pQueryInfo->tagCond.joinInfo.joinTables[tidx]->tagJoin; - - tableNum = taosArrayGetSize(tagCond); - assert(tableNum >= 2); - - for (int32_t i = 0; i < tableNum; ++i) { - tableMIdx = taosArrayGet(tagCond, i); - SMergeCtx* tctx = &ctxlist[*tableMIdx]; - tctx->compared = 1; - } - - for (int32_t i = 0; i < tableNum; ++i) { - tableMIdx = taosArrayGet(tagCond, i); - SMergeCtx* tctx = &ctxlist[*tableMIdx]; - if (tctx->p->num <= 0 || tctx->p->pIdTagList == NULL) { - mergeDone = 1; - break; - } - } - - if (mergeDone) { - mergeDone = 0; - continue; - } - - tableMIdx = taosArrayGet(tagCond, slot); - - pctx = &ctxlist[*tableMIdx]; - - prev = (STidTags*) varDataVal(pctx->p->pIdTagList + pctx->idx * pctx->p->tagSize); - - ctxStack[stackidx++] = pctx; - - tableMIdx = taosArrayGet(tagCond, ++slot); - - equalNum = 1; - - while (1) { - ctx = &ctxlist[*tableMIdx]; - - cur = (STidTags*) varDataVal(ctx->p->pIdTagList + ctx->idx * ctx->p->tagSize); - - assert(cur->tid != 0 && prev->tid != 0); - - ctxStack[stackidx++] = ctx; - - int32_t ret = doCompare(prev->tag, cur->tag, pColSchema->type, pColSchema->bytes); - if (ret == 0) { - if (++equalNum < tableNum) { - prev = cur; - pctx = ctx; - - if (++slot >= tableNum) { - slot = 0; - } - - tableMIdx = taosArrayGet(tagCond, slot); - continue; - } - - tscDebug("0x%"PRIx64" tag matched, vgId:%d, val:%d, tid:%d, uid:%"PRIu64", tid:%d, uid:%"PRIu64, pParentSql->self, prev->vgId, - *(int*) prev->tag, prev->tid, prev->uid, cur->tid, cur->uid); - - assert(stackidx == tableNum); - - for (int32_t i = 0; i < stackidx; ++i) { - SMergeCtx* tctx = ctxStack[i]; - prev = (STidTags*) varDataVal(tctx->p->pIdTagList + tctx->idx * tctx->p->tagSize); - - taosArrayPush(tctx->res, prev); - } - - for (int32_t i = 0; i < stackidx; ++i) { - SMergeCtx* tctx = ctxStack[i]; - - if (++tctx->idx >= tctx->p->num) { - mergeDone = 1; - break; - } - } - - if (mergeDone) { - break; - } - - stackidx = 0; - equalNum = 1; - - prev = (STidTags*) varDataVal(pctx->p->pIdTagList + pctx->idx * pctx->p->tagSize); - - ctxStack[stackidx++] = pctx; - } else if (ret > 0) { - stackidx--; - - if (++ctx->idx >= ctx->p->num) { - break; - } - } else { - stackidx--; - - for (int32_t i = 0; i < stackidx; ++i) { - SMergeCtx* tctx = ctxStack[i]; - if (++tctx->idx >= tctx->p->num) { - mergeDone = 1; - break; - } - } - - if (mergeDone) { - break; - } - - stackidx = 0; - equalNum = 1; - - prev = (STidTags*) varDataVal(pctx->p->pIdTagList + pctx->idx * pctx->p->tagSize); - ctxStack[stackidx++] = pctx; - } - - } - - slot = 0; - mergeDone = 0; - stackidx = 0; - } - - for (int32_t i = 0; i < joinNum; ++i) { - // reorganize the tid-tag value according to both the vgroup id and tag values - // sort according to the tag value - size_t num = taosArrayGetSize(ctxlist[i].res); - - qsort((ctxlist[i].res)->pData, num, size, tidTagsCompar); - - taosArrayPush(resList, &ctxlist[i].res); - - tscDebug("0x%"PRIx64" tags match complete, result num: %"PRIzu, pParentSql->self, num); - } - - return TSDB_CODE_SUCCESS; -} - -bool emptyTagList(SArray* resList, int32_t size) { - size_t rsize = taosArrayGetSize(resList); - if (rsize != size) { - return true; - } - - for (int32_t i = 0; i < size; ++i) { - SArray** s = taosArrayGet(resList, i); - if (taosArrayGetSize(*s) <= 0) { - return true; - } - } - - return false; -} - -static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRows) { - SJoinSupporter* pSupporter = (SJoinSupporter*)param; - - SSqlObj* pParentSql = pSupporter->pObj; - - SSqlObj* pSql = (SSqlObj*)tres; - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - // todo, the type may not include TSDB_QUERY_TYPE_TAG_FILTER_QUERY - assert(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)); - - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" abort query due to other subquery failure. code:%d, global code:%d", pSql->self, numOfRows, pParentSql->res.code); - if (quitAllSubquery(pSql, pParentSql, pSupporter)) { - return; - } - - tscAsyncResultOnError(pParentSql); - - return; - } - - // check for the error code firstly - if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { - // todo retry if other subqueries are not failed - - assert(numOfRows < 0 && numOfRows == taos_errno(pSql)); - tscError("0x%"PRIx64" sub query failed, code:%s, index:%d", pSql->self, tstrerror(numOfRows), pSupporter->subqueryIndex); - - pParentSql->res.code = numOfRows; - if (quitAllSubquery(pSql, pParentSql, pSupporter)) { - return; - } - - tscAsyncResultOnError(pParentSql); - return; - } - - // keep the results in memory - if (numOfRows > 0) { - size_t validLen = (size_t)(pSupporter->tagSize * pRes->numOfRows); - size_t length = pSupporter->totalLen + validLen; - - // todo handle memory error - char* tmp = realloc(pSupporter->pIdTagList, length); - if (tmp == NULL) { - tscError("0x%"PRIx64" failed to malloc memory", pSql->self); - - pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - if (quitAllSubquery(pSql, pParentSql, pSupporter)) { - return; - } - - tscAsyncResultOnError(pParentSql); - return; - } - - pSupporter->pIdTagList = tmp; - - memcpy(pSupporter->pIdTagList + pSupporter->totalLen, pRes->data, validLen); - pSupporter->totalLen += (int32_t)validLen; - pSupporter->num += (int32_t)pRes->numOfRows; - - // query not completed, continue to retrieve tid + tag tuples - if (!pRes->completed) { - taos_fetch_rows_a(tres, tidTagRetrieveCallback, param); - return; - } - } - - // data in current vnode has all returned to client, try next vnode if exits - // tuples have been retrieved to client, try tuples from the next vnode - if (hasMoreVnodesToTry(pSql)) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups; - pTableMetaInfo->vgroupIndex += 1; - assert(pTableMetaInfo->vgroupIndex < totalVgroups); - - tscDebug("0x%"PRIx64" tid_tag from vgroup index:%d completed, try next vgroup:%d. total vgroups:%d. current numOfRes:%d", - pSql->self, pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pSupporter->num); - - pCmd->command = TSDB_SQL_SELECT; - tscResetForNextRetrieve(&pSql->res); - - // set the callback function - pSql->fp = tscJoinQueryCallback; - tscBuildAndSendRequest(pSql, NULL); - return; - } - - // no data exists in next vnode, mark the query completed - // only when there is no subquery exits any more, proceeds to get the intersect of the tuple sets. - if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { - //tscDebug("0x%"PRIx64" tagRetrieve:%p,%d completed, total:%d", pParentSql->self, tres, pSupporter->subqueryIndex, pParentSql->subState.numOfSub); - return; - } - - SArray* resList = taosArrayInit(pParentSql->subState.numOfSub, sizeof(SArray *)); - - int32_t code = getIntersectionOfTableTuple(pQueryInfo, pParentSql, resList); - if (code != TSDB_CODE_SUCCESS) { - freeJoinSubqueryObj(pParentSql); - pParentSql->res.code = code; - tscAsyncResultOnError(pParentSql); - - taosArrayDestroy(resList); - return; - } - - if (emptyTagList(resList, pParentSql->subState.numOfSub)) { // no results,return. - assert(pParentSql->fp != tscJoinQueryCallback); - - tscDebug("0x%"PRIx64" tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", - pParentSql->self); - freeJoinSubqueryObj(pParentSql); - - // set no result command - pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - assert(pParentSql->fp != tscJoinQueryCallback); - - (*pParentSql->fp)(pParentSql->param, pParentSql, 0); - } else { - for (int32_t m = 0; m < pParentSql->subState.numOfSub; ++m) { - // proceed to for ts_comp query - SSqlCmd* pSubCmd = &pParentSql->pSubs[m]->cmd; - SArray** s = taosArrayGet(resList, m); - - SQueryInfo* pQueryInfo1 = tscGetQueryInfo(pSubCmd); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo1, 0); - tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo, *s); - - SSqlObj* psub = pParentSql->pSubs[m]; - ((SJoinSupporter*)psub->param)->pVgroupTables = tscVgroupTableInfoDup(pTableMetaInfo->pVgroupTables); - - memset(pParentSql->subState.states, 0, sizeof(pParentSql->subState.states[0]) * pParentSql->subState.numOfSub); - tscDebug("0x%"PRIx64" reset all sub states to 0", pParentSql->self); - - issueTsCompQuery(psub, psub->param, pParentSql); - } - } - - size_t rsize = taosArrayGetSize(resList); - for (int32_t i = 0; i < rsize; ++i) { - SArray** s = taosArrayGet(resList, i); - if (*s) { - taosArrayDestroy(*s); - } - } - - taosArrayDestroy(resList); -} - -static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRows) { - SJoinSupporter* pSupporter = (SJoinSupporter*)param; - - SSqlObj* pParentSql = pSupporter->pObj; - - SSqlObj* pSql = (SSqlObj*)tres; - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - assert(!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)); - - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" abort query due to other subquery failure. code:%d, global code:%d", pSql->self, numOfRows, pParentSql->res.code); - if (quitAllSubquery(pSql, pParentSql, pSupporter)){ - return; - } - - tscAsyncResultOnError(pParentSql); - - return; - } - - // check for the error code firstly - if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { - // todo retry if other subqueries are not failed yet - assert(numOfRows < 0 && numOfRows == taos_errno(pSql)); - tscError("0x%"PRIx64" sub query failed, code:%s, index:%d", pSql->self, tstrerror(numOfRows), pSupporter->subqueryIndex); - - pParentSql->res.code = numOfRows; - if (quitAllSubquery(pSql, pParentSql, pSupporter)){ - return; - } - - tscAsyncResultOnError(pParentSql); - return; - } - - if (numOfRows > 0) { // write the compressed timestamp to disk file - if(pSupporter->f == NULL) { - pSupporter->f = fopen(pSupporter->path, "wb"); - - if (pSupporter->f == NULL) { - tscError("0x%"PRIx64" failed to create tmp file:%s, reason:%s", pSql->self, pSupporter->path, strerror(errno)); - pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - - if (quitAllSubquery(pSql, pParentSql, pSupporter)) { - return; - } - - tscAsyncResultOnError(pParentSql); - - return; - } - } - - fwrite(pRes->data, (size_t)pRes->numOfRows, 1, pSupporter->f); - fclose(pSupporter->f); - pSupporter->f = NULL; - - STSBuf* pBuf = tsBufCreateFromFile(pSupporter->path, true); - if (pBuf == NULL) { // in error process, close the fd - tscError("0x%"PRIx64" invalid ts comp file from vnode, abort subquery, file size:%d", pSql->self, numOfRows); - - pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - if (quitAllSubquery(pSql, pParentSql, pSupporter)){ - return; - } - - tscAsyncResultOnError(pParentSql); - - return; - } - - if (pSupporter->pTSBuf == NULL) { - tscDebug("0x%"PRIx64" create tmp file for ts block:%s, size:%d bytes", pSql->self, pBuf->path, numOfRows); - pSupporter->pTSBuf = pBuf; - } else { - assert(pQueryInfo->numOfTables == 1); // for subquery, only one - tsBufMerge(pSupporter->pTSBuf, pBuf); - tsBufDestroy(pBuf); - } - - // continue to retrieve ts-comp data from vnode - if (!pRes->completed) { - taosGetTmpfilePath("ts-join", pSupporter->path); - pSupporter->f = fopen(pSupporter->path, "wb"); - pRes->row = pRes->numOfRows; - - taos_fetch_rows_a(tres, tsCompRetrieveCallback, param); - return; - } - } - - if (hasMoreVnodesToTry(pSql)) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups; - pTableMetaInfo->vgroupIndex += 1; - assert(pTableMetaInfo->vgroupIndex < totalVgroups); - - tscDebug("0x%"PRIx64" results from vgroup index:%d completed, try next vgroup:%d. total vgroups:%d. current numOfRes:%" PRId64, - pSql->self, pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, - pRes->numOfClauseTotal); - - pCmd->command = TSDB_SQL_SELECT; - tscResetForNextRetrieve(&pSql->res); - - assert(pSupporter->f == NULL); - taosGetTmpfilePath("ts-join", pSupporter->path); - - // TODO check for failure - pSupporter->f = fopen(pSupporter->path, "wb"); - pRes->row = pRes->numOfRows; - - // set the callback function - pSql->fp = tscJoinQueryCallback; - tscBuildAndSendRequest(pSql, NULL); - return; - } - - if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { - return; - } - - tscDebug("0x%"PRIx64" all subquery retrieve ts complete, do ts block intersect", pParentSql->self); - - STimeWindow win = TSWINDOW_INITIALIZER; - int64_t num = doTSBlockIntersect(pParentSql, &win); - if (num <= 0) { // no result during ts intersect - tscDebug("0x%"PRIx64" no results generated in ts intersection, free all sub SqlObj and quit", pParentSql->self); - freeJoinSubqueryObj(pParentSql); - - // set no result command - pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - (*pParentSql->fp)(pParentSql->param, pParentSql, 0); - return; - } - - // launch the query the retrieve actual results from vnode along with the filtered timestamp - SQueryInfo* pPQueryInfo = tscGetQueryInfo(&pParentSql->cmd); - updateQueryTimeRange(pPQueryInfo, &win); - - //update the vgroup that involved in real data query - tscLaunchRealSubqueries(pParentSql); -} - -static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfRows) { - SJoinSupporter* pSupporter = (SJoinSupporter*)param; - - SSqlObj* pParentSql = pSupporter->pObj; - - SSqlObj* pSql = (SSqlObj*)tres; - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" abort query due to other subquery failure. code:%d, global code:%d", pSql->self, numOfRows, pParentSql->res.code); - if (quitAllSubquery(pSql, pParentSql, pSupporter)) { - return; - } - - tscAsyncResultOnError(pParentSql); - return; - } - - - if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { - assert(numOfRows == taos_errno(pSql)); - - pParentSql->res.code = numOfRows; - tscError("0x%"PRIx64" retrieve failed, index:%d, code:%s", pSql->self, pSupporter->subqueryIndex, tstrerror(numOfRows)); - - tscAsyncResultOnError(pParentSql); - return; - } - - if (numOfRows >= 0) { - pRes->numOfTotal += pRes->numOfRows; - } - - SSubqueryState* pState = &pParentSql->subState; - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && numOfRows == 0) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - assert(pQueryInfo->numOfTables == 1); - - // for projection query, need to try next vnode if current vnode is exhausted - int32_t numOfVgroups = 0; // TODO refactor - if (pTableMetaInfo->pVgroupTables != NULL) { - numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - } else { - numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; - } - - if ((++pTableMetaInfo->vgroupIndex) < numOfVgroups) { - tscDebug("0x%"PRIx64" no result in current vnode anymore, try next vnode, vgIndex:%d", pSql->self, pTableMetaInfo->vgroupIndex); - pSql->cmd.command = TSDB_SQL_SELECT; - pSql->fp = tscJoinQueryCallback; - - tscBuildAndSendRequest(pSql, NULL); - return; - } else { - tscDebug("0x%"PRIx64" no result in current subquery anymore", pSql->self); - } - } - - if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { - //tscDebug("0x%"PRIx64" sub:0x%"PRIx64",%d completed, total:%d", pParentSql->self, pSql->self, pSupporter->subqueryIndex, pState->numOfSub); - return; - } - - tscDebug("0x%"PRIx64" all %d secondary subqueries retrieval completed, code:%d", pSql->self, pState->numOfSub, pParentSql->res.code); - - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - freeJoinSubqueryObj(pParentSql); - pParentSql->res.completed = true; - } - - // update the records for each subquery in parent sql object. - bool stableQuery = tscIsTwoStageSTableQuery(pQueryInfo, 0); - for (int32_t i = 0; i < pState->numOfSub; ++i) { - if (pParentSql->pSubs[i] == NULL) { - tscDebug("0x%"PRIx64" %p sub:%d not retrieve data", pParentSql->self, NULL, i); - continue; - } - - SSqlRes* pRes1 = &pParentSql->pSubs[i]->res; - - pParentSql->res.precision = pRes1->precision; - - if (pRes1->row > 0 && pRes1->numOfRows > 0) { - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" index:%d numOfRows:%d total:%"PRId64 " (not retrieve)", pParentSql->self, - pParentSql->pSubs[i]->self, i, pRes1->numOfRows, pRes1->numOfTotal); - assert(pRes1->row < pRes1->numOfRows); - } else { - if (!stableQuery) { - pRes1->numOfClauseTotal += pRes1->numOfRows; - } - - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" index:%d numOfRows:%d total:%"PRId64, pParentSql->self, - pParentSql->pSubs[i]->self, i, pRes1->numOfRows, pRes1->numOfTotal); - } - } - - // data has retrieved to client, build the join results - tscBuildResFromSubqueries(pParentSql); -} - -void tscFetchDatablockForSubquery(SSqlObj* pSql) { - assert(pSql->subState.numOfSub >= 1); - - int32_t numOfFetch = 0; - bool hasData = true; - bool reachLimit = false; - - // if the subquery is NULL, it does not involved in the final result generation - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - SSqlRes *pRes = &pSub->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSub->cmd); - if (!tscHasReachLimitation(pQueryInfo, pRes)) { - if (pRes->row >= pRes->numOfRows) { - // no data left in current result buffer - hasData = false; - - // The current query is completed for the active vnode, try next vnode if exists - // If it is completed, no need to fetch anymore. - if (!pRes->completed) { - numOfFetch++; - } - } - } else { // has reach the limitation, no data anymore - if (pRes->row >= pRes->numOfRows) { - reachLimit = true; - hasData = false; - break; - } - } - } - - // has data remains in client side, and continue to return data to app - if (hasData) { - tscBuildResFromSubqueries(pSql); - return; - } - - // If at least one subquery is completed in current vnode, try the next vnode in case of multi-vnode - // super table projection query. - if (reachLimit) { - pSql->res.completed = true; - freeJoinSubqueryObj(pSql); - - if (pSql->res.code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, 0); - } else { - tscAsyncResultOnError(pSql); - } - - return; - } - - if (numOfFetch <= 0) { - bool tryNextVnode = false; - - bool orderedPrjQuery = false; - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - SQueryInfo* p = tscGetQueryInfo(&pSub->cmd); - orderedPrjQuery = tscNonOrderedProjectionQueryOnSTable(p, 0); - if (orderedPrjQuery) { - break; - } - } - - - if (orderedPrjQuery) { - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub != NULL && pSub->res.row >= pSub->res.numOfRows && pSub->res.completed) { - subquerySetState(pSub, &pSql->subState, i, 0); - } - } - } - - - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSub->cmd); - - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && pSub->res.row >= pSub->res.numOfRows && - pSub->res.completed) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - assert(pQueryInfo->numOfTables == 1); - - // for projection query, need to try next vnode if current vnode is exhausted - int32_t numOfVgroups = 0; // TODO refactor - if (pTableMetaInfo->pVgroupTables != NULL) { - numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - } else { - numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; - } - - if ((++pTableMetaInfo->vgroupIndex) < numOfVgroups) { - tscDebug("0x%"PRIx64" no result in current vnode anymore, try next vnode, vgIndex:%d", pSub->self, - pTableMetaInfo->vgroupIndex); - pSub->cmd.command = TSDB_SQL_SELECT; - pSub->fp = tscJoinQueryCallback; - - tscBuildAndSendRequest(pSub, NULL); - tryNextVnode = true; - } else { - tscDebug("0x%"PRIx64" no result in current subquery anymore", pSub->self); - } - } - } - - if (tryNextVnode) { - return; - } - - pSql->res.completed = true; - freeJoinSubqueryObj(pSql); - - if (pSql->res.code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, 0); - } else { - tscAsyncResultOnError(pSql); - } - - return; - } - - // TODO multi-vnode retrieve for projection query with limitation has bugs, since the global limiation is not handled - // retrieve data from current vnode. - tscDebug("0x%"PRIx64" retrieve data from %d subqueries", pSql->self, numOfFetch); - SJoinSupporter* pSupporter = NULL; - - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSql1 = pSql->pSubs[i]; - if (pSql1 == NULL) { - continue; - } - - - - SSqlRes* pRes1 = &pSql1->res; - if (pRes1->row >= pRes1->numOfRows) { - subquerySetState(pSql1, &pSql->subState, i, 0); - } - } - - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSql1 = pSql->pSubs[i]; - if (pSql1 == NULL) { - continue; - } - - SSqlRes* pRes1 = &pSql1->res; - SSqlCmd* pCmd1 = &pSql1->cmd; - - pSupporter = (SJoinSupporter*)pSql1->param; - - // wait for all subqueries completed - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd1); - assert(pRes1->numOfRows >= 0 && pQueryInfo->numOfTables == 1); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - if (pRes1->row >= pRes1->numOfRows) { - tscDebug("0x%"PRIx64" subquery:0x%"PRIx64" retrieve data from vnode, subquery:%d, vgroupIndex:%d", pSql->self, pSql1->self, - pSupporter->subqueryIndex, pTableMetaInfo->vgroupIndex); - - tscResetForNextRetrieve(pRes1); - pSql1->fp = joinRetrieveFinalResCallback; - - if (pCmd1->command < TSDB_SQL_LOCAL) { - pCmd1->command = (pCmd1->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; - } - - tscBuildAndSendRequest(pSql1, NULL); - } - } -} - -// all subqueries return, set the result output index -void tscSetupOutputColumnIndex(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - // the column transfer support struct has been built - if (pRes->pColumnIndex != NULL) { - return; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - int32_t numOfExprs = (int32_t)tscNumOfExprs(pQueryInfo); - pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * numOfExprs); - if (pRes->pColumnIndex == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return; - } - - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - - int32_t tableIndexOfSub = -1; - for (int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, j); - if (pTableMetaInfo->pTableMeta->id.uid == pExpr->base.uid) { - tableIndexOfSub = j; - break; - } - } - - assert(tableIndexOfSub >= 0 && tableIndexOfSub < pQueryInfo->numOfTables); - - SSqlCmd* pSubCmd = &pSql->pSubs[tableIndexOfSub]->cmd; - SQueryInfo* pSubQueryInfo = tscGetQueryInfo(pSubCmd); - - size_t numOfSubExpr = taosArrayGetSize(pSubQueryInfo->exprList); - for (int32_t k = 0; k < numOfSubExpr; ++k) { - SExprInfo* pSubExpr = tscExprGet(pSubQueryInfo, k); - if (pExpr->base.functionId == pSubExpr->base.functionId && pExpr->base.colInfo.colId == pSubExpr->base.colInfo.colId) { - pRes->pColumnIndex[i] = (SColumnIndex){.tableIndex = tableIndexOfSub, .columnIndex = k}; - break; - } - } - } - - // restore the offset value for super table query in case of final result. -// tscRestoreFuncForSTableQuery(pQueryInfo); -// tscFieldInfoUpdateOffset(pQueryInfo); -} - -void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { - SSqlObj* pSql = (SSqlObj*)tres; - - SJoinSupporter* pSupporter = (SJoinSupporter*)param; - SSqlObj* pParentSql = pSupporter->pObj; - - // There is only one subquery and table for each subquery. - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - assert(pQueryInfo->numOfTables == 1); - - // retrieve actual query results from vnode during the second stage join subquery - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" abort query due to other subquery failure. code:%d, global code:%d", pSql->self, code, pParentSql->res.code); - if (quitAllSubquery(pSql, pParentSql, pSupporter)) { - return; - } - - tscAsyncResultOnError(pParentSql); - - return; - } - - // TODO here retry is required, not directly returns to client - if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { - assert(taos_errno(pSql) == code); - - tscError("0x%"PRIx64" abort query, code:%s, global code:%s", pSql->self, tstrerror(code), tstrerror(pParentSql->res.code)); - pParentSql->res.code = code; - - if (quitAllSubquery(pSql, pParentSql, pSupporter)) { - return; - } - - tscAsyncResultOnError(pParentSql); - - return; - } - - // retrieve tuples from vnode - if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) { - pSql->fp = tidTagRetrieveCallback; - pSql->cmd.command = TSDB_SQL_FETCH; - tscBuildAndSendRequest(pSql, NULL); - return; - } - - // retrieve ts_comp info from vnode - if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { - pSql->fp = tsCompRetrieveCallback; - pSql->cmd.command = TSDB_SQL_FETCH; - tscBuildAndSendRequest(pSql, NULL); - return; - } - - // In case of consequence query from other vnode, do not wait for other query response here. - if (!(pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0))) { - if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { - return; - } - } - - tscSetupOutputColumnIndex(pParentSql); - - /** - * if the query is a continue query (vgroupIndex > 0 for projection query) for next vnode, do the retrieval of - * data instead of returning to its invoker - */ - if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - pSql->fp = joinRetrieveFinalResCallback; // continue retrieve data - pSql->cmd.command = TSDB_SQL_FETCH; - - tscBuildAndSendRequest(pSql, NULL); - } else { // first retrieve from vnode during the secondary stage sub-query - // set the command flag must be after the semaphore been correctly set. - if (pParentSql->res.code == TSDB_CODE_SUCCESS) { - (*pParentSql->fp)(pParentSql->param, pParentSql, 0); - } else { - tscAsyncResultOnError(pParentSql); - } - } -} - -///////////////////////////////////////////////////////////////////////////////////////// -static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code); - -static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); - -int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter *pSupporter) { - SSqlCmd * pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - pSql->res.qId = 0x1; - assert(pSql->res.numOfRows == 0); - - if (pSql->pSubs == NULL) { - pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES); - if (pSql->pSubs == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - SSqlObj *pNew = createSubqueryObj(pSql, tableIndex, tscJoinQueryCallback, pSupporter, TSDB_SQL_SELECT, NULL); - if (pNew == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pSql->pSubs[tableIndex] = pNew; - - if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { - addGroupInfoForSubquery(pSql, pNew, 0, tableIndex); - - // refactor as one method - SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); - assert(pNewQueryInfo != NULL); - - pSupporter->colList = pNewQueryInfo->colList; - pNewQueryInfo->colList = NULL; - - pSupporter->exprList = pNewQueryInfo->exprList; - pNewQueryInfo->exprList = NULL; - - pSupporter->fieldsInfo = pNewQueryInfo->fieldsInfo; - - // this data needs to be transfer to support struct - memset(&pNewQueryInfo->fieldsInfo, 0, sizeof(SFieldInfo)); - if (tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond) != 0) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pSupporter->groupInfo = pNewQueryInfo->groupbyExpr; - memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SGroupbyExpr)); - - pNew->cmd.numOfCols = 0; - pNewQueryInfo->interval.interval = 0; - pSupporter->limit = pNewQueryInfo->limit; - - pNewQueryInfo->limit.limit = -1; - pNewQueryInfo->limit.offset = 0; - taosArrayDestroy(pNewQueryInfo->pUpstream); - - pNewQueryInfo->order.orderColId = INT32_MIN; - - // backup the data and clear it in the sqlcmd object - memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SGroupbyExpr)); - - STimeWindow range = pNewQueryInfo->window; - tscInitQueryInfo(pNewQueryInfo); - - pNewQueryInfo->window = range; - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // return the tableId & tag - SColumnIndex colIndex = {0}; - - pSupporter->colCond = pNewQueryInfo->colCond; - pNewQueryInfo->colCond = NULL; - - STagCond* pTagCond = &pSupporter->tagCond; - assert(pTagCond->joinInfo.hasJoin); - - int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid); - SSchema* s = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); - - colIndex.columnIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, tagColId); - - int16_t bytes = 0; - int16_t type = 0; - int32_t inter = 0; - - getResultDataInfo(s->type, s->bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0, NULL); - - SSchema s1 = {.colId = s->colId, .type = (uint8_t)type, .bytes = bytes}; - pSupporter->tagSize = s1.bytes; - assert(isValidDataType(s1.type) && s1.bytes > 0); - - // set get tags query type - TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); - tscAddFuncInSelectClause(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &colIndex, &s1, TSDB_COL_TAG, getNewResColId(pCmd)); - size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); - - tscDebug( - "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, transfer to tid_tag query to retrieve (tableId, tags), " - "exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, tagIndex:%d, name:%s", - pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscNumOfExprs(pNewQueryInfo), - numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, colIndex.columnIndex, tNameGetTableName(&pNewQueryInfo->pTableMetaInfo[0]->name)); - } else { - SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; - SColumnIndex colIndex = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddFuncInSelectClause(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &colIndex, &colSchema, TSDB_COL_NORMAL, getNewResColId(pCmd)); - - // set the tags value for ts_comp function - SExprInfo *pExpr = tscExprGet(pNewQueryInfo, 0); - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->id.uid); - pExpr->base.param->i64 = tagColId; - pExpr->base.numOfParams = 1; - } - - // add the filter tag column - if (pSupporter->colList != NULL) { - size_t s = taosArrayGetSize(pSupporter->colList); - - for (int32_t i = 0; i < s; ++i) { - SColumn *pCol = taosArrayGetP(pSupporter->colList, i); - - if (pCol->info.flist.numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. - SColumn *p = tscColumnClone(pCol); - taosArrayPush(pNewQueryInfo->colList, &p); - } - } - } - - size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); - - tscDebug( - "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%u, transfer to ts_comp query to retrieve timestamps, " - "exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", - pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscNumOfExprs(pNewQueryInfo), - numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pNewQueryInfo->pTableMetaInfo[0]->name)); - } - } else { - assert(0); - SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); - pNewQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; - } - - return TSDB_CODE_SUCCESS; -} - -void tscHandleMasterJoinQuery(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - assert((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0); - - int32_t code = TSDB_CODE_SUCCESS; - pSql->subState.numOfSub = pQueryInfo->numOfTables; - - if (pSql->subState.states == NULL) { - pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(*pSql->subState.states)); - if (pSql->subState.states == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - pthread_mutex_init(&pSql->subState.mutex, NULL); - } - - memset(pSql->subState.states, 0, sizeof(*pSql->subState.states) * pSql->subState.numOfSub); - tscDebug("0x%"PRIx64" reset all sub states to 0, start subquery, total:%d", pSql->self, pQueryInfo->numOfTables); - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, i); - if (pSupporter == NULL) { // failed to create support struct, abort current query - tscError("0x%"PRIx64" tableIndex:%d, failed to allocate join support object, abort further query", pSql->self, i); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - code = tscCreateJoinSubquery(pSql, i, pSupporter); - if (code != TSDB_CODE_SUCCESS) { // failed to create subquery object, quit query - tscDestroyJoinSupporter(pSupporter); - goto _error; - } - - SSqlObj* pSub = pSql->pSubs[i]; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSub->cmd, 0); - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) && (pTableMetaInfo->vgroupList->numOfVgroups == 0)) { - pSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - break; - } - } - - if (pSql->cmd.command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { // at least one subquery is empty, do nothing and return - freeJoinSubqueryObj(pSql); - (*pSql->fp)(pSql->param, pSql, 0); - } else { - int fail = 0; - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (fail) { - (*pSub->fp)(pSub->param, pSub, 0); - continue; - } - - if ((code = tscBuildAndSendRequest(pSub, NULL)) != TSDB_CODE_SUCCESS) { - pRes->code = code; - (*pSub->fp)(pSub->param, pSub, 0); - fail = 1; - } - } - - if(fail) { - return; - } - - pSql->cmd.command = TSDB_SQL_TABLE_JOIN_RETRIEVE; - } - - return; - - _error: - pRes->code = code; - tscAsyncResultOnError(pSql); -} - -void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) { - assert(numOfSubs <= pSql->subState.numOfSub && numOfSubs >= 0); - - for(int32_t i = 0; i < numOfSubs; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - assert(pSub != NULL); - - tscFreeRetrieveSup(pSub); - - taos_free_result(pSub); - } -} - -void tscLockByThread(int64_t *lockedBy) { - int64_t tid = taosGetSelfPthreadId(); - int i = 0; - while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) { - if (++i % 100 == 0) { - sched_yield(); - } - } -} - -void tscUnlockByThread(int64_t *lockedBy) { - int64_t tid = taosGetSelfPthreadId(); - if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) { - assert(false); - } -} - -typedef struct SFirstRoundQuerySup { - SSqlObj *pParent; - int32_t numOfRows; - SArray *pColsInfo; - int32_t tagLen; - STColumn *pTagCols; - SArray *pResult; // SArray - int64_t interval; - char* buf; - int32_t bufLen; -} SFirstRoundQuerySup; - -void doAppendData(SInterResult* pInterResult, TAOS_ROW row, int32_t numOfCols, SQueryInfo* pQueryInfo) { - TSKEY key = INT64_MIN; - for(int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag) || pExpr->base.functionId == TSDB_FUNC_PRJ) { - continue; - } - - if (pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - key = *(TSKEY*) row[i]; - continue; - } - - double v = 0; - if (row[i] != NULL) { - v = *(double*) row[i]; - } else { - SET_DOUBLE_NULL(&v); - } - - int32_t id = pExpr->base.colInfo.colId; - int32_t numOfQueriedCols = (int32_t) taosArrayGetSize(pInterResult->pResult); - - SArray* p = NULL; - for(int32_t j = 0; j < numOfQueriedCols; ++j) { - SStddevInterResult* pColRes = taosArrayGet(pInterResult->pResult, j); - if (pColRes->colId == id) { - p = pColRes->pResult; - break; - } - } - - if (p && taosArrayGetSize(p) > 0) { - SResPair *l = taosArrayGetLast(p); - if (l->key == key && key == INT64_MIN) { - continue; - } - } - - //append a new column - if (p == NULL) { - SStddevInterResult t = {.colId = id, .pResult = taosArrayInit(10, sizeof(SResPair)),}; - taosArrayPush(pInterResult->pResult, &t); - p = t.pResult; - } - - SResPair pair = {.avg = v, .key = key}; - taosArrayPush(p, &pair); - } -} - -static void destroySup(SFirstRoundQuerySup* pSup) { - taosArrayDestroyEx(pSup->pResult, freeInterResult); - taosArrayDestroy(pSup->pColsInfo); - tfree(pSup); -} - -void tscFirstRoundRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { - SSqlObj* pSql = (SSqlObj*)tres; - SSqlRes* pRes = &pSql->res; - - SFirstRoundQuerySup* pSup = param; - - SSqlObj* pParent = pSup->pParent; - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - int32_t code = taos_errno(pSql); - if (code != TSDB_CODE_SUCCESS) { - destroySup(pSup); - taos_free_result(pSql); - pParent->res.code = code; - tscAsyncResultOnError(pParent); - return; - } - - if (numOfRows > 0) { // the number is not correct for group by column in super table query - TAOS_ROW row = NULL; - int32_t numOfCols = taos_field_count(tres); - - if (pSup->tagLen == 0) { // no tags, all rows belong to one group - SInterResult interResult = {.tags = NULL, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))}; - taosArrayPush(pSup->pResult, &interResult); - - while ((row = taos_fetch_row(tres)) != NULL) { - doAppendData(&interResult, row, numOfCols, pQueryInfo); - pSup->numOfRows += 1; - } - } else { // tagLen > 0 - char* p = calloc(1, pSup->tagLen); - - while ((row = taos_fetch_row(tres)) != NULL) { - int32_t* length = taos_fetch_lengths(tres); - memset(p, 0, pSup->tagLen); - - int32_t offset = 0; - for (int32_t i = 0; i < numOfCols && offset < pSup->tagLen; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - - // tag or group by column - if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag) || pExpr->base.functionId == TSDB_FUNC_PRJ) { - if (row[i] == NULL) { - setNull(p + offset, pExpr->base.resType, pExpr->base.resBytes); - } else { - memcpy(p + offset, row[i], length[i]); - } - offset += pExpr->base.resBytes; - } - } - - assert(offset == pSup->tagLen); - size_t size = taosArrayGetSize(pSup->pResult); - - if (size > 0) { - SInterResult* pInterResult = taosArrayGetLast(pSup->pResult); - if (memcmp(pInterResult->tags, p, pSup->tagLen) == 0) { // belongs to the same group - doAppendData(pInterResult, row, numOfCols, pQueryInfo); - } else { - char* tags = malloc( pSup->tagLen); - memcpy(tags, p, pSup->tagLen); - - SInterResult interResult = {.tags = tags, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))}; - taosArrayPush(pSup->pResult, &interResult); - doAppendData(&interResult, row, numOfCols, pQueryInfo); - } - } else { - char* tags = malloc(pSup->tagLen); - memcpy(tags, p, pSup->tagLen); - - SInterResult interResult = {.tags = tags, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))}; - taosArrayPush(pSup->pResult, &interResult); - doAppendData(&interResult, row, numOfCols, pQueryInfo); - } - - pSup->numOfRows += 1; - } - - tfree(p); - } - } - - if (!pRes->completed && numOfRows > 0) { - taos_fetch_rows_a(tres, tscFirstRoundRetrieveCallback, param); - return; - } - - // set the parameters for the second round query process - SSqlCmd *pPCmd = &pParent->cmd; - SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pPCmd); - int32_t resRows = pSup->numOfRows; - - if (pSup->numOfRows > 0) { - SBufferWriter bw = tbufInitWriter(NULL, false); - interResToBinary(&bw, pSup->pResult, pSup->tagLen); - - pQueryInfo1->bufLen = (int32_t) tbufTell(&bw); - pQueryInfo1->buf = tbufGetData(&bw, true); - - // set the serialized binary string as the parameter of arithmetic expression - tbufCloseWriter(&bw); - } - - taosArrayDestroyEx(pSup->pResult, freeInterResult); - taosArrayDestroy(pSup->pColsInfo); - tfree(pSup); - - taos_free_result(pSql); - - if (resRows == 0) { - pParent->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - (*pParent->fp)(pParent->param, pParent, 0); - return; - } - - pQueryInfo1->round = 1; - executeQuery(pParent, pQueryInfo1); -} - -void tscFirstRoundCallback(void* param, TAOS_RES* tres, int code) { - SFirstRoundQuerySup* pSup = (SFirstRoundQuerySup*) param; - - SSqlObj* pSql = (SSqlObj*) tres; - int32_t c = taos_errno(pSql); - - if (c != TSDB_CODE_SUCCESS) { - SSqlObj* parent = pSup->pParent; - - destroySup(pSup); - taos_free_result(pSql); - parent->res.code = code; - tscAsyncResultOnError(parent); - return; - } - - taos_fetch_rows_a(tres, tscFirstRoundRetrieveCallback, param); -} - -int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMetaInfo* pTableMetaInfo1 = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - - SFirstRoundQuerySup *pSup = calloc(1, sizeof(SFirstRoundQuerySup)); - - pSup->pParent = pSql; - pSup->interval = pQueryInfo->interval.interval; - pSup->pResult = taosArrayInit(6, sizeof(SStddevInterResult)); - pSup->pColsInfo = taosArrayInit(6, sizeof(int16_t)); // result column id - - SSqlObj *pNew = createSubqueryObj(pSql, 0, tscFirstRoundCallback, pSup, TSDB_SQL_SELECT, NULL); - SSqlCmd *pCmd = &pNew->cmd; - - SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pCmd); - assert(pQueryInfo->numOfTables == 1); - - SArray* pColList = pNewQueryInfo->colList; - pNewQueryInfo->colList = NULL; - pNewQueryInfo->fillType = TSDB_FILL_NONE; - - tscClearSubqueryInfo(pCmd); - tscFreeSqlResult(pSql); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); - - tscInitQueryInfo(pNewQueryInfo); - - // add the group cond - pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; - if (pQueryInfo->groupbyExpr.columnInfo != NULL) { - pNewQueryInfo->groupbyExpr.columnInfo = taosArrayDup(pQueryInfo->groupbyExpr.columnInfo); - if (pNewQueryInfo->groupbyExpr.columnInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - } - - // add the tag filter cond - if (tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (tscColCondCopy(&pNewQueryInfo->colCond, pQueryInfo->colCond, pTableMetaInfo->pTableMeta->id.uid, 0) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - pNewQueryInfo->window = pQueryInfo->window; - pNewQueryInfo->interval = pQueryInfo->interval; - pNewQueryInfo->sessionWindow = pQueryInfo->sessionWindow; - - pCmd->command = TSDB_SQL_SELECT; - pNew->fp = tscFirstRoundCallback; - - int32_t numOfExprs = (int32_t) tscNumOfExprs(pQueryInfo); - - int32_t index = 0; - for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_TS && pQueryInfo->interval.interval > 0) { - taosArrayPush(pSup->pColsInfo, &pExpr->base.resColId); - - SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - SSchema* schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->base.colInfo.colId); - - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TS, &colIndex, schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); - p->base.resColId = pExpr->base.resColId; // update the result column id - } else if (pExpr->base.functionId == TSDB_FUNC_STDDEV_DST) { - taosArrayPush(pSup->pColsInfo, &pExpr->base.resColId); - - SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pExpr->base.colInfo.colIndex}; - SSchema schema = {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)}; - tstrncpy(schema.name, pExpr->base.aliasName, tListLen(schema.name)); - - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_AVG, &colIndex, &schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); - p->base.resColId = pExpr->base.resColId; // update the result column id - } else if (pExpr->base.functionId == TSDB_FUNC_TAG) { - pSup->tagLen += pExpr->base.resBytes; - SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pExpr->base.colInfo.colIndex}; - - SSchema* schema = NULL; - if (pExpr->base.colInfo.colId != TSDB_TBNAME_COLUMN_INDEX) { - schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->base.colInfo.colId); - } else { - schema = tGetTbnameColumnSchema(); - } - - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TAG, &colIndex, schema, TSDB_COL_TAG, getNewResColId(pCmd)); - p->base.resColId = pExpr->base.resColId; - } else if (pExpr->base.functionId == TSDB_FUNC_PRJ) { - int32_t num = (int32_t) taosArrayGetSize(pNewQueryInfo->groupbyExpr.columnInfo); - for(int32_t k = 0; k < num; ++k) { - SColIndex* pIndex = taosArrayGet(pNewQueryInfo->groupbyExpr.columnInfo, k); - if (pExpr->base.colInfo.colId == pIndex->colId) { - pSup->tagLen += pExpr->base.resBytes; - taosArrayPush(pSup->pColsInfo, &pExpr->base.resColId); - - SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pIndex->colIndex}; - SSchema* schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->base.colInfo.colId); - - //doLimitOutputNormalColOfGroupby - SExprInfo* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_PRJ, &colIndex, schema, TSDB_COL_NORMAL, getNewResColId(pCmd)); - p->base.numOfParams = 1; - p->base.param[0].i64 = 1; - p->base.param[0].nType = TSDB_DATA_TYPE_INT; - p->base.resColId = pExpr->base.resColId; // update the result column id - } - } - } - } - - // add the normal column filter cond - if (pColList != NULL) { - size_t s = taosArrayGetSize(pColList); - for (int32_t i = 0; i < s; ++i) { - SColumn *pCol = taosArrayGetP(pColList, i); - - if (pCol->info.flist.numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. - int32_t index1 = tscColumnExists(pNewQueryInfo->colList, pCol->columnIndex, pCol->tableUid); - if (index1 >= 0) { - SColumn* x = taosArrayGetP(pNewQueryInfo->colList, index1); - tscColumnCopy(x, pCol); - } else { - SSchema ss = {.type = (uint8_t)pCol->info.type, .bytes = pCol->info.bytes, .colId = (int16_t)pCol->columnIndex}; - tscColumnListInsert(pNewQueryInfo->colList, pCol->columnIndex, pCol->tableUid, &ss); - int32_t ti = tscColumnExists(pNewQueryInfo->colList, pCol->columnIndex, pCol->tableUid); - assert(ti >= 0); - SColumn* x = taosArrayGetP(pNewQueryInfo->colList, ti); - tscColumnCopy(x, pCol); - } - } - } - - tscColumnListDestroy(pColList); - } - - tscInsertPrimaryTsSourceColumn(pNewQueryInfo, pTableMetaInfo->pTableMeta->id.uid); - tscTansformFuncForSTableQuery(pNewQueryInfo); - - tscDebug( - "0x%"PRIx64" first round subquery:0x%"PRIx64" tableIndex:%d, vgroupIndex:%d, numOfVgroups:%d, type:%d, query to retrieve timestamps, " - "numOfExpr:%" PRIzu ", colList:%d, numOfOutputFields:%d, name:%s", - pSql->self, pNew->self, 0, pTableMetaInfo->vgroupIndex, pTableMetaInfo->vgroupList->numOfVgroups, pNewQueryInfo->type, - tscNumOfExprs(pNewQueryInfo), index+1, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); - - tscHandleMasterSTableQuery(pNew); - return TSDB_CODE_SUCCESS; - - _error: - destroySup(pSup); - taos_free_result(pNew); - pSql->res.code = terrno; - tscAsyncResultOnError(pSql); - return terrno; -} - -typedef struct SPair { - int32_t first; - int32_t second; -} SPair; - -static void doSendQueryReqs(SSchedMsg* pSchedMsg) { - SSqlObj* pSql = pSchedMsg->ahandle; - SPair* p = pSchedMsg->msg; - - for (int32_t i = p->first; i < p->second; ++i) { - if (i >= pSql->subState.numOfSub) { - tfree(p); - return; - } - SSqlObj* pSub = pSql->pSubs[i]; - SRetrieveSupport* pSupport = pSub->param; - - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" launch subquery, orderOfSub:%d.", pSql->self, pSub->self, pSupport->subqueryIndex); - tscBuildAndSendRequest(pSub, NULL); - } - - tfree(p); -} - -static void doConcurrentlySendSubQueries(SSqlObj* pSql) { - SSubqueryState *pState = &pSql->subState; - - // concurrently sent the query requests. - const int32_t MAX_REQUEST_PER_TASK = 4; - - int32_t numOfTasks = (pState->numOfSub + MAX_REQUEST_PER_TASK - 1)/MAX_REQUEST_PER_TASK; - assert(numOfTasks >= 1); - - int32_t num; - if (pState->numOfSub / numOfTasks == MAX_REQUEST_PER_TASK) { - num = MAX_REQUEST_PER_TASK; - } else { - num = pState->numOfSub / numOfTasks + 1; - } - tscDebug("0x%"PRIx64 " query will be sent by %d threads", pSql->self, numOfTasks); - - for(int32_t j = 0; j < numOfTasks; ++j) { - SSchedMsg schedMsg = {0}; - schedMsg.fp = doSendQueryReqs; - schedMsg.ahandle = (void*)pSql; - - schedMsg.thandle = NULL; - SPair* p = calloc(1, sizeof(SPair)); - p->first = j * num; - - if (j == numOfTasks - 1) { - p->second = pState->numOfSub; - } else { - p->second = (j + 1) * num; - } - - schedMsg.msg = p; - taosScheduleTask(tscQhandle, &schedMsg); - } -} - -int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - // pRes->code check only serves in launching super table sub-queries - if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { - pCmd->command = TSDB_SQL_RETRIEVE_GLOBALMERGE; // enable the abort of kill super table function. - return pRes->code; - } - - tExtMemBuffer **pMemoryBuf = NULL; - tOrderDescriptor *pDesc = NULL; - - pRes->qId = 0x1; // hack the qhandle check - - const uint32_t nBufferSize = (1u << 18u); // 256KB, default buffer size - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SSubqueryState *pState = &pSql->subState; - - int32_t numOfSub = (pTableMetaInfo->pVgroupTables == NULL) ? pTableMetaInfo->vgroupList->numOfVgroups - : (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - - int32_t ret = doInitSubState(pSql, numOfSub); - if (ret != 0) { - tscAsyncResultOnError(pSql); - return ret; - } - - ret = tscCreateGlobalMergerEnv(pQueryInfo, &pMemoryBuf, pSql->subState.numOfSub, &pDesc, nBufferSize, pSql->self); - if (ret != 0) { - pRes->code = ret; - tscAsyncResultOnError(pSql); - tfree(pDesc); - tfree(pMemoryBuf); - return ret; - } - - tscDebug("0x%"PRIx64" retrieved query data from %d vnode(s)", pSql->self, pState->numOfSub); - pRes->code = TSDB_CODE_SUCCESS; - - int32_t i = 0; - for (; i < pState->numOfSub; ++i) { - SRetrieveSupport *trs = (SRetrieveSupport *)calloc(1, sizeof(SRetrieveSupport)); - if (trs == NULL) { - tscError("0x%"PRIx64" failed to malloc buffer for SRetrieveSupport, orderOfSub:%d, reason:%s", pSql->self, i, strerror(errno)); - break; - } - - trs->pExtMemBuffer = pMemoryBuf; - trs->pOrderDescriptor = pDesc; - - trs->localBuffer = (tFilePage *)malloc(nBufferSize + sizeof(tFilePage)); - if (trs->localBuffer == NULL) { - tscError("0x%"PRIx64" failed to malloc buffer for local buffer, orderOfSub:%d, reason:%s", pSql->self, i, strerror(errno)); - tfree(trs); - break; - } - - trs->localBuffer->num = 0; - trs->subqueryIndex = i; - trs->pParentSql = pSql; - - SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL); - if (pNew == NULL) { - tscError("0x%"PRIx64" failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql->self, i, strerror(errno)); - tfree(trs->localBuffer); - tfree(trs); - break; - } - - // todo handle multi-vnode situation - if (pQueryInfo->tsBuf) { - SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); - pNewQueryInfo->tsBuf = tsBufClone(pQueryInfo->tsBuf); - assert(pNewQueryInfo->tsBuf != NULL); - } - - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" create subquery success. orderOfSub:%d", pSql->self, pNew->self, - trs->subqueryIndex); - } - - if (i < pState->numOfSub) { - tscError("0x%"PRIx64" failed to prepare subquery structure and launch subqueries", pSql->self); - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - - tscDestroyGlobalMergerEnv(pMemoryBuf, pDesc, pState->numOfSub); - doCleanupSubqueries(pSql, i); - return pRes->code; // free all allocated resource - } - - if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { - tscDestroyGlobalMergerEnv(pMemoryBuf, pDesc, pState->numOfSub); - doCleanupSubqueries(pSql, i); - return pRes->code; - } - - doConcurrentlySendSubQueries(pSql); - return TSDB_CODE_SUCCESS; -} - -void tscFreeRetrieveSup(SSqlObj *pSql) { - SRetrieveSupport *trsupport = pSql->param; - - void* p = atomic_val_compare_exchange_ptr(&pSql->param, trsupport, 0); - if (p == NULL) { - tscDebug("0x%"PRIx64" retrieve supp already released", pSql->self); - return; - } - - tscDebug("0x%"PRIx64" start to free subquery supp obj:%p", pSql->self, trsupport); - tfree(trsupport->localBuffer); - tfree(trsupport); -} - -static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); -static void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows); - -static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t code) { -// set no disk space error info - tscError("sub:0x%"PRIx64" failed to flush data to disk, reason:%s", ((SSqlObj *)tres)->self, tstrerror(code)); - SSqlObj* pParentSql = trsupport->pParentSql; - - pParentSql->res.code = code; - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscHandleSubqueryError(trsupport, tres, pParentSql->res.code); -} - -/* - * current query failed, and the retry count is less than the available - * count, retry query clear previous retrieved data, then launch a new sub query - */ -static int32_t tscReissueSubquery(SRetrieveSupport *oriTrs, SSqlObj *pSql, int32_t code, int32_t *sent) { - *sent = 0; - - SRetrieveSupport *trsupport = malloc(sizeof(SRetrieveSupport)); - if (trsupport == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - memcpy(trsupport, oriTrs, sizeof(*trsupport)); - - const uint32_t nBufferSize = (1u << 16u); // 64KB - trsupport->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage)); - if (trsupport->localBuffer == NULL) { - tscError("0x%"PRIx64" failed to malloc buffer for local buffer, reason:%s", pSql->self, strerror(errno)); - tfree(trsupport); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SSqlObj *pParentSql = trsupport->pParentSql; - int32_t subqueryIndex = trsupport->subqueryIndex; - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - SVgroupMsg* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; - - tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]); - - // clear local saved number of results - trsupport->localBuffer->num = 0; - tscError("0x%"PRIx64" sub:0x%"PRIx64" retrieve/query failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql->self, pSql->self, - tstrerror(code), subqueryIndex, trsupport->numOfRetry); - - SSqlObj *pNew = tscCreateSTableSubquery(trsupport->pParentSql, trsupport, pSql); - if (pNew == NULL) { - tscError("0x%"PRIx64" sub:0x%"PRIx64" failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d", - oriTrs->pParentSql->self, pSql->self, tstrerror(terrno), pVgroup->vgId, oriTrs->subqueryIndex); - - pParentSql->res.code = terrno; - oriTrs->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - - tfree(trsupport); - return pParentSql->res.code; - } - - int32_t ret = tscBuildAndSendRequest(pNew, NULL); - - *sent = 1; - - // if failed to process sql, let following code handle the pSql - if (ret == TSDB_CODE_SUCCESS) { - tscFreeRetrieveSup(pSql); - taos_free_result(pSql); - return ret; - } else { - pParentSql->pSubs[trsupport->subqueryIndex] = pSql; - tscFreeRetrieveSup(pNew); - taos_free_result(pNew); - return ret; - } -} - -void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { - // it has been freed already - if (pSql->param != trsupport || pSql->param == NULL) { - return; - } - - SSqlObj *pParentSql = trsupport->pParentSql; - int32_t subqueryIndex = trsupport->subqueryIndex; - - assert(pSql != NULL); - - SSubqueryState* pState = &pParentSql->subState; - - // retrieved in subquery failed. OR query cancelled in retrieve phase. - if (taos_errno(pSql) == TSDB_CODE_SUCCESS && pParentSql->res.code != TSDB_CODE_SUCCESS) { - - /* - * kill current sub-query connection, which may retrieve data from vnodes; - * Here we get: pPObj->res.code == TSDB_CODE_TSC_QUERY_CANCELLED - */ - pSql->res.numOfRows = 0; - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; // disable retry efforts - tscDebug("0x%"PRIx64" query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%s", pParentSql->self, pSql, - subqueryIndex, tstrerror(pParentSql->res.code)); - } - - if (numOfRows >= 0) { // current query is successful, but other sub query failed, still abort current query. - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" retrieve numOfRows:%d,orderOfSub:%d", pParentSql->self, pSql->self, numOfRows, subqueryIndex); - tscError("0x%"PRIx64" sub:0x%"PRIx64" abort further retrieval due to other queries failure,orderOfSub:%d,code:%s", pParentSql->self, pSql->self, - subqueryIndex, tstrerror(pParentSql->res.code)); - } else { - if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pParentSql->res.code == TSDB_CODE_SUCCESS) { - int32_t sent = 0; - - tscReissueSubquery(trsupport, pSql, numOfRows, &sent); - if (sent) { - return; - } - } else { // reach the maximum retry count, abort - atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, numOfRows); - tscError("0x%"PRIx64" sub:0x%"PRIx64" retrieve failed, code:%s, orderOfSub:%d FAILED. no more retry, set global code:%s", pParentSql->self, pSql->self, - tstrerror(numOfRows), subqueryIndex, tstrerror(pParentSql->res.code)); - } - } - - if (!subAndCheckDone(pSql, pParentSql, subqueryIndex)) { - tscDebug("0x%"PRIx64" sub:0x%"PRIx64",%d freed, not finished, total:%d", pParentSql->self, - pSql->self, trsupport->subqueryIndex, pState->numOfSub); - - tscFreeRetrieveSup(pSql); - return; - } - - // all subqueries are failed - tscError("0x%"PRIx64" retrieve from %d vnode(s) completed,code:%s.FAILED.", pParentSql->self, pState->numOfSub, - tstrerror(pParentSql->res.code)); - - // release allocated resource - tscDestroyGlobalMergerEnv(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor, pState->numOfSub); - tscFreeRetrieveSup(pSql); - - // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pParentSql->cmd); - - if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { - - int32_t code = pParentSql->res.code; - SSqlObj *userSql = NULL; - if (pParentSql->param) { - userSql = ((SRetrieveSupport*)pParentSql->param)->pParentSql; - } - - if (userSql == NULL) { - userSql = pParentSql; - } - - if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) && userSql->retry < userSql->maxRetry) { - if (userSql != pParentSql) { - tscFreeRetrieveSup(pParentSql); - } - - tscFreeSubobj(userSql); - tfree(userSql->pSubs); - - userSql->res.code = TSDB_CODE_SUCCESS; - userSql->retry++; - - tscDebug("0x%"PRIx64" retry parse sql and send query, prev error: %s, retry:%d", userSql->self, - tstrerror(code), userSql->retry); - - tscResetSqlCmd(&userSql->cmd, true, userSql->self); - code = tsParseSql(userSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - return; - } - - if (code != TSDB_CODE_SUCCESS) { - userSql->res.code = code; - tscAsyncResultOnError(userSql); - return; - } - - pQueryInfo = tscGetQueryInfo(&userSql->cmd); - executeQuery(userSql, pQueryInfo); - } else { - (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code); - } - } else { // regular super table query - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscAsyncResultOnError(pParentSql); - } - } -} - -static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) { - if (trsupport->pExtMemBuffer == NULL){ - return; - } - int32_t idx = trsupport->subqueryIndex; - SSqlObj * pParentSql = trsupport->pParentSql; - tOrderDescriptor *pDesc = trsupport->pOrderDescriptor; - - SSubqueryState* pState = &pParentSql->subState; - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - - // data in from current vnode is stored in cache and disk - uint32_t numOfRowsFromSubquery = (uint32_t)(trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->num); - SVgroupsInfo* vgroupsInfo = pTableMetaInfo->vgroupList; - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" all data retrieved from ep:%s, vgId:%d, numOfRows:%d, orderOfSub:%d", pParentSql->self, - pSql->self, vgroupsInfo->vgroups[0].epAddr[0].fqdn, vgroupsInfo->vgroups[0].vgId, numOfRowsFromSubquery, idx); - - tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); - -#ifdef _DEBUG_VIEW - printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->num); - SSrcColumnInfo colInfo[256] = {0}; - tscGetSrcColumnInfo(colInfo, pQueryInfo); - tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->num, - trsupport->localBuffer->num, colInfo); -#endif - - if (tsTotalTmpDirGB != 0 && tsAvailTmpDirectorySpace < tsReservedTmpDirectorySpace) { - tscError("0x%"PRIx64" sub:0x%"PRIx64" client disk space remain %.3f GB, need at least %.3f GB, stop query", pParentSql->self, pSql->self, - tsAvailTmpDirectorySpace, tsReservedTmpDirectorySpace); - tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_TSC_NO_DISKSPACE); - return; - } - - // each result for a vnode is ordered as an independant list, - // then used as an input of loser tree for disk-based merge - int32_t code = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pQueryInfo->groupbyExpr.orderType); - if (code != 0) { // set no disk space error info, and abort retry - tscAbortFurtherRetryRetrieval(trsupport, pSql, code); - return; - } - - if (!subAndCheckDone(pSql, pParentSql, idx)) { - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" orderOfSub:%d freed, not finished", pParentSql->self, pSql->self, - trsupport->subqueryIndex); - - tscFreeRetrieveSup(pSql); - return; - } - - // all sub-queries are returned, start to local merge process - pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; - - tscDebug("0x%"PRIx64" retrieve from %d vnodes completed.final NumOfRows:%" PRId64 ",start to build loser tree", - pParentSql->self, pState->numOfSub, pState->numOfRetrievedRows); - - SQueryInfo *pPQueryInfo = tscGetQueryInfo(&pParentSql->cmd); - - code = tscCreateGlobalMerger(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, pPQueryInfo, &pParentSql->res.pMerger, pParentSql->self); - pParentSql->res.code = code; - - if (code == TSDB_CODE_SUCCESS && trsupport->pExtMemBuffer == NULL) { - pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; // no result, set the result empty - } else { - pParentSql->cmd.command = TSDB_SQL_RETRIEVE_GLOBALMERGE; - } - - tscCreateResPointerInfo(&pParentSql->res, pPQueryInfo); - - tscDebug("0x%"PRIx64" build loser tree completed", pParentSql->self); - - pParentSql->res.precision = pSql->res.precision; - pParentSql->res.numOfRows = 0; - pParentSql->res.row = 0; - - tscFreeRetrieveSup(pSql); - - // set the command flag must be after the semaphore been correctly set. - if (pParentSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - pParentSql->cmd.command = TSDB_SQL_RETRIEVE_GLOBALMERGE; - - SQueryInfo *pQueryInfo2 = tscGetQueryInfo(&pParentSql->cmd); - - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t j = 0; j < size; ++j) { - SExprInfo* pExprInfo = tscExprGet(pQueryInfo2, j); - - int32_t functionId = pExprInfo->base.functionId; - if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo2->pUdfInfo, -1 * functionId - 1); - code = initUdfInfo(pUdfInfo); - if (code != TSDB_CODE_SUCCESS) { - pParentSql->res.code = code; - tscAsyncResultOnError(pParentSql); - } - } - } - } - - if (pParentSql->res.code == TSDB_CODE_SUCCESS) { - (*pParentSql->fp)(pParentSql->param, pParentSql, 0); - } else { - tscAsyncResultOnError(pParentSql); - } -} - -static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { - SSqlObj *pSql = (SSqlObj *)tres; - assert(pSql != NULL); - - // this query has been freed already - SRetrieveSupport *trsupport = (SRetrieveSupport *)param; - if (pSql->param == NULL || param == NULL) { - tscDebug("0x%"PRIx64" already freed in dnodecallback", pSql->self); - return; - } - - tOrderDescriptor *pDesc = trsupport->pOrderDescriptor; - int32_t idx = trsupport->subqueryIndex; - SSqlObj * pParentSql = trsupport->pParentSql; - - SSubqueryState* pState = &pParentSql->subState; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - SVgroupMsg *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; - - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscDebug("0x%"PRIx64" query cancelled/failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", - pParentSql->self, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(numOfRows), tstrerror(pParentSql->res.code)); - - tscHandleSubqueryError(param, tres, numOfRows); - return; - } - - if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { - assert(numOfRows == taos_errno(pSql)); - - if (numOfRows == TSDB_CODE_TSC_QUERY_CANCELLED) { - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - } - - if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscError("0x%"PRIx64" sub:0x%"PRIx64" failed code:%s, retry:%d", pParentSql->self, pSql->self, tstrerror(numOfRows), trsupport->numOfRetry); - - int32_t sent = 0; - - tscReissueSubquery(trsupport, pSql, numOfRows, &sent); - if (sent) { - return; - } - } else { - tscDebug("0x%"PRIx64" sub:%p reach the max retry times, set global code:%s", pParentSql->self, pSql, tstrerror(numOfRows)); - atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, numOfRows); // set global code and abort - } - - tscHandleSubqueryError(param, tres, numOfRows); - return; - } - - SSqlRes * pRes = &pSql->res; - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - if (numOfRows > 0) { - assert(pRes->numOfRows == numOfRows); - int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows); - - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" retrieve numOfRows:%d totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d", - pParentSql->self, pSql->self, pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx); - - if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0) && !(tscGetQueryInfo(&pParentSql->cmd)->distinct)) { - tscError("0x%"PRIx64" sub:0x%"PRIx64" num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64, - pParentSql->self, pSql->self, tsMaxNumOfOrderedResults, num); - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_SORTED_RES_TOO_MANY); - return; - } - -#ifdef _DEBUG_VIEW - printf("received data from vnode: %"PRIu64" rows\n", pRes->numOfRows); - SSrcColumnInfo colInfo[256] = {0}; - - tscGetSrcColumnInfo(colInfo, pQueryInfo); - tColModelDisplayEx(pDesc->pColumnModel, pRes->data, pRes->numOfRows, pRes->numOfRows, colInfo); -#endif - - // no disk space for tmp directory - if (tsTotalTmpDirGB != 0 && tsAvailTmpDirectorySpace < tsReservedTmpDirectorySpace) { - tscError("0x%"PRIx64" sub:0x%"PRIx64" client disk space remain %.3f GB, need at least %.3f GB, stop query", pParentSql->self, pSql->self, - tsAvailTmpDirectorySpace, tsReservedTmpDirectorySpace); - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE); - return; - } - - int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data, - pRes->numOfRows, pQueryInfo->groupbyExpr.orderType); - if (ret != 0) { // set no disk space error info, and abort retry - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE); - } else if (pRes->completed) { - tscAllDataRetrievedFromDnode(trsupport, pSql); - } else { // continue fetch data from dnode - taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); - } - - } else { // all data has been retrieved to client - tscAllDataRetrievedFromDnode(trsupport, pSql); - } -} - -static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { - const int32_t table_index = 0; - - SSqlObj *pNew = createSubqueryObj(pSql, table_index, tscRetrieveDataRes, trsupport, TSDB_SQL_SELECT, prevSqlObj); - if (pNew != NULL) { // the sub query of two-stage super table query - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pNew->cmd); - - pNew->cmd.active = pQueryInfo; - pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; - - // clear the limit/offset info, since it should not be sent to vnode to be executed. - pQueryInfo->limit.limit = -1; - pQueryInfo->limit.offset = 0; - - assert(trsupport->subqueryIndex < pSql->subState.numOfSub); - - // launch subquery for each vnode, so the subquery index equals to the vgroupIndex. - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, table_index); - pTableMetaInfo->vgroupIndex = trsupport->subqueryIndex; - - pSql->pSubs[trsupport->subqueryIndex] = pNew; - } - - return pNew; -} - -// todo there is are race condition in this function, while cancel is called by user. -void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { - // the param may be null, since it may be done by other query threads. and the asyncOnError may enter in this - // function while kill query by a user. - if (param == NULL) { - assert(code != TSDB_CODE_SUCCESS); - return; - } - - SRetrieveSupport *trsupport = (SRetrieveSupport *) param; - - SSqlObj* pParentSql = trsupport->pParentSql; - SSqlObj* pSql = (SSqlObj *) tres; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - assert(pQueryInfo->numOfTables == 1); - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - SVgroupMsg* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex]; - - // stable query killed or other subquery failed, all query stopped - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscError("0x%"PRIx64" query cancelled or failed, sub:0x%"PRIx64", vgId:%d, orderOfSub:%d, code:%s, global code:%s", - pParentSql->self, pSql->self, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(code), tstrerror(pParentSql->res.code)); - - tscHandleSubqueryError(param, tres, code); - return; - } - - /* - * if a subquery on a vnode failed, all retrieve operations from vnode that occurs later - * than this one are actually not necessary, we simply call the tscRetrieveFromDnodeCallBack - * function to abort current and remain retrieve process. - * - * NOTE: thread safe is required. - */ - if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { - assert(code == taos_errno(pSql)); - - if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && (code != TSDB_CODE_TDB_INVALID_TABLE_ID && code != TSDB_CODE_VND_INVALID_VGROUP_ID)) { - tscError("0x%"PRIx64" sub:0x%"PRIx64" failed code:%s, retry:%d", pParentSql->self, pSql->self, tstrerror(code), trsupport->numOfRetry); - - int32_t sent = 0; - tscReissueSubquery(trsupport, pSql, code, &sent); - if (sent) { - return; - } - } else { - tscError("0x%"PRIx64" sub:0x%"PRIx64" reach the max retry times or no need to retry, set global code:%s", pParentSql->self, pSql->self, tstrerror(code)); - atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, code); // set global code and abort - } - - tscHandleSubqueryError(param, tres, pParentSql->res.code); - return; - } - - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql->self, - pSql->self, pVgroup->epAddr[pSql->epSet.inUse].fqdn, pVgroup->vgId, trsupport->subqueryIndex); - - if (pSql->res.qId == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode - tscRetrieveFromDnodeCallBack(param, pSql, 0); - } else { - taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); - } -} - -static bool needRetryInsert(SSqlObj* pParentObj, int32_t numOfSub) { - if (pParentObj->retry > pParentObj->maxRetry) { - tscError("0x%"PRIx64" max retry reached, abort the retry effort", pParentObj->self); - return false; - } - - for (int32_t i = 0; i < numOfSub; ++i) { - int32_t code = pParentObj->pSubs[i]->res.code; - if (code == TSDB_CODE_SUCCESS) { - continue; - } - - if (code != TSDB_CODE_TDB_TABLE_RECONFIGURE && code != TSDB_CODE_TDB_INVALID_TABLE_ID && - code != TSDB_CODE_VND_INVALID_VGROUP_ID && code != TSDB_CODE_RPC_NETWORK_UNAVAIL && - code != TSDB_CODE_APP_NOT_READY) { - pParentObj->res.code = code; - return false; - } - } - - return true; -} - -static void doFreeInsertSupporter(SSqlObj* pSqlObj) { - assert(pSqlObj != NULL && pSqlObj->subState.numOfSub > 0); - - for(int32_t i = 0; i < pSqlObj->subState.numOfSub; ++i) { - SSqlObj* pSql = pSqlObj->pSubs[i]; - tfree(pSql->param); - } -} - -static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) { - SInsertSupporter *pSupporter = (SInsertSupporter *)param; - SSqlObj* pParentObj = pSupporter->pSql; - - // record the total inserted rows - if (numOfRows > 0) { - atomic_add_fetch_32(&pParentObj->res.numOfRows, numOfRows); - } - - if (taos_errno(tres) != TSDB_CODE_SUCCESS) { - SSqlObj* pSql = (SSqlObj*) tres; - assert(pSql != NULL && pSql->res.code == numOfRows); - - pParentObj->res.code = pSql->res.code; - - // set the flag in the parent sqlObj - if (pSql->cmd.insertParam.schemaAttached) { - pParentObj->cmd.insertParam.schemaAttached = 1; - } - } - - if (!subAndCheckDone(tres, pParentObj, pSupporter->index)) { - // concurrency problem, other thread already release pParentObj - //tscDebug("0x%"PRIx64" insert:%p,%d completed, total:%d", pParentObj->self, tres, suppIdx, pParentObj->subState.numOfSub); - return; - } - - // restore user defined fp - pParentObj->fp = pParentObj->fetchFp; - int32_t numOfSub = pParentObj->subState.numOfSub; - doFreeInsertSupporter(pParentObj); - - if (pParentObj->res.code == TSDB_CODE_SUCCESS) { - tscDebug("0x%"PRIx64" Async insertion completed, total inserted:%d", pParentObj->self, pParentObj->res.numOfRows); - - // todo remove this parameter in async callback function definition. - // all data has been sent to vnode, call user function - int32_t v = (pParentObj->res.code != TSDB_CODE_SUCCESS) ? pParentObj->res.code : (int32_t)pParentObj->res.numOfRows; - (*pParentObj->fp)(pParentObj->param, pParentObj, v); - } else { - if (!needRetryInsert(pParentObj, numOfSub)) { - tscAsyncResultOnError(pParentObj); - return; - } - - int32_t numOfFailed = 0; - for(int32_t i = 0; i < numOfSub; ++i) { - SSqlObj* pSql = pParentObj->pSubs[i]; - if (pSql->res.code != TSDB_CODE_SUCCESS) { - numOfFailed += 1; - - // clean up tableMeta in cache - tscFreeQueryInfo(&pSql->cmd, false, pSql->self); - SQueryInfo* pQueryInfo = tscGetQueryInfoS(&pSql->cmd); - STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentObj->cmd, 0); - tscAddTableMetaInfo(pQueryInfo, &pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); - - subquerySetState(pSql, &pParentObj->subState, i, 0); - - tscDebug("0x%"PRIx64", failed sub:%d, %p", pParentObj->self, i, pSql); - } - } - - tscError("0x%"PRIx64" Async insertion completed, total inserted:%d rows, numOfFailed:%d, numOfTotal:%d", pParentObj->self, - pParentObj->res.numOfRows, numOfFailed, numOfSub); - - tscDebug("0x%"PRIx64" cleanup %d tableMeta in hashTable before reparse sql", pParentObj->self, pParentObj->cmd.insertParam.numOfTables); - for(int32_t i = 0; i < pParentObj->cmd.insertParam.numOfTables; ++i) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(pParentObj->cmd.insertParam.pTableNameList[i], name); - taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - } - - pParentObj->res.code = TSDB_CODE_SUCCESS; - tscResetSqlCmd(&pParentObj->cmd, false, pParentObj->self); - - // in case of insert, redo parsing the sql string and build new submit data block for two reasons: - // 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly. - // 2. vnode may need the schema information along with submit block to update its local table schema. - tscDebug("0x%"PRIx64" re-parse sql to generate submit data, retry:%d", pParentObj->self, pParentObj->retry); - pParentObj->retry++; - - int32_t code = tsParseSql(pParentObj, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return; - - if (code != TSDB_CODE_SUCCESS) { - pParentObj->res.code = code; - tscAsyncResultOnError(pParentObj); - return; - } - - tscHandleMultivnodeInsert(pParentObj); - } -} - -/** - * it is a subquery, so after parse the sql string, copy the submit block to payload of itself - * @param pSql - * @return - */ -int32_t tscHandleInsertRetry(SSqlObj* pParent, SSqlObj* pSql) { - assert(pSql != NULL && pSql->param != NULL); - SSqlRes* pRes = &pSql->res; - - SInsertSupporter* pSupporter = (SInsertSupporter*) pSql->param; - assert(pSupporter->index < pSupporter->pSql->subState.numOfSub); - - STableDataBlocks* pTableDataBlock = taosArrayGetP(pParent->cmd.insertParam.pDataBlocks, pSupporter->index); - int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock); - - if ((pRes->code = code)!= TSDB_CODE_SUCCESS) { - tscAsyncResultOnError(pSql); - return code; // here the pSql may have been released already. - } - - return tscBuildAndSendRequest(pSql, NULL); -} - -int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - // it is the failure retry insert - if (pSql->pSubs != NULL) { - int32_t blockNum = (int32_t)taosArrayGetSize(pCmd->insertParam.pDataBlocks); - if (pSql->subState.numOfSub != blockNum) { - tscError("0x%"PRIx64" sub num:%d is not same with data block num:%d", pSql->self, pSql->subState.numOfSub, blockNum); - pRes->code = TSDB_CODE_TSC_APP_ERROR; - return pRes->code; - } - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - SInsertSupporter* pSup = calloc(1, sizeof(SInsertSupporter)); - pSup->index = i; - pSup->pSql = pSql; - - pSub->param = pSup; - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" launch sub insert, orderOfSub:%d", pSql->self, pSub->self, i); - if (pSub->res.code != TSDB_CODE_SUCCESS) { - tscHandleInsertRetry(pSql, pSub); - } - } - - return TSDB_CODE_SUCCESS; - } - - pSql->subState.numOfSub = (uint16_t)taosArrayGetSize(pCmd->insertParam.pDataBlocks); - assert(pSql->subState.numOfSub > 0); - - pRes->code = TSDB_CODE_SUCCESS; - - // the number of already initialized subqueries - int32_t numOfSub = 0; - - if (pSql->subState.states == NULL) { - pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(*pSql->subState.states)); - if (pSql->subState.states == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - pthread_mutex_init(&pSql->subState.mutex, NULL); - } - - memset(pSql->subState.states, 0, sizeof(*pSql->subState.states) * pSql->subState.numOfSub); - tscDebug("0x%"PRIx64" reset all sub states to 0", pSql->self); - - pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES); - if (pSql->pSubs == NULL) { - goto _error; - } - - tscDebug("0x%"PRIx64" submit data to %d vnode(s)", pSql->self, pSql->subState.numOfSub); - - while(numOfSub < pSql->subState.numOfSub) { - SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter)); - if (pSupporter == NULL) { - goto _error; - } - - pSupporter->pSql = pSql; - pSupporter->index = numOfSub; - - SSqlObj *pNew = createSimpleSubObj(pSql, multiVnodeInsertFinalize, pSupporter, TSDB_SQL_INSERT); - if (pNew == NULL) { - tscError("0x%"PRIx64" failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql->self, numOfSub, strerror(errno)); - goto _error; - } - - /* - * assign the callback function to fetchFp to make sure that the error process function can restore - * the callback function (multiVnodeInsertFinalize) correctly. - */ - pNew->fetchFp = pNew->fp; - pSql->pSubs[numOfSub] = pNew; - - STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->insertParam.pDataBlocks, numOfSub); - pRes->code = tscCopyDataBlockToPayload(pNew, pTableDataBlock); - if (pRes->code == TSDB_CODE_SUCCESS) { - tscDebug("0x%"PRIx64" sub:%p create subObj success. orderOfSub:%d", pSql->self, pNew, numOfSub); - numOfSub++; - } else { - tscDebug("0x%"PRIx64" prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%s", pSql->self, numOfSub, - pSql->subState.numOfSub, tstrerror(pRes->code)); - goto _error; - } - } - - if (numOfSub < pSql->subState.numOfSub) { - tscError("0x%"PRIx64" failed to prepare subObj structure and launch sub-insertion", pSql->self); - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); - - // use the local variable - for (int32_t j = 0; j < numOfSub; ++j) { - SSqlObj *pSub = pSql->pSubs[j]; - tscDebug("0x%"PRIx64" sub:%p launch sub insert, orderOfSub:%d", pSql->self, pSub, j); - tscBuildAndSendRequest(pSub, NULL); - } - - return TSDB_CODE_SUCCESS; - - _error: - return TSDB_CODE_TSC_OUT_OF_MEMORY; -} - -static char* getResultBlockPosition(SSqlCmd* pCmd, SSqlRes* pRes, int32_t columnIndex, int16_t* bytes) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - SInternalField* pInfo = (SInternalField*) TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, columnIndex); - assert(pInfo->pExpr->pExpr == NULL); - - *bytes = pInfo->pExpr->base.resBytes; - if (pRes->data != NULL) { - return pRes->data + pInfo->pExpr->base.offset * pRes->numOfRows + pRes->row * (*bytes); - } else { - return ((char*)pRes->urow[columnIndex]) + pRes->row * (*bytes); - } -} - -static void doBuildResFromSubqueries(SSqlObj* pSql) { - SSqlRes* pRes = &pSql->res; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - int32_t numOfRes = INT32_MAX; - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - int32_t remain = (int32_t)(pSub->res.numOfRows - pSub->res.row); - numOfRes = (int32_t)(TMIN(numOfRes, remain)); - } - - if (numOfRes == 0) { // no result any more, free all subquery objects - freeJoinSubqueryObj(pSql); - return; - } - - int32_t rowSize = tscGetResRowLength(pQueryInfo->exprList); - - assert(numOfRes * rowSize > 0); - char* tmp = realloc(pRes->pRsp, numOfRes * rowSize + sizeof(tFilePage)); - if (tmp == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return; - } else { - pRes->pRsp = tmp; - } - - tFilePage* pFilePage = (tFilePage*) pRes->pRsp; - pFilePage->num = numOfRes; - - pRes->data = pFilePage->data; - char* data = pRes->data; - - int16_t bytes = 0; - - tscRestoreFuncForSTableQuery(pQueryInfo); - tscFieldInfoUpdateOffset(pQueryInfo); - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - SQueryInfo* pSubQueryInfo = pSub->cmd.pQueryInfo; - tscRestoreFuncForSTableQuery(pSubQueryInfo); - tscFieldInfoUpdateOffset(pSubQueryInfo); - } - - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for(int32_t i = 0; i < numOfExprs; ++i) { - SColumnIndex* pIndex = &pRes->pColumnIndex[i]; - SSqlRes* pRes1 = &pSql->pSubs[pIndex->tableIndex]->res; - SSqlCmd* pCmd1 = &pSql->pSubs[pIndex->tableIndex]->cmd; - - char* pData = getResultBlockPosition(pCmd1, pRes1, pIndex->columnIndex, &bytes); - memcpy(data, pData, bytes * numOfRes); - - data += bytes * numOfRes; - } - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - pSub->res.row += numOfRes; - assert(pSub->res.row <= pSub->res.numOfRows); - } - - pRes->numOfRows = numOfRes; - pRes->numOfClauseTotal += numOfRes; - - int32_t finalRowSize = 0; - for(int32_t i = 0; i < tscNumOfFields(pQueryInfo); ++i) { - TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - finalRowSize += pField->bytes; - } - - doArithmeticCalculate(pQueryInfo, pFilePage, rowSize, finalRowSize); - - pRes->data = pFilePage->data; - tscSetResRawPtr(pRes, pQueryInfo); -} - -void tscBuildResFromSubqueries(SSqlObj *pSql) { - SSqlRes* pRes = &pSql->res; - - if (pRes->code != TSDB_CODE_SUCCESS) { - tscAsyncResultOnError(pSql); - return; - } - - if (pRes->tsrow == NULL) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - pRes->numOfCols = (int16_t) tscNumOfExprs(pQueryInfo); - - pRes->tsrow = calloc(pRes->numOfCols, POINTER_BYTES); - pRes->urow = calloc(pRes->numOfCols, POINTER_BYTES); - pRes->buffer = calloc(pRes->numOfCols, POINTER_BYTES); - pRes->length = calloc(pRes->numOfCols, sizeof(int32_t)); - - if (pRes->tsrow == NULL || pRes->buffer == NULL || pRes->length == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscAsyncResultOnError(pSql); - return; - } - } - - assert (pRes->row >= pRes->numOfRows); - doBuildResFromSubqueries(pSql); - if (pRes->code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); - } else { - tscAsyncResultOnError(pSql); - } -} - -char *getArithmeticInputSrc(void *param, const char *name, int32_t colId) { - SArithmeticSupport *pSupport = (SArithmeticSupport *) param; - - int32_t index = -1; - SExprInfo* pExpr = NULL; - - for (int32_t i = 0; i < pSupport->numOfCols; ++i) { - pExpr = taosArrayGetP(pSupport->exprList, i); - if (strncmp(name, pExpr->base.aliasName, sizeof(pExpr->base.aliasName) - 1) == 0) { - index = i; - break; - } - } - - assert(index >= 0 && index < pSupport->numOfCols); - return pSupport->data[index] + pSupport->offset * pExpr->base.resBytes; -} - -TAOS_ROW doSetResultRowData(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); - if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker - tfree(pRes->tsrow); - return pRes->tsrow; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - size_t size = tscNumOfFields(pQueryInfo); - - int32_t j = 0; - for (int i = 0; i < size; ++i) { - SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); - if (!pInfo->visible) { - continue; - } - - int32_t type = pInfo->field.type; - int32_t bytes = pInfo->field.bytes; - - if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) { - pRes->tsrow[j] = isNull(pRes->urow[i], type) ? NULL : pRes->urow[i]; - } else { - pRes->tsrow[j] = isNull(pRes->urow[i], type) ? NULL : varDataVal(pRes->urow[i]); - pRes->length[j] = varDataLen(pRes->urow[i]); - } - - ((char**) pRes->urow)[i] += bytes; - j += 1; - } - - pRes->row++; // index increase one-step - return pRes->tsrow; -} - -static UNUSED_FUNC bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) { - bool hasData = true; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - bool allSubqueryExhausted = true; - - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - if (pSql->pSubs[i] == NULL) { - continue; - } - - SSqlRes *pRes1 = &pSql->pSubs[i]->res; - SSqlCmd *pCmd1 = &pSql->pSubs[i]->cmd; - - SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pCmd1); - assert(pQueryInfo1->numOfTables == 1); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo1, 0); - - /* - * if the global limitation is not reached, and current result has not exhausted, or next more vnodes are - * available, goes on - */ - if (pTableMetaInfo->vgroupIndex < pTableMetaInfo->vgroupList->numOfVgroups && pRes1->row < pRes1->numOfRows && - (!tscHasReachLimitation(pQueryInfo1, pRes1))) { - allSubqueryExhausted = false; - break; - } - } - - hasData = !allSubqueryExhausted; - } else { // otherwise, in case inner join, if any subquery exhausted, query completed. - for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - if (pSql->pSubs[i] == 0) { - continue; - } - - SSqlRes * pRes1 = &pSql->pSubs[i]->res; - SQueryInfo *pQueryInfo1 = tscGetQueryInfo(&pSql->pSubs[i]->cmd); - - if ((pRes1->row >= pRes1->numOfRows && tscHasReachLimitation(pQueryInfo1, pRes1) && - tscIsProjectionQuery(pQueryInfo1)) || - (pRes1->numOfRows == 0)) { - hasData = false; - break; - } - } - } - - return hasData; -} - -void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, STableGroupInfo* pTableGroupInfo, SOperatorInfo* pSourceOperator, - char* sql, void* merger, int32_t stage, uint64_t qId) { - assert(pQueryInfo != NULL); - SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); - if (pQInfo == NULL) { - goto _cleanup; - } - - // to make sure third party won't overwrite this structure - pQInfo->signature = pQInfo; - pQInfo->qId = qId; - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = &pQInfo->query; - - pRuntimeEnv->pQueryAttr = pQueryAttr; - tscCreateQueryFromQueryInfo(pQueryInfo, pQueryAttr, NULL); - - pQueryAttr->tableGroupInfo = *pTableGroupInfo; - - // calculate the result row size - SExprInfo* pEx = NULL; - int32_t num = 0; - if (pQueryAttr->pExpr3 != NULL) { - pEx = pQueryAttr->pExpr3; - num = pQueryAttr->numOfExpr3; - } else if (pQueryAttr->pExpr2 != NULL) { - pEx = pQueryAttr->pExpr2; - num = pQueryAttr->numOfExpr2; - } else { - pEx = pQueryAttr->pExpr1; - num = pQueryAttr->numOfOutput; - } - - for (int16_t col = 0; col < num; ++col) { - pQueryAttr->resultRowSize += pEx[col].base.resBytes; - - // keep the tag length - if (TSDB_COL_IS_TAG(pEx[col].base.colInfo.flag)) { - pQueryAttr->tagLen += pEx[col].base.resBytes; - } - } - - size_t numOfGroups = 0; - if (pTableGroupInfo->pGroupList != NULL) { - numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList); - STableGroupInfo* pTableqinfo = &pQInfo->runtimeEnv.tableqinfoGroupInfo; - - pTableqinfo->pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); - pTableqinfo->numOfTables = pTableGroupInfo->numOfTables; - pTableqinfo->map = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - } - - pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); - if (pQInfo->pBuf == NULL) { - goto _cleanup; - } - - pQInfo->dataReady = QUERY_RESULT_NOT_READY; - pQInfo->rspContext = NULL; - pQInfo->sql = sql; - - pthread_mutex_init(&pQInfo->lock, NULL); - tsem_init(&pQInfo->ready, 0, 0); - - int32_t index = 0; - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray* pa = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); - - size_t s = taosArrayGetSize(pa); - SArray* p1 = taosArrayInit(s, POINTER_BYTES); - if (p1 == NULL) { - goto _cleanup; - } - - taosArrayPush(pRuntimeEnv->tableqinfoGroupInfo.pGroupList, &p1); - - STimeWindow window = pQueryAttr->window; - for(int32_t j = 0; j < s; ++j) { - STableKeyInfo* info = taosArrayGet(pa, j); - window.skey = info->lastKey; - - void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); - STableQueryInfo* item = createTableQueryInfo(pQueryAttr, info->pTable, pQueryAttr->groupbyColumn, window, buf); - if (item == NULL) { - goto _cleanup; - } - - item->groupIndex = i; - taosArrayPush(p1, &item); - - STableId id = {.tid = 0, .uid = 0}; - taosHashPut(pRuntimeEnv->tableqinfoGroupInfo.map, &id.tid, sizeof(id.tid), &item, POINTER_BYTES); - index += 1; - } - } - - // todo refactor: filter should not be applied here. - createFilterInfo(pQueryAttr, 0); - - SArray* pa = NULL; - if (stage == MASTER_SCAN) { - pQueryAttr->createFilterOperator = false; // no need for parent query - pa = createExecOperatorPlan(pQueryAttr); - } else { - pa = createGlobalMergePlan(pQueryAttr); - } - - STsBufInfo bufInfo = {0}; - SQueryParam param = {.pOperator = pa}; - /*int32_t code = */initQInfo(&bufInfo, NULL, pSourceOperator, pQInfo, ¶m, NULL, 0, merger); - taosArrayDestroy(pa); - - return pQInfo; - - _cleanup: - freeQInfo(pQInfo); - return NULL; -} diff --git a/2.0/src/client/src/tscSystem.c b/2.0/src/client/src/tscSystem.c deleted file mode 100644 index 897a629d57ef3666045e45a7cbb450dc91250e42..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscSystem.c +++ /dev/null @@ -1,439 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tmsg.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tnote.h" -#include "tref.h" -#include "trpc.h" -#include "tscLog.h" -#include "tsched.h" -#include "tsclient.h" -#include "tscript.h" -#include "ttimer.h" -#include "ttimezone.h" - -// global, not configurable -#define TSC_VAR_NOT_RELEASE 1 -#define TSC_VAR_RELEASED 0 - -int32_t sentinel = TSC_VAR_NOT_RELEASE; - -SHashObj *tscVgroupMap; // hash map to keep the vgroup info from mnode -SHashObj *tscTableMetaMap; // table meta info buffer -SCacheObj *tscVgroupListBuf; // super table vgroup list information, only survives 5 seconds for each super table vgroup list - -int32_t tscObjRef = -1; -void *tscTmr; -void *tscQhandle; -int32_t tscRefId = -1; -int32_t tscNumOfObj = 0; // number of sqlObj in current process. -static void *tscCheckDiskUsageTmr; -void *tscRpcCache; // cache to keep rpc obj -int32_t tscNumOfThreads = 1; // num of rpc threads -char tscLogFileName[12] = "taoslog"; -int tscLogFileNum = 10; - -static pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently -static pthread_once_t tscinit = PTHREAD_ONCE_INIT; - -// pthread_once can not return result code, so result code is set to a global variable. -static volatile int tscInitRes = 0; - -void tscCheckDiskUsage(void *UNUSED_PARAM(para), void *UNUSED_PARAM(param)) { - taosGetDisk(); - taosTmrReset(tscCheckDiskUsage, 20 * 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); -} - -void tscFreeRpcObj(void *param) { - assert(param); - SRpcObj *pRpcObj = (SRpcObj *)(param); - tscDebug("free rpcObj:%p and free pDnodeConn: %p", pRpcObj, pRpcObj->pDnodeConn); - rpcClose(pRpcObj->pDnodeConn); -} - -void tscReleaseRpc(void *param) { - if (param == NULL) { - return; - } - - taosCacheRelease(tscRpcCache, (void *)¶m, false); -} - -int32_t tscAcquireRpc(const char *key, const char *user, const char *secretEncrypt, void **ppRpcObj) { - pthread_mutex_lock(&rpcObjMutex); - - SRpcObj *pRpcObj = (SRpcObj *)taosCacheAcquireByKey(tscRpcCache, key, strlen(key)); - if (pRpcObj != NULL) { - *ppRpcObj = pRpcObj; - pthread_mutex_unlock(&rpcObjMutex); - return 0; - } - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "TSC"; - rpcInit.numOfThreads = tscNumOfThreads; - rpcInit.cfp = tscProcessMsgFromServer; - rpcInit.sessions = tsMaxConnections; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.user = (char *)user; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.ckey = "key"; - rpcInit.spi = 1; - rpcInit.secret = (char *)secretEncrypt; - - SRpcObj rpcObj; - memset(&rpcObj, 0, sizeof(rpcObj)); - strncpy(rpcObj.key, key, strlen(key)); - rpcObj.pDnodeConn = rpcOpen(&rpcInit); - if (rpcObj.pDnodeConn == NULL) { - pthread_mutex_unlock(&rpcObjMutex); - tscError("failed to init connection to TDengine"); - return -1; - } - - pRpcObj = taosCachePut(tscRpcCache, rpcObj.key, strlen(rpcObj.key), &rpcObj, sizeof(rpcObj), 1000*5); - if (pRpcObj == NULL) { - rpcClose(rpcObj.pDnodeConn); - pthread_mutex_unlock(&rpcObjMutex); - return -1; - } - - *ppRpcObj = pRpcObj; - pthread_mutex_unlock(&rpcObjMutex); - return 0; -} - -void taos_init_imp(void) { - char temp[128] = {0}; - - // In the APIs of other program language, taos_cleanup is not available yet. - // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. - atexit(taos_cleanup); - - errno = TSDB_CODE_SUCCESS; - srand(taosGetTimestampSec()); - deltaToUtcInitOnce(); - - if (tscEmbedded == 0) { - - // Read global configuration. - taosInitGlobalCfg(); - taosReadGlobalLogCfg(); - - // For log directory - if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { - printf("failed to create log dir:%s\n", tsLogDir); - } - - sprintf(temp, "%s/%s", tsLogDir, tscLogFileName); - if (taosInitLog(temp, tsNumOfLogLines, tscLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); - } - - taosReadGlobalCfg(); - if (taosCheckGlobalCfg() != 0) { - tscInitRes = -1; - return; - } - - taosInitNotes(); - - rpcInit(); - - scriptEnvPoolInit(); - - tscDebug("starting to initialize TAOS client ..."); - tscDebug("Local End Point is:%s", tsLocalEp); - } - - taosSetCoreDump(); - tscInitMsgsFp(); - - double factor = (tscEmbedded == 0)? 2.0:4.0; - tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor); - if (tscNumOfThreads < 2) { - tscNumOfThreads = 2; - } - - int32_t queueSize = tsMaxConnections*2; - tscQhandle = taosInitScheduler(queueSize, tscNumOfThreads, "tsc"); - if (NULL == tscQhandle) { - tscError("failed to init task queue"); - tscInitRes = -1; - return; - } - - tscDebug("client task queue is initialized, numOfWorkers: %d", tscNumOfThreads); - - tscTmr = taosTmrInit(tsMaxConnections * 2, 200, 60000, "TSC"); - if(0 == tscEmbedded){ - taosTmrReset(tscCheckDiskUsage, 20 * 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); - } - - if (tscTableMetaMap == NULL) { - tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj); - tscVgroupMap = taosHashInit(256, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - tscTableMetaMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - tscVgroupListBuf = taosCacheInit(TSDB_DATA_TYPE_BINARY, 5, false, NULL, "stable-vgroup-list"); - tscDebug("TableMeta:%p, vgroup:%p is initialized", tscTableMetaMap, tscVgroupMap); - } - - int refreshTime = 5; - tscRpcCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, true, tscFreeRpcObj, "rpcObj"); - pthread_mutex_init(&rpcObjMutex, NULL); - - tscRefId = taosOpenRef(200, tscCloseTscObj); - - tscDebug("client is initialized successfully"); -} - -int taos_init() { - pthread_once(&tscinit, taos_init_imp); - return tscInitRes; -} - -// this function may be called by user or system, or by both simultaneously. -void taos_cleanup(void) { - tscDebug("start to cleanup client environment"); - - if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { - return; - } - - if (tscEmbedded == 0) { - scriptEnvPoolCleanup(); - } - - taosHashCleanup(tscTableMetaMap); - tscTableMetaMap = NULL; - - taosHashCleanup(tscVgroupMap); - tscVgroupMap = NULL; - - int32_t id = tscObjRef; - tscObjRef = -1; - taosCloseRef(id); - - void* p = tscQhandle; - tscQhandle = NULL; - taosCleanUpScheduler(p); - - id = tscRefId; - tscRefId = -1; - taosCloseRef(id); - - taosCleanupKeywordsTable(); - - p = tscRpcCache; - tscRpcCache = NULL; - - if (p != NULL) { - taosCacheCleanup(p); - pthread_mutex_destroy(&rpcObjMutex); - } - - taosCacheCleanup(tscVgroupListBuf); - tscVgroupListBuf = NULL; - - if (tscEmbedded == 0) { - rpcCleanup(); - taosCloseLog(); - }; - - p = tscTmr; - tscTmr = NULL; - taosTmrCleanUp(p); -} - -static int taos_options_imp(TSDB_OPTION option, const char *pStr) { - SGlobalCfg *cfg = NULL; - - switch (option) { - case TSDB_OPTION_CONFIGDIR: - cfg = taosGetConfigOption("configDir"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(configDir, pStr, TSDB_FILENAME_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscInfo("set config file directory:%s", pStr); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } - break; - - case TSDB_OPTION_SHELL_ACTIVITY_TIMER: - cfg = taosGetConfigOption("shellActivityTimer"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tsShellActivityTimer = atoi(pStr); - if (tsShellActivityTimer < 1) tsShellActivityTimer = 1; - if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600; - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, pStr, - tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); - } - break; - - case TSDB_OPTION_LOCALE: { // set locale - cfg = taosGetConfigOption("locale"); - assert(cfg != NULL); - - size_t len = strlen(pStr); - if (len == 0 || len > TSDB_LOCALE_LEN) { - tscInfo("Invalid locale:%s, use default", pStr); - return -1; - } - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - char sep = '.'; - - if (strlen(tsLocale) == 0) { // locale does not set yet - char* defaultLocale = setlocale(LC_CTYPE, ""); - - // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. - // The launch of current system will abort soon. - if (defaultLocale == NULL) { - tscError("failed to get default locale, please set the correct locale in current OS"); - return -1; - } - - tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN); - } - - // set the user specified locale - char *locale = setlocale(LC_CTYPE, pStr); - - if (locale != NULL) { // failed to set the user specified locale - tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { // set the user specified locale failed, use default LC_CTYPE as current locale - locale = setlocale(LC_CTYPE, tsLocale); - tscInfo("failed to set locale:%s, current locale:%s", pStr, tsLocale); - } - - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - - char *charset = strrchr(tsLocale, sep); - if (charset != NULL) { - charset += 1; - - charset = taosCharsetReplace(charset); - - if (taosValidateEncodec(charset)) { - if (strlen(tsCharset) == 0) { - tscInfo("charset set:%s", charset); - } else { - tscInfo("charset changed from %s to %s", tsCharset, charset); - } - - tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - - } else { - tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset); - } - - free(charset); - } else { // it may be windows system - tscInfo("charset remains:%s", tsCharset); - } - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } - break; - } - - case TSDB_OPTION_CHARSET: { - /* set charset will override the value of charset, assigned during system locale changed */ - cfg = taosGetConfigOption("charset"); - assert(cfg != NULL); - - size_t len = strlen(pStr); - if (len == 0 || len > TSDB_LOCALE_LEN) { - tscInfo("failed to set charset:%s", pStr); - return -1; - } - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - if (taosValidateEncodec(pStr)) { - if (strlen(tsCharset) == 0) { - tscInfo("charset is set:%s", pStr); - } else { - tscInfo("charset changed from %s to %s", tsCharset, pStr); - } - - tstrncpy(tsCharset, pStr, TSDB_LOCALE_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { - tscInfo("charset:%s not valid", pStr); - } - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } - - break; - } - - case TSDB_OPTION_TIMEZONE: - cfg = taosGetConfigOption("timezone"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(tsTimezone, pStr, TSDB_TIMEZONE_LEN); - tsSetTimeZone(); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, pStr); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } - break; - - default: - // TODO return the correct error code to client in the format for taos_errstr() - tscError("Invalid option %d", option); - return -1; - } - - return 0; -} - -int taos_options(TSDB_OPTION option, const void *arg, ...) { - static int32_t lock = 0; - - for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) { - if (i % 1000 == 0) { - tscInfo("haven't acquire lock after spin %d times.", i); - sched_yield(); - } - } - - int ret = taos_options_imp(option, (const char*)arg); - - atomic_store_32(&lock, 0); - return ret; -} diff --git a/2.0/src/client/src/tscUtil.c b/2.0/src/client/src/tscUtil.c deleted file mode 100644 index 449f1ea4530a39f330181cb29e26661a7e02bcf9..0000000000000000000000000000000000000000 --- a/2.0/src/client/src/tscUtil.c +++ /dev/null @@ -1,5132 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tscUtil.h" -#include "hash.h" -#include "os.h" -#include "tmsg.h" -#include "texpr.h" -#include "tkey.h" -#include "tmd5.h" -#include "tscGlobalmerge.h" -#include "tscLog.h" -#include "tscProfile.h" -#include "tscSubquery.h" -#include "tsched.h" -#include "qTableMeta.h" -#include "tsclient.h" -#include "ttimer.h" -#include "ttokendef.h" -#include "httpInt.h" - -static void freeQueryInfoImpl(SQueryInfo* pQueryInfo); - -int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) { - int32_t n = 0; - - switch (type) { - case TSDB_DATA_TYPE_NULL: - n = sprintf(str, "null"); - break; - - case TSDB_DATA_TYPE_BOOL: - n = sprintf(str, (*(int8_t*)buf) ? "true" : "false"); - break; - - case TSDB_DATA_TYPE_TINYINT: - n = sprintf(str, "%d", *(int8_t*)buf); - break; - - case TSDB_DATA_TYPE_SMALLINT: - n = sprintf(str, "%d", *(int16_t*)buf); - break; - - case TSDB_DATA_TYPE_INT: - n = sprintf(str, "%d", *(int32_t*)buf); - break; - - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - n = sprintf(str, "%" PRId64, *(int64_t*)buf); - break; - - case TSDB_DATA_TYPE_FLOAT: - n = sprintf(str, "%e", GET_FLOAT_VAL(buf)); - break; - - case TSDB_DATA_TYPE_DOUBLE: - n = sprintf(str, "%e", GET_DOUBLE_VAL(buf)); - break; - - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - if (bufSize < 0) { - tscError("invalid buf size"); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - *str = '"'; - memcpy(str + 1, buf, bufSize); - *(str + bufSize + 1) = '"'; - n = bufSize + 2; - break; - - case TSDB_DATA_TYPE_UTINYINT: - n = sprintf(str, "%d", *(uint8_t*)buf); - break; - - case TSDB_DATA_TYPE_USMALLINT: - n = sprintf(str, "%d", *(uint16_t*)buf); - break; - - case TSDB_DATA_TYPE_UINT: - n = sprintf(str, "%u", *(uint32_t*)buf); - break; - - case TSDB_DATA_TYPE_UBIGINT: - n = sprintf(str, "%" PRIu64, *(uint64_t*)buf); - break; - - default: - tscError("unsupported type:%d", type); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - *len = n; - - return TSDB_CODE_SUCCESS; -} - - -static void tscStrToLower(char *str, int32_t n) { - if (str == NULL || n <= 0) { return;} - for (int32_t i = 0; i < n; i++) { - if (str[i] >= 'A' && str[i] <= 'Z') { - str[i] -= ('A' - 'a'); - } - } -} -SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { - if (pTagCond->pCond == NULL) { - return NULL; - } - - size_t size = taosArrayGetSize(pTagCond->pCond); - for (int32_t i = 0; i < size; ++i) { - SCond* pCond = taosArrayGet(pTagCond->pCond, i); - - if (uid == pCond->uid) { - return pCond; - } - } - - return NULL; -} - -STblCond* tsGetTableFilter(SArray* filters, uint64_t uid, int16_t idx) { - if (filters == NULL) { - return NULL; - } - - size_t size = taosArrayGetSize(filters); - for (int32_t i = 0; i < size; ++i) { - STblCond* cond = taosArrayGet(filters, i); - - if (uid == cond->uid && (idx >= 0 && cond->idx == idx)) { - return cond; - } - } - - return NULL; -} - - -void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) { - if (tbufTell(bw) == 0) { - return; - } - - SCond cond = { - .uid = uid, - .len = (int32_t)(tbufTell(bw)), - .cond = NULL, - }; - - cond.cond = tbufGetData(bw, true); - - if (pTagCond->pCond == NULL) { - pTagCond->pCond = taosArrayInit(3, sizeof(SCond)); - } - - taosArrayPush(pTagCond->pCond, &cond); -} - -bool tscQueryTags(SQueryInfo* pQueryInfo) { - int32_t numOfCols = (int32_t) tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - int32_t functId = pExpr->base.functionId; - - // "select count(tbname)" query - if (functId == TSDB_FUNC_COUNT && pExpr->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - continue; - } - - if (functId != TSDB_FUNC_TAGPRJ && functId != TSDB_FUNC_TID_TAG) { - return false; - } - } - - return true; -} - -bool tscQueryBlockInfo(SQueryInfo* pQueryInfo) { - int32_t numOfCols = (int32_t) tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - int32_t functId = pExpr->base.functionId; - - if (functId == TSDB_FUNC_BLKINFO) { - return true; - } - } - - return false; -} - -bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { - if (pQueryInfo == NULL) { - return false; - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - if (pTableMetaInfo == NULL) { - return false; - } - - if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) { - return false; - } - - // for ordered projection query, iterate all qualified vnodes sequentially - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, tableIndex)) { - return false; - } - - if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->command == TSDB_SQL_SELECT) { - return UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - } - - return false; -} - -bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - - /* - * In following cases, return false for non ordered project query on super table - * 1. failed to get tableMeta from server; 2. not a super table; 3. limitation is 0; - * 4. show queries, instead of a select query - */ - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) || - pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || numOfExprs == 0) { - return false; - } - - for (int32_t i = 0; i < numOfExprs; ++i) { - int32_t functionId = tscExprGet(pQueryInfo, i)->base.functionId; - - if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - return false; - } - - continue; - } - - if (functionId != TSDB_FUNC_PRJ && - functionId != TSDB_FUNC_TAGPRJ && - functionId != TSDB_FUNC_TAG && - functionId != TSDB_FUNC_TS && - functionId != TSDB_FUNC_ARITHM && - functionId != TSDB_FUNC_TS_COMP && - functionId != TSDB_FUNC_DIFF && - functionId != TSDB_FUNC_DERIVATIVE && - functionId != TSDB_FUNC_TS_DUMMY && - functionId != TSDB_FUNC_TID_TAG) { - return false; - } - } - - return true; -} - -// not order by timestamp projection query on super table -bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { - if (!tscIsProjectionQueryOnSTable(pQueryInfo, tableIndex)) { - return false; - } - - // order by columnIndex exists, not a non-ordered projection query - return pQueryInfo->order.orderColId < 0; -} - -bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { - if (!tscIsProjectionQueryOnSTable(pQueryInfo, tableIndex)) { - return false; - } - - // order by columnIndex exists, a non-ordered projection query - return pQueryInfo->order.orderColId >= 0; -} - -bool tscIsProjectionQuery(SQueryInfo* pQueryInfo) { - size_t size = tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < size; ++i) { - int32_t f = tscExprGet(pQueryInfo, i)->base.functionId; - if (f == TSDB_FUNC_TS_DUMMY) { - continue; - } - - if (f != TSDB_FUNC_PRJ && f != TSDB_FUNC_TAGPRJ && f != TSDB_FUNC_TAG && - f != TSDB_FUNC_TS && f != TSDB_FUNC_ARITHM && f != TSDB_FUNC_DIFF && - f != TSDB_FUNC_DERIVATIVE) { - return false; - } - } - - return true; -} - -bool tscIsDiffDerivQuery(SQueryInfo* pQueryInfo) { - size_t size = tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < size; ++i) { - int32_t f = tscExprGet(pQueryInfo, i)->base.functionId; - if (f == TSDB_FUNC_TS_DUMMY) { - continue; - } - - if (f == TSDB_FUNC_DIFF || f == TSDB_FUNC_DERIVATIVE) { - return true; - } - } - - return false; -} - - -bool tscHasColumnFilter(SQueryInfo* pQueryInfo) { - // filter on primary timestamp column - if (pQueryInfo->window.skey != INT64_MIN || pQueryInfo->window.ekey != INT64_MAX) { - return true; - } - - size_t size = taosArrayGetSize(pQueryInfo->colList); - for (int32_t i = 0; i < size; ++i) { - SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i); - if (pCol->info.flist.numOfFilters > 0) { - return true; - } - } - - return false; -} - -bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) { - size_t size = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - assert(pExpr != NULL); - - int32_t functionId = pExpr->base.functionId; - if (functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS) { - continue; - } - - if (functionId != TSDB_FUNC_INTERP) { - return false; - } - } - - return true; -} - -bool tsIsArithmeticQueryOnAggResult(SQueryInfo* pQueryInfo) { - if (tscIsProjectionQuery(pQueryInfo)) { - return false; - } - - size_t numOfOutput = tscNumOfFields(pQueryInfo); - for(int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExprInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i)->pExpr; - if (pExprInfo->pExpr != NULL) { - return true; - } - } - - if (tscIsProjectionQuery(pQueryInfo)) { - return false; - } - - return false; -} - -bool tscGroupbyColumn(SQueryInfo* pQueryInfo) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - SGroupbyExpr* pGroupbyExpr = &pQueryInfo->groupbyExpr; - for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { - SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); - if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < numOfCols) { // group by normal columns - return true; - } - } - - return false; -} - -int32_t tscGetTopBotQueryExprIndex(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - if (pExpr->base.functionId == TSDB_FUNC_TS) { - continue; - } - - if (pExpr->base.functionId == TSDB_FUNC_TOP || pExpr->base.functionId == TSDB_FUNC_BOTTOM) { - return i; - } - } - - return -1; -} - -bool tscIsTopBotQuery(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - if (pExpr->base.functionId == TSDB_FUNC_TS) { - continue; - } - - if (pExpr->base.functionId == TSDB_FUNC_TOP || pExpr->base.functionId == TSDB_FUNC_BOTTOM) { - return true; - } - } - - return false; -} - -bool isTsCompQuery(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - SExprInfo* pExpr1 = tscExprGet(pQueryInfo, 0); - if (numOfExprs != 1) { - return false; - } - - return pExpr1->base.functionId == TSDB_FUNC_TS_COMP; -} - -bool hasTagValOutput(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - SExprInfo* pExpr1 = tscExprGet(pQueryInfo, 0); - - if (numOfExprs == 1 && pExpr1->base.functionId == TSDB_FUNC_TS_COMP) { - return true; - } - - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - // ts_comp column required the tag value for join filter - if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag)) { - return true; - } - } - - return false; -} - -bool timeWindowInterpoRequired(SQueryInfo *pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - int32_t functionId = pExpr->base.functionId; - if (functionId == TSDB_FUNC_TWA || functionId == TSDB_FUNC_INTERP) { - return true; - } - } - - return false; -} - -bool isStabledev(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - int32_t functionId = pExpr->base.functionId; - if (functionId == TSDB_FUNC_STDDEV_DST) { - return true; - } - } - - return false; -} - -bool tscIsTWAQuery(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - if (pExpr->base.functionId == TSDB_FUNC_TWA) { - return true; - } - } - - return false; -} - -bool tscIsIrateQuery(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - if (pExpr->base.functionId == TSDB_FUNC_IRATE) { - return true; - } - } - - return false; -} - -bool tscIsSessionWindowQuery(SQueryInfo* pQueryInfo) { - return pQueryInfo->sessionWindow.gap > 0; -} - -bool tscNeedReverseScan(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - int32_t functionId = pExpr->base.functionId; - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) { - continue; - } - - if ((functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_FIRST_DST) && pQueryInfo->order.order == TSDB_ORDER_DESC) { - return true; - } - - if (functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_LAST_DST) { - // the scan order to acquire the last result of the specified column - int32_t order = (int32_t)pExpr->base.param[0].i64; - if (order != pQueryInfo->order.order) { - return true; - } - } - } - - return false; -} - -bool isSimpleAggregateRv(SQueryInfo* pQueryInfo) { - if (pQueryInfo->interval.interval > 0 || pQueryInfo->sessionWindow.gap > 0) { - return false; - } - - if (tscIsDiffDerivQuery(pQueryInfo)) { - return false; - } - - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) { - continue; - } - - int32_t functionId = pExpr->base.functionId; - if (functionId < 0) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - return true; - } - - continue; - } - - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY) { - continue; - } - - if ((!IS_MULTIOUTPUT(aAggs[functionId].status)) || - (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TS_COMP)) { - return true; - } - } - - return false; -} - -bool isBlockDistQuery(SQueryInfo* pQueryInfo) { - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); - return (numOfExprs == 1 && pExpr->base.functionId == TSDB_FUNC_BLKINFO); -} - -void tscClearInterpInfo(SQueryInfo* pQueryInfo) { - if (!tscIsPointInterpQuery(pQueryInfo)) { - return; - } - - pQueryInfo->fillType = TSDB_FILL_NONE; - tfree(pQueryInfo->fillVal); -} - -int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { - pRes->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - - if (pRes->tsrow == NULL) { - pRes->tsrow = calloc(pRes->numOfCols, POINTER_BYTES); - pRes->urow = calloc(pRes->numOfCols, POINTER_BYTES); - pRes->length = calloc(pRes->numOfCols, sizeof(int32_t)); - pRes->buffer = calloc(pRes->numOfCols, POINTER_BYTES); - - // not enough memory - if (pRes->tsrow == NULL || pRes->urow == NULL || pRes->length == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { - tfree(pRes->tsrow); - tfree(pRes->urow); - tfree(pRes->length); - tfree(pRes->buffer); - - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; - } - } - - return TSDB_CODE_SUCCESS; -} - -static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bool convertNchar) { - // generated the user-defined column result - if (pInfo->pExpr->pExpr == NULL && TSDB_COL_IS_UD_COL(pInfo->pExpr->base.colInfo.flag)) { - if (pInfo->pExpr->base.param[1].nType == TSDB_DATA_TYPE_NULL) { - setNullN(pRes->urow[i], pInfo->field.type, pInfo->field.bytes, (int32_t) pRes->numOfRows); - } else { - if (pInfo->field.type == TSDB_DATA_TYPE_NCHAR || pInfo->field.type == TSDB_DATA_TYPE_BINARY) { - assert(pInfo->pExpr->base.param[1].nLen <= pInfo->field.bytes); - - for (int32_t k = 0; k < pRes->numOfRows; ++k) { - char* p = ((char**)pRes->urow)[i] + k * pInfo->field.bytes; - - memcpy(varDataVal(p), pInfo->pExpr->base.param[1].pz, pInfo->pExpr->base.param[1].nLen); - varDataSetLen(p, pInfo->pExpr->base.param[1].nLen); - } - } else { - for (int32_t k = 0; k < pRes->numOfRows; ++k) { - char* p = ((char**)pRes->urow)[i] + k * pInfo->field.bytes; - memcpy(p, &pInfo->pExpr->base.param[1].i64, pInfo->field.bytes); - } - } - } - - } else if (convertNchar && pInfo->field.type == TSDB_DATA_TYPE_NCHAR) { - // convert unicode to native code in a temporary buffer extra one byte for terminated symbol - char* buffer = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); - if(buffer == NULL) - return ; - pRes->buffer[i] = buffer; - // string terminated char for binary data - memset(pRes->buffer[i], 0, pInfo->field.bytes * pRes->numOfRows); - - char* p = pRes->urow[i]; - for (int32_t k = 0; k < pRes->numOfRows; ++k) { - char* dst = pRes->buffer[i] + k * pInfo->field.bytes; - - if (isNull(p, TSDB_DATA_TYPE_NCHAR)) { - memcpy(dst, p, varDataTLen(p)); - } else if (varDataLen(p) > 0) { - int32_t length = taosUcs4ToMbs(varDataVal(p), varDataLen(p), varDataVal(dst)); - varDataSetLen(dst, length); - - if (length == 0) { - tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)p); - } - } else { - varDataSetLen(dst, 0); - } - - p += pInfo->field.bytes; - } - - memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); - } -} - -void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo) { - assert(pRes->numOfCols > 0); - if (pRes->numOfRows == 0) { - return; - } - int32_t offset = 0; - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); - - pRes->urow[i] = pRes->data + offset * pRes->numOfRows; - pRes->length[i] = pInfo->field.bytes; - - offset += pInfo->field.bytes; - setResRawPtrImpl(pRes, pInfo, i, true); - } -} - -void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock, bool convertNchar) { - assert(pRes->numOfCols > 0); - - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); - - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); - - pRes->urow[i] = pColData->pData; - pRes->length[i] = pInfo->field.bytes; - - setResRawPtrImpl(pRes, pInfo, i, convertNchar); - /* - // generated the user-defined column result - if (pInfo->pExpr->pExpr == NULL && TSDB_COL_IS_UD_COL(pInfo->pExpr->base.ColName.flag)) { - if (pInfo->pExpr->base.param[1].nType == TSDB_DATA_TYPE_NULL) { - setNullN(pRes->urow[i], pInfo->field.type, pInfo->field.bytes, (int32_t) pRes->numOfRows); - } else { - if (pInfo->field.type == TSDB_DATA_TYPE_NCHAR || pInfo->field.type == TSDB_DATA_TYPE_BINARY) { - assert(pInfo->pExpr->base.param[1].nLen <= pInfo->field.bytes); - - for (int32_t k = 0; k < pRes->numOfRows; ++k) { - char* p = ((char**)pRes->urow)[i] + k * pInfo->field.bytes; - - memcpy(varDataVal(p), pInfo->pExpr->base.param[1].pz, pInfo->pExpr->base.param[1].nLen); - varDataSetLen(p, pInfo->pExpr->base.param[1].nLen); - } - } else { - for (int32_t k = 0; k < pRes->numOfRows; ++k) { - char* p = ((char**)pRes->urow)[i] + k * pInfo->field.bytes; - memcpy(p, &pInfo->pExpr->base.param[1].i64, pInfo->field.bytes); - } - } - } - - } else if (convertNchar && pInfo->field.type == TSDB_DATA_TYPE_NCHAR) { - // convert unicode to native code in a temporary buffer extra one byte for terminated symbol - pRes->buffer[i] = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); - - // string terminated char for binary data - memset(pRes->buffer[i], 0, pInfo->field.bytes * pRes->numOfRows); - - char* p = pRes->urow[i]; - for (int32_t k = 0; k < pRes->numOfRows; ++k) { - char* dst = pRes->buffer[i] + k * pInfo->field.bytes; - - if (isNull(p, TSDB_DATA_TYPE_NCHAR)) { - memcpy(dst, p, varDataTLen(p)); - } else if (varDataLen(p) > 0) { - int32_t length = taosUcs4ToMbs(varDataVal(p), varDataLen(p), varDataVal(dst)); - varDataSetLen(dst, length); - - if (length == 0) { - tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)p); - } - } else { - varDataSetLen(dst, 0); - } - - p += pInfo->field.bytes; - } - - memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); - }*/ - } -} - -static SColumnInfo* extractColumnInfoFromResult(SArray* pTableCols) { - int32_t numOfCols = (int32_t) taosArrayGetSize(pTableCols); - SColumnInfo* pColInfo = calloc(numOfCols, sizeof(SColumnInfo)); - for(int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(pTableCols, i); - pColInfo[i] = pCol->info;//[index].type; - } - - return pColInfo; -} - -typedef struct SDummyInputInfo { - SSDataBlock *block; - STableQueryInfo *pTableQueryInfo; - SSqlObj *pSql; // refactor: remove it - SFilterInfo *pFilterInfo; -} SDummyInputInfo; - -typedef struct SJoinStatus { - SSDataBlock* pBlock; // point to the upstream block - int32_t index; - bool completed;// current upstream is completed or not -} SJoinStatus; - -typedef struct SJoinOperatorInfo { - SSDataBlock *pRes; - SJoinStatus *status; - int32_t numOfUpstream; - SRspResultInfo resultInfo; // todo refactor, add this info for each operator -} SJoinOperatorInfo; - -static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, SFilterInfo* pFilterInfo) { - int32_t offset = 0; - char* pData = pRes->data; - - for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); - if (pData != NULL) { - pColData->pData = pData + offset * pBlock->info.rows; - } else { - pColData->pData = pRes->urow[i]; - } - - offset += pColData->info.bytes; - } - - // filter data if needed - if (pFilterInfo) { - //doSetFilterColumnInfo(pFilterInfo, numOfFilterCols, pBlock); - filterSetColFieldData(pFilterInfo, pBlock->info.numOfCols, pBlock->pDataBlock); - bool gotNchar = false; - filterConverNcharColumns(pFilterInfo, pBlock->info.rows, &gotNchar); - int8_t* p = NULL; - //bool all = doFilterDataBlock(pFilterInfo, numOfFilterCols, pBlock->info.rows, p); - bool all = filterExecute(pFilterInfo, pBlock->info.rows, &p, NULL, 0); - if (gotNchar) { - filterFreeNcharColumns(pFilterInfo); - } - if (!all) { - if (p) { - doCompactSDataBlock(pBlock, pBlock->info.rows, p); - } else { - pBlock->info.rows = 0; - pBlock->pBlockStatis = NULL; // clean the block statistics info - } - } - - tfree(p); - } - - // todo refactor: extract method - // set the timestamp range of current result data block - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, 0); - if (pColData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - pBlock->info.window.skey = ((int64_t*)pColData->pData)[0]; - pBlock->info.window.ekey = ((int64_t*)pColData->pData)[pBlock->info.rows-1]; - } - - pRes->numOfRows = 0; -} - -// NOTE: there is already exists data blocks before this function calls. -SSDataBlock* doGetDataBlock(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SDummyInputInfo *pInput = pOperator->info; - SSqlObj* pSql = pInput->pSql; - SSqlRes* pRes = &pSql->res; - - SSDataBlock* pBlock = pInput->block; - if (pOperator->pRuntimeEnv != NULL) { - pOperator->pRuntimeEnv->current = pInput->pTableQueryInfo; - } - - pBlock->info.rows = pRes->numOfRows; - if (pRes->numOfRows != 0) { - doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo); - *newgroup = false; - return pBlock; - } - - // No data block exists. So retrieve and transfer it into to SSDataBlock - TAOS_ROW pRow = NULL; - taos_fetch_block(pSql, &pRow); - - if (pRes->numOfRows == 0) { - pOperator->status = OP_EXEC_DONE; - return NULL; - } - - pBlock->info.rows = pRes->numOfRows; - doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo); - *newgroup = false; - return pBlock; -} - -static void fetchNextBlockIfCompleted(SOperatorInfo* pOperator, bool* newgroup) { - SJoinOperatorInfo* pJoinInfo = pOperator->info; - - for (int32_t i = 0; i < pOperator->numOfUpstream; ++i) { - SJoinStatus* pStatus = &pJoinInfo->status[i]; - if (pStatus->pBlock == NULL || pStatus->index >= pStatus->pBlock->info.rows) { - tscDebug("Retrieve nest query result, index:%d, total:%d", i, pOperator->numOfUpstream); - - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pStatus->pBlock = pOperator->upstream[i]->exec(pOperator->upstream[i], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - pStatus->index = 0; - - if (pStatus->pBlock == NULL) { - pOperator->status = OP_EXEC_DONE; - pJoinInfo->resultInfo.total += pJoinInfo->pRes->info.rows; - break; - } - } - } -} - -SSDataBlock* doDataBlockJoin(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - assert(pOperator->numOfUpstream > 1); - - SJoinOperatorInfo* pJoinInfo = pOperator->info; - pJoinInfo->pRes->info.rows = 0; - - while(1) { - fetchNextBlockIfCompleted(pOperator, newgroup); - if (pOperator->status == OP_EXEC_DONE) { - return pJoinInfo->pRes; - } - - SJoinStatus* st0 = &pJoinInfo->status[0]; - SColumnInfoData* p0 = taosArrayGet(st0->pBlock->pDataBlock, 0); - int64_t* ts0 = (int64_t*) p0->pData; - - if (st0->index >= st0->pBlock->info.rows) { - continue; - } - - bool prefixEqual = true; - - while(1) { - prefixEqual = true; - for (int32_t i = 1; i < pJoinInfo->numOfUpstream; ++i) { - SJoinStatus* st = &pJoinInfo->status[i]; - ts0 = (int64_t*) p0->pData; - - SColumnInfoData* p = taosArrayGet(st->pBlock->pDataBlock, 0); - int64_t* ts = (int64_t*)p->pData; - - if (st->index >= st->pBlock->info.rows || st0->index >= st0->pBlock->info.rows) { - fetchNextBlockIfCompleted(pOperator, newgroup); - if (pOperator->status == OP_EXEC_DONE) { - return pJoinInfo->pRes; - } - - prefixEqual = false; - break; - } - - if (ts[st->index] < ts0[st0->index]) { // less than the first - prefixEqual = false; - - if ((++(st->index)) >= st->pBlock->info.rows) { - fetchNextBlockIfCompleted(pOperator, newgroup); - if (pOperator->status == OP_EXEC_DONE) { - return pJoinInfo->pRes; - } - } - } else if (ts[st->index] > ts0[st0->index]) { // greater than the first; - if (prefixEqual == true) { - prefixEqual = false; - for (int32_t j = 0; j < i; ++j) { - SJoinStatus* stx = &pJoinInfo->status[j]; - if ((++(stx->index)) >= stx->pBlock->info.rows) { - - fetchNextBlockIfCompleted(pOperator, newgroup); - if (pOperator->status == OP_EXEC_DONE) { - return pJoinInfo->pRes; - } - } - } - } else { - if ((++(st0->index)) >= st0->pBlock->info.rows) { - fetchNextBlockIfCompleted(pOperator, newgroup); - if (pOperator->status == OP_EXEC_DONE) { - return pJoinInfo->pRes; - } - } - } - } - } - - if (prefixEqual) { - int32_t offset = 0; - bool completed = false; - for (int32_t i = 0; i < pOperator->numOfUpstream; ++i) { - SJoinStatus* st1 = &pJoinInfo->status[i]; - int32_t rows = pJoinInfo->pRes->info.rows; - - for (int32_t j = 0; j < st1->pBlock->info.numOfCols; ++j) { - SColumnInfoData* pCol1 = taosArrayGet(pJoinInfo->pRes->pDataBlock, j + offset); - SColumnInfoData* pSrc = taosArrayGet(st1->pBlock->pDataBlock, j); - - int32_t bytes = pSrc->info.bytes; - memcpy(pCol1->pData + rows * bytes, pSrc->pData + st1->index * bytes, bytes); - } - - offset += st1->pBlock->info.numOfCols; - if ((++(st1->index)) == st1->pBlock->info.rows) { - completed = true; - } - } - - if ((++pJoinInfo->pRes->info.rows) >= pJoinInfo->resultInfo.capacity) { - pJoinInfo->resultInfo.total += pJoinInfo->pRes->info.rows; - return pJoinInfo->pRes; - } - - if (completed == true) { - break; - } - } - } -/* - while (st0->index < st0->pBlock->info.rows && st1->index < st1->pBlock->info.rows) { - SColumnInfoData* p0 = taosArrayGet(st0->pBlock->pDataBlock, 0); - SColumnInfoData* p1 = taosArrayGet(st1->pBlock->pDataBlock, 0); - - int64_t* ts0 = (int64_t*)p0->pData; - int64_t* ts1 = (int64_t*)p1->pData; - if (ts0[st0->index] == ts1[st1->index]) { // add to the final result buffer - // check if current output buffer is over the threshold to pause current loop - int32_t rows = pJoinInfo->pRes->info.rows; - for (int32_t j = 0; j < st0->pBlock->info.numOfCols; ++j) { - SColumnInfoData* pCol1 = taosArrayGet(pJoinInfo->pRes->pDataBlock, j); - SColumnInfoData* pSrc = taosArrayGet(st0->pBlock->pDataBlock, j); - - int32_t bytes = pSrc->info.bytes; - memcpy(pCol1->pData + rows * bytes, pSrc->pData + st0->index * bytes, bytes); - } - - for (int32_t j = 0; j < st1->pBlock->info.numOfCols; ++j) { - SColumnInfoData* pCol1 = taosArrayGet(pJoinInfo->pRes->pDataBlock, j + st0->pBlock->info.numOfCols); - SColumnInfoData* pSrc = taosArrayGet(st1->pBlock->pDataBlock, j); - - int32_t bytes = pSrc->info.bytes; - memcpy(pCol1->pData + rows * bytes, pSrc->pData + st1->index * bytes, bytes); - } - - st0->index++; - st1->index++; - - if ((++pJoinInfo->pRes->info.rows) >= pJoinInfo->resultInfo.capacity) { - pJoinInfo->resultInfo.total += pJoinInfo->pRes->info.rows; - return pJoinInfo->pRes; - } - } else if (ts0[st0->index] < ts1[st1->index]) { - st0->index++; - } else { - st1->index++; - } - }*/ - } -} - -static void destroyDummyInputOperator(void* param, int32_t numOfOutput) { - SDummyInputInfo* pInfo = (SDummyInputInfo*) param; - - // tricky - for(int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pInfo->block->pDataBlock, i); - pColInfoData->pData = NULL; - } - - pInfo->block = destroyOutputBuf(pInfo->block); - pInfo->pSql = NULL; - - filterFreeInfo(pInfo->pFilterInfo); - - cleanupResultRowInfo(&pInfo->pTableQueryInfo->resInfo); - tfree(pInfo->pTableQueryInfo); -} - -// todo this operator servers as the adapter for Operator tree and SqlRes result, remove it later -SOperatorInfo* createDummyInputOperator(SSqlObj* pSql, SSchema* pSchema, int32_t numOfCols, SFilterInfo* pFilters) { - assert(numOfCols > 0); - STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; - - SDummyInputInfo* pInfo = calloc(1, sizeof(SDummyInputInfo)); - - pInfo->pSql = pSql; - pInfo->pFilterInfo = pFilters; - pInfo->pTableQueryInfo = createTmpTableQueryInfo(win); - - pInfo->block = calloc(numOfCols, sizeof(SSDataBlock)); - pInfo->block->info.numOfCols = numOfCols; - - pInfo->block->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData colData = {{0}}; - colData.info.bytes = pSchema[i].bytes; - colData.info.type = pSchema[i].type; - colData.info.colId = pSchema[i].colId; - - taosArrayPush(pInfo->block->pDataBlock, &colData); - } - - SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo)); - pOptr->name = "DummyInputOperator"; - pOptr->operatorType = OP_DummyInput; - pOptr->numOfOutput = numOfCols; - pOptr->blockingOptr = false; - pOptr->info = pInfo; - pOptr->exec = doGetDataBlock; - pOptr->cleanup = destroyDummyInputOperator; - return pOptr; -} - -static void destroyJoinOperator(void* param, int32_t numOfOutput) { - SJoinOperatorInfo* pInfo = (SJoinOperatorInfo*) param; - tfree(pInfo->status); - - pInfo->pRes = destroyOutputBuf(pInfo->pRes); -} - -SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput) { - SJoinOperatorInfo* pInfo = calloc(1, sizeof(SJoinOperatorInfo)); - - pInfo->numOfUpstream = numOfUpstream; - pInfo->status = calloc(numOfUpstream, sizeof(SJoinStatus)); - - SRspResultInfo* pResInfo = &pInfo->resultInfo; - pResInfo->capacity = 4096; - pResInfo->threshold = (int32_t) (4096 * 0.8); - - pInfo->pRes = calloc(1, sizeof(SSDataBlock)); - pInfo->pRes->info.numOfCols = numOfOutput; - pInfo->pRes->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); - for(int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData colData = {{0}}; - colData.info.bytes = pSchema[i].bytes; - colData.info.type = pSchema[i].type; - colData.info.colId = pSchema[i].colId; - colData.pData = calloc(1, colData.info.bytes * pResInfo->capacity); - - taosArrayPush(pInfo->pRes->pDataBlock, &colData); - } - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "JoinOperator"; - pOperator->operatorType = OP_Join; - pOperator->numOfOutput = numOfOutput; - pOperator->blockingOptr = false; - pOperator->info = pInfo; - pOperator->exec = doDataBlockJoin; - pOperator->cleanup = destroyJoinOperator; - - for(int32_t i = 0; i < numOfUpstream; ++i) { - appendUpstream(pOperator, pUpstream[i]); - } - - return pOperator; -} - -void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, bool convertNchar) { - // set the correct result - SSDataBlock* p = pQueryInfo->pQInfo->runtimeEnv.outputBuf; - pRes->numOfRows = (p != NULL)? p->info.rows: 0; - - if (pRes->code == TSDB_CODE_SUCCESS && pRes->numOfRows > 0) { - tscCreateResPointerInfo(pRes, pQueryInfo); - tscSetResRawPtrRv(pRes, pQueryInfo, p, convertNchar); - } - - tscDebug("0x%"PRIx64" retrieve result in pRes, numOfRows:%d", objId, pRes->numOfRows); - pRes->row = 0; - pRes->completed = (pRes->numOfRows == 0); -} - -/* -static void createInputDataFilterInfo(SQueryInfo* px, int32_t numOfCol1, int32_t* numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo) { - SColumnInfo* tableCols = calloc(numOfCol1, sizeof(SColumnInfo)); - for(int32_t i = 0; i < numOfCol1; ++i) { - SColumn* pCol = taosArrayGetP(px->colList, i); - if (pCol->info.flist.numOfFilters > 0) { - (*numOfFilterCols) += 1; - } - - tableCols[i] = pCol->info; - } - - if ((*numOfFilterCols) > 0) { - doCreateFilterInfo(tableCols, numOfCol1, (*numOfFilterCols), pFilterInfo, 0); - } - - tfree(tableCols); -} -*/ - -void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQueryInfo* px, SSqlObj* pSql) { - SSqlRes* pOutput = &pSql->res; - - // handle the following query process - if (px->pQInfo == NULL) { - SColumnInfo* pColumnInfo = extractColumnInfoFromResult(px->colList); - - STableMeta* pTableMeta = tscGetMetaInfo(px, 0)->pTableMeta; - SSchema* pSchema = tscGetTableSchema(pTableMeta); - - STableGroupInfo tableGroupInfo = { - .numOfTables = 1, - .pGroupList = taosArrayInit(1, POINTER_BYTES), - }; - - tableGroupInfo.map = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - - STableKeyInfo tableKeyInfo = {.pTable = NULL, .lastKey = INT64_MIN}; - - SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); - taosArrayPush(group, &tableKeyInfo); - - taosArrayPush(tableGroupInfo.pGroupList, &group); - - // if it is a join query, create join operator here - int32_t numOfCol1 = pTableMeta->tableInfo.numOfColumns; - - SFilterInfo *pFilters = NULL; - STblCond *pCond = NULL; - if (px->colCond) { - pCond = tsGetTableFilter(px->colCond, pTableMeta->id.uid, 0); - if (pCond && pCond->cond) { - createQueryFilter(pCond->cond, pCond->len, &pFilters); - } - } - - SOperatorInfo* pSourceOperator = createDummyInputOperator(pSqlObjList[0], pSchema, numOfCol1, pFilters); - - pOutput->precision = pSqlObjList[0]->res.precision; - - SSchema* schema = NULL; - if (px->numOfTables > 1) { - SOperatorInfo** p = calloc(px->numOfTables, POINTER_BYTES); - p[0] = pSourceOperator; - - int32_t num = (int32_t) taosArrayGetSize(px->colList); - schema = calloc(num, sizeof(SSchema)); - memcpy(schema, pSchema, numOfCol1*sizeof(SSchema)); - - int32_t offset = pSourceOperator->numOfOutput; - - for(int32_t i = 1; i < px->numOfTables; ++i) { - STableMeta* pTableMeta1 = tscGetMetaInfo(px, i)->pTableMeta; - numOfCol1 = pTableMeta1->tableInfo.numOfColumns; - SFilterInfo *pFilters1 = NULL; - - SSchema* pSchema1 = tscGetTableSchema(pTableMeta1); - int32_t n = pTableMeta1->tableInfo.numOfColumns; - - if (px->colCond) { - pCond = tsGetTableFilter(px->colCond, pTableMeta1->id.uid, i); - if (pCond && pCond->cond) { - createQueryFilter(pCond->cond, pCond->len, &pFilters1); - } - //createInputDataFilterInfo(px, numOfCol1, &numOfFilterCols1, &pFilterInfo1); - } - - p[i] = createDummyInputOperator(pSqlObjList[i], pSchema1, n, pFilters1); - memcpy(&schema[offset], pSchema1, n * sizeof(SSchema)); - offset += n; - } - - pSourceOperator = createJoinOperatorInfo(p, px->numOfTables, schema, num); - tfree(p); - } else { - size_t num = taosArrayGetSize(px->colList); - schema = calloc(num, sizeof(SSchema)); - memcpy(schema, pSchema, numOfCol1*sizeof(SSchema)); - } - - // update the exprinfo - int32_t numOfOutput = (int32_t)tscNumOfExprs(px); - for(int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pex = taosArrayGetP(px->exprList, i); - int32_t colId = pex->base.colInfo.colId; - for(int32_t j = 0; j < pSourceOperator->numOfOutput; ++j) { - if (colId == schema[j].colId) { - pex->base.colInfo.colIndex = j; - break; - } - } - } - - tscDebug("0x%"PRIx64" create QInfo 0x%"PRIx64" to execute the main query while all nest queries are ready", pSql->self, pSql->self); - px->pQInfo = createQInfoFromQueryNode(px, &tableGroupInfo, pSourceOperator, NULL, NULL, MASTER_SCAN, pSql->self); - - tfree(pColumnInfo); - tfree(schema); - - // set the pRuntimeEnv for pSourceOperator - pSourceOperator->pRuntimeEnv = &px->pQInfo->runtimeEnv; - } - - uint64_t qId = pSql->self; - qTableQuery(px->pQInfo, &qId); - convertQueryResult(pOutput, px, pSql->self, false); -} - -static void tscDestroyResPointerInfo(SSqlRes* pRes) { - if (pRes->buffer != NULL) { // free all buffers containing the multibyte string - for (int i = 0; i < pRes->numOfCols; i++) { - tfree(pRes->buffer[i]); - } - - pRes->numOfCols = 0; - } - - tfree(pRes->pRsp); - - tfree(pRes->tsrow); - tfree(pRes->length); - tfree(pRes->buffer); - tfree(pRes->urow); - - tfree(pRes->pColumnIndex); - tfree(pRes->final); - - pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free -} - -void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeCachedMeta, uint64_t id) { - if (pCmd == NULL) { - return; - } - - SQueryInfo* pQueryInfo = pCmd->pQueryInfo; - - while(pQueryInfo != NULL) { - SQueryInfo* p = pQueryInfo->sibling; - - size_t numOfUpstream = taosArrayGetSize(pQueryInfo->pUpstream); - for(int32_t i = 0; i < numOfUpstream; ++i) { - SQueryInfo* pUpQueryInfo = taosArrayGetP(pQueryInfo->pUpstream, i); - freeQueryInfoImpl(pUpQueryInfo); - - clearAllTableMetaInfo(pUpQueryInfo, removeCachedMeta, id); - if (pUpQueryInfo->pQInfo != NULL) { - qDestroyQueryInfo(pUpQueryInfo->pQInfo); - pUpQueryInfo->pQInfo = NULL; - } - - tfree(pUpQueryInfo); - } - - if (pQueryInfo->udfCopy) { - pQueryInfo->pUdfInfo = taosArrayDestroy(pQueryInfo->pUdfInfo); - } else { - pQueryInfo->pUdfInfo = tscDestroyUdfArrayList(pQueryInfo->pUdfInfo); - } - - freeQueryInfoImpl(pQueryInfo); - clearAllTableMetaInfo(pQueryInfo, removeCachedMeta, id); - - if (pQueryInfo->pQInfo != NULL) { - qDestroyQueryInfo(pQueryInfo->pQInfo); - pQueryInfo->pQInfo = NULL; - } - - tfree(pQueryInfo); - pQueryInfo = p; - } - - pCmd->pQueryInfo = NULL; - pCmd->active = NULL; -} - -void destroyTableNameList(SInsertStatementParam* pInsertParam) { - if (pInsertParam->numOfTables == 0) { - assert(pInsertParam->pTableNameList == NULL); - return; - } - - for(int32_t i = 0; i < pInsertParam->numOfTables; ++i) { - tfree(pInsertParam->pTableNameList[i]); - } - - pInsertParam->numOfTables = 0; - tfree(pInsertParam->pTableNameList); -} - -void tscResetSqlCmd(SSqlCmd* pCmd, bool clearCachedMeta, uint64_t id) { - pCmd->command = 0; - pCmd->numOfCols = 0; - pCmd->count = 0; - pCmd->msgType = 0; - - pCmd->insertParam.sql = NULL; - destroyTableNameList(&pCmd->insertParam); - - pCmd->insertParam.pTableBlockHashList = tscDestroyBlockHashTable(pCmd->insertParam.pTableBlockHashList, clearCachedMeta); - pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); - tfree(pCmd->insertParam.tagData.data); - pCmd->insertParam.tagData.dataLen = 0; - - tscFreeQueryInfo(pCmd, clearCachedMeta, id); - pCmd->pTableMetaMap = tscCleanupTableMetaMap(pCmd->pTableMetaMap); -} - -void* tscCleanupTableMetaMap(SHashObj* pTableMetaMap) { - if (pTableMetaMap == NULL) { - return NULL; - } - - STableMetaVgroupInfo* p = taosHashIterate(pTableMetaMap, NULL); - while (p) { - taosArrayDestroy(p->vgroupIdList); - tfree(p->pTableMeta); - p = taosHashIterate(pTableMetaMap, p); - } - - taosHashCleanup(pTableMetaMap); - return NULL; -} - -void tscFreeSqlResult(SSqlObj* pSql) { - SSqlRes* pRes = &pSql->res; - - tscDestroyGlobalMerger(pRes->pMerger); - pRes->pMerger = NULL; - - tscDestroyResPointerInfo(pRes); - memset(&pSql->res, 0, sizeof(SSqlRes)); -} - -void tscFreeSubobj(SSqlObj* pSql) { - if (pSql->subState.numOfSub == 0) { - return; - } - - tscDebug("0x%"PRIx64" start to free sub SqlObj, numOfSub:%d", pSql->self, pSql->subState.numOfSub); - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - tscDebug("0x%"PRIx64" free sub SqlObj:0x%"PRIx64", index:%d", pSql->self, pSql->pSubs[i]->self, i); - taos_free_result(pSql->pSubs[i]); - pSql->pSubs[i] = NULL; - } - - if (pSql->subState.states) { - pthread_mutex_destroy(&pSql->subState.mutex); - } - - tfree(pSql->subState.states); - pSql->subState.numOfSub = 0; -} - -/** - * The free operation will cause the pSql to be removed from hash table and free it in - * the function of processmsgfromserver is impossible in this case, since it will fail - * to retrieve pSqlObj in hashtable. - * - * @param pSql - */ -void tscFreeRegisteredSqlObj(void *pSql) { - assert(pSql != NULL); - - SSqlObj* p = *(SSqlObj**)pSql; - STscObj* pTscObj = p->pTscObj; - assert(RID_VALID(p->self)); - - int32_t num = atomic_sub_fetch_32(&pTscObj->numOfObj, 1); - int32_t total = atomic_sub_fetch_32(&tscNumOfObj, 1); - - tscDebug("0x%"PRIx64" free SqlObj, total in tscObj:%d, total:%d", p->self, num, total); - tscFreeSqlObj(p); - taosReleaseRef(tscRefId, pTscObj->rid); -} - -void tscFreeMetaSqlObj(int64_t *rid){ - if (RID_VALID(*rid)) { - SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, *rid); - if (pSql) { - taosRemoveRef(tscObjRef, *rid); - taosReleaseRef(tscObjRef, *rid); - } - - *rid = 0; - } -} - -void tscFreeSqlObj(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) { - return; - } - - tscDebug("0x%"PRIx64" start to free sqlObj", pSql->self); - - pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - - tscFreeMetaSqlObj(&pSql->metaRid); - tscFreeMetaSqlObj(&pSql->svgroupRid); - - SSqlCmd* pCmd = &pSql->cmd; - int32_t cmd = pCmd->command; - if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_GLOBALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || - cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscRemoveFromSqlList(pSql); - } - - tscFreeSubobj(pSql); - - pSql->signature = NULL; - pSql->fp = NULL; - tfree(pSql->sqlstr); - tfree(pSql->pBuf); - - tfree(pSql->pSubs); - pSql->subState.numOfSub = 0; - pSql->self = 0; - - tscFreeSqlResult(pSql); - tscResetSqlCmd(pCmd, false, pSql->self); - - tfree(pCmd->payload); - pCmd->allocSize = 0; - - tsem_destroy(&pSql->rspSem); - memset(pSql, 0, sizeof(*pSql)); - free(pSql); -} - -void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo) { - tfree(pColInfo->boundColumns); - tfree(pColInfo->cols); - tfree(pColInfo->colIdxInfo); -} - -void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { - if (pDataBlock == NULL) { - return; - } - - tfree(pDataBlock->pData); - - if (removeMeta) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pDataBlock->tableName, name); - - taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - } - - if (!pDataBlock->cloned) { - tfree(pDataBlock->params); - - // free the refcount for metermeta - if (pDataBlock->pTableMeta != NULL) { - tfree(pDataBlock->pTableMeta); - } - - tscDestroyBoundColumnInfo(&pDataBlock->boundColumnInfo); - } - - tfree(pDataBlock); -} - -SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, - uint32_t offset) { - uint32_t needed = pDataBlock->numOfParams + 1; - if (needed > pDataBlock->numOfAllocedParams) { - needed *= 2; - void* tmp = realloc(pDataBlock->params, needed * sizeof(SParamInfo)); - if (tmp == NULL) { - return NULL; - } - pDataBlock->params = (SParamInfo*)tmp; - pDataBlock->numOfAllocedParams = needed; - } - - SParamInfo* param = pDataBlock->params + pDataBlock->numOfParams; - param->idx = -1; - param->type = type; - param->timePrec = timePrec; - param->bytes = bytes; - param->offset = offset; - - ++pDataBlock->numOfParams; - return param; -} - -void* tscDestroyBlockArrayList(SArray* pDataBlockList) { - if (pDataBlockList == NULL) { - return NULL; - } - - size_t size = taosArrayGetSize(pDataBlockList); - for (int32_t i = 0; i < size; i++) { - void* d = taosArrayGetP(pDataBlockList, i); - tscDestroyDataBlock(d, false); - } - - taosArrayDestroy(pDataBlockList); - return NULL; -} - - -void freeUdfInfo(SUdfInfo* pUdfInfo) { - if (pUdfInfo == NULL) { - return; - } - - if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) { - (*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init); - } - - tfree(pUdfInfo->name); - - if (pUdfInfo->path) { - unlink(pUdfInfo->path); - } - - tfree(pUdfInfo->path); - - tfree(pUdfInfo->content); - - taosCloseDll(pUdfInfo->handle); -} - -// todo refactor -void* tscDestroyUdfArrayList(SArray* pUdfList) { - if (pUdfList == NULL) { - return NULL; - } - - size_t size = taosArrayGetSize(pUdfList); - for (int32_t i = 0; i < size; i++) { - SUdfInfo* udf = taosArrayGet(pUdfList, i); - freeUdfInfo(udf); - } - - taosArrayDestroy(pUdfList); - return NULL; -} - - - -void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta) { - if (pBlockHashTable == NULL) { - return NULL; - } - - STableDataBlocks** p = taosHashIterate(pBlockHashTable, NULL); - while(p) { - tscDestroyDataBlock(*p, removeMeta); - p = taosHashIterate(pBlockHashTable, p); - } - - taosHashCleanup(pBlockHashTable); - return NULL; -} - -int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { - SSqlCmd* pCmd = &pSql->cmd; - assert(pDataBlock->pTableMeta != NULL && pDataBlock->size <= pDataBlock->nAllocSize && pDataBlock->size > sizeof(SMsgDesc)); - - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - // todo remove it later - // set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache - if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) { - tNameAssign(&pTableMetaInfo->name, &pDataBlock->tableName); - - if (pTableMetaInfo->pTableMeta != NULL) { - tfree(pTableMetaInfo->pTableMeta); - } - - pTableMetaInfo->pTableMeta = tscTableMetaDup(pDataBlock->pTableMeta); - pTableMetaInfo->tableMetaSize = tscGetTableMetaSize(pDataBlock->pTableMeta); - pTableMetaInfo->tableMetaCapacity = (size_t)(pTableMetaInfo->tableMetaSize); - } - - /* - * the format of submit message is as follows [RPC header|message body|digest] - * the dataBlock only includes the RPC Header buffer and actual submit message body, - * space for digest needs additional space. - */ - int ret = tscAllocPayload(pCmd, pDataBlock->size); - if (TSDB_CODE_SUCCESS != ret) { - return ret; - } - - memcpy(pCmd->payload, pDataBlock->pData, pDataBlock->size); - - //the payloadLen should be actual message body size, the old value of payloadLen is the allocated payload size - pCmd->payloadLen = pDataBlock->size; - assert(pCmd->allocSize >= (uint32_t)(pCmd->payloadLen)); - - // NOTE: shell message size should not include SMsgDesc - int32_t size = pCmd->payloadLen - sizeof(SMsgDesc); - - SMsgDesc* pMsgDesc = (SMsgDesc*) pCmd->payload; - pMsgDesc->numOfVnodes = htonl(1); // always for one vnode - - SSubmitMsg *pShellMsg = (SSubmitMsg *)(pCmd->payload + sizeof(SMsgDesc)); - pShellMsg->header.vgId = htonl(pDataBlock->pTableMeta->vgId); // data in current block all routes to the same vgroup - pShellMsg->header.contLen = htonl(size); // the length not includes the size of SMsgDesc - pShellMsg->length = pShellMsg->header.contLen; - pShellMsg->numOfBlocks = htonl(pDataBlock->numOfTables); // the number of tables to be inserted - - tscDebug("0x%"PRIx64" submit msg built, vgId:%d numOfTables:%d", pSql->self, pDataBlock->pTableMeta->vgId, pDataBlock->numOfTables); - return TSDB_CODE_SUCCESS; -} - -SQueryInfo* tscGetQueryInfo(SSqlCmd* pCmd) { - return pCmd->active; -} - -/** - * create the in-memory buffer for each table to keep the submitted data block - * @param initialSize - * @param rowSize - * @param startOffset - * @param name - * @param dataBlocks - * @return - */ -int32_t tscCreateDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, SName* name, - STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { - STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); - if (dataBuf == NULL) { - tscError("failed to allocated memory, reason:%s", strerror(errno)); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - dataBuf->nAllocSize = (uint32_t)defaultSize; - dataBuf->headerSize = startOffset; - - // the header size will always be the startOffset value, reserved for the subumit block header - if (dataBuf->nAllocSize <= dataBuf->headerSize) { - dataBuf->nAllocSize = dataBuf->headerSize * 2; - } - - //dataBuf->pData = calloc(1, dataBuf->nAllocSize); - dataBuf->pData = malloc(dataBuf->nAllocSize); - if (dataBuf->pData == NULL) { - tscError("failed to allocated memory, reason:%s", strerror(errno)); - tfree(dataBuf); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); - - //Here we keep the tableMeta to avoid it to be remove by other threads. - dataBuf->pTableMeta = tscTableMetaDup(pTableMeta); - - SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; - SSchema* pSchema = tscGetTableSchema(dataBuf->pTableMeta); - tscSetBoundColumnInfo(pColInfo, pSchema, dataBuf->pTableMeta->tableInfo.numOfColumns); - - dataBuf->ordered = true; - dataBuf->prevTS = INT64_MIN; - dataBuf->rowSize = rowSize; - dataBuf->size = startOffset; - dataBuf->tsSource = -1; - dataBuf->vgId = dataBuf->pTableMeta->vgId; - - tNameAssign(&dataBuf->tableName, name); - - assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); - - *dataBlocks = dataBuf; - return TSDB_CODE_SUCCESS; -} - -int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, - SArray* pBlockList) { - *dataBlocks = NULL; - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); - if (t1 != NULL) { - *dataBlocks = *t1; - } - - if (*dataBlocks == NULL) { - int32_t ret = tscCreateDataBlock((size_t)size, rowSize, startOffset, name, pTableMeta, dataBlocks); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES); - if (pBlockList) { - taosArrayPush(pBlockList, dataBlocks); - } - } - - return TSDB_CODE_SUCCESS; -} - -// Erase the empty space reserved for binary data -static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SInsertStatementParam* insertParam, - SBlockKeyTuple* blkKeyTuple) { - // TODO: optimize this function, handle the case while binary is not presented - STableMeta* pTableMeta = pTableDataBlock->pTableMeta; - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - SSchema* pSchema = tscGetTableSchema(pTableMeta); - - SSubmitBlk* pBlock = pDataBlock; - memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk)); - pDataBlock = (char*)pDataBlock + sizeof(SSubmitBlk); - - int32_t flen = 0; // original total length of row - - // schema needs to be included into the submit data block - if (insertParam->schemaAttached) { - int32_t numOfCols = tscGetNumOfColumns(pTableDataBlock->pTableMeta); - for(int32_t j = 0; j < numOfCols; ++j) { - STColumn* pCol = (STColumn*) pDataBlock; - pCol->colId = htons(pSchema[j].colId); - pCol->type = pSchema[j].type; - pCol->bytes = htons(pSchema[j].bytes); - pCol->offset = 0; - - pDataBlock = (char*)pDataBlock + sizeof(STColumn); - flen += TYPE_BYTES[pSchema[j].type]; - } - - int32_t schemaSize = sizeof(STColumn) * numOfCols; - pBlock->schemaLen = schemaSize; - } else { - if (IS_RAW_PAYLOAD(insertParam->payloadType)) { - for (int32_t j = 0; j < tinfo.numOfColumns; ++j) { - flen += TYPE_BYTES[pSchema[j].type]; - } - } - pBlock->schemaLen = 0; - } - - char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); - pBlock->dataLen = 0; - int32_t numOfRows = htons(pBlock->numOfRows); - - if (IS_RAW_PAYLOAD(insertParam->payloadType)) { - for (int32_t i = 0; i < numOfRows; ++i) { - SMemRow memRow = (SMemRow)pDataBlock; - memRowSetType(memRow, SMEM_ROW_DATA); - SDataRow trow = memRowDataBody(memRow); - dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + flen)); - dataRowSetVersion(trow, pTableMeta->sversion); - - int toffset = 0; - for (int32_t j = 0; j < tinfo.numOfColumns; j++) { - tdAppendColVal(trow, p, pSchema[j].type, toffset); - toffset += TYPE_BYTES[pSchema[j].type]; - p += pSchema[j].bytes; - } - - pDataBlock = (char*)pDataBlock + memRowTLen(memRow); - pBlock->dataLen += memRowTLen(memRow); - } - } else { - for (int32_t i = 0; i < numOfRows; ++i) { - char* payload = (blkKeyTuple + i)->payloadAddr; - TDRowLenT rowTLen = memRowTLen(payload); - memcpy(pDataBlock, payload, rowTLen); - pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); - pBlock->dataLen += rowTLen; - } - } - - int32_t len = pBlock->dataLen + pBlock->schemaLen; - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->schemaLen = htonl(pBlock->schemaLen); - - return len; -} - -static int32_t getRowExpandSize(STableMeta* pTableMeta) { - int32_t result = TD_MEM_ROW_DATA_HEAD_SIZE; - int32_t columns = tscGetNumOfColumns(pTableMeta); - SSchema* pSchema = tscGetTableSchema(pTableMeta); - for (int32_t i = 0; i < columns; i++) { - if (IS_VAR_DATA_TYPE((pSchema + i)->type)) { - result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; - } - } - return result; -} - -static void extractTableNameList(SInsertStatementParam *pInsertParam, bool freeBlockMap) { - pInsertParam->numOfTables = (int32_t) taosHashGetSize(pInsertParam->pTableBlockHashList); - if (pInsertParam->pTableNameList == NULL) { - pInsertParam->pTableNameList = malloc(pInsertParam->numOfTables * POINTER_BYTES); - } - - STableDataBlocks **p1 = taosHashIterate(pInsertParam->pTableBlockHashList, NULL); - int32_t i = 0; - while(p1) { - STableDataBlocks* pBlocks = *p1; - //tfree(pInsertParam->pTableNameList[i]); - - pInsertParam->pTableNameList[i++] = tNameDup(&pBlocks->tableName); - p1 = taosHashIterate(pInsertParam->pTableBlockHashList, p1); - } - - if (freeBlockMap) { - pInsertParam->pTableBlockHashList = tscDestroyBlockHashTable(pInsertParam->pTableBlockHashList, false); - } -} - -int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBlockMap) { - const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); - int code = 0; - bool isRawPayload = IS_RAW_PAYLOAD(pInsertParam->payloadType); - void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); - - STableDataBlocks** p = taosHashIterate(pInsertParam->pTableBlockHashList, NULL); - - STableDataBlocks* pOneTableBlock = *p; - - SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock - - while(pOneTableBlock) { - SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; - if (pBlocks->numOfRows > 0) { - // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; - STableDataBlocks* dataBuf = NULL; - - int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, - INSERT_HEAD_SIZE, 0, &pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); - if (ret != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" failed to prepare the data block buffer for merging table data, code:%d", pInsertParam->objectId, ret); - taosHashCleanup(pVnodeDataBlockHashList); - tscDestroyBlockArrayList(pVnodeDataBlockList); - tfree(blkKeyInfo.pKeyTuple); - return ret; - } - - int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + - sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); - - if (dataBuf->nAllocSize < destSize) { - dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); - - char* tmp = realloc(dataBuf->pData, dataBuf->nAllocSize); - if (tmp != NULL) { - dataBuf->pData = tmp; - //memset(dataBuf->pData + dataBuf->size, 0, dataBuf->nAllocSize - dataBuf->size); - } else { // failed to allocate memory, free already allocated memory and return error code - tscError("0x%"PRIx64" failed to allocate memory for merging submit block, size:%d", pInsertParam->objectId, dataBuf->nAllocSize); - - taosHashCleanup(pVnodeDataBlockHashList); - tscDestroyBlockArrayList(pVnodeDataBlockList); - tfree(dataBuf->pData); - tfree(blkKeyInfo.pKeyTuple); - - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - if (isRawPayload) { - tscSortRemoveDataBlockDupRowsRaw(pOneTableBlock); - char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize * (pBlocks->numOfRows - 1); - - tscDebug("0x%" PRIx64 " name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, - pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), pBlocks->tid, - pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); - } else { - if ((code = tscSortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) { - taosHashCleanup(pVnodeDataBlockHashList); - tscDestroyBlockArrayList(pVnodeDataBlockList); - tfree(dataBuf->pData); - tfree(blkKeyInfo.pKeyTuple); - return code; - } - ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); - - SBlockKeyTuple* pLastKeyTuple = blkKeyInfo.pKeyTuple + pBlocks->numOfRows - 1; - tscDebug("0x%" PRIx64 " name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, - pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), pBlocks->tid, - pBlocks->numOfRows, pBlocks->sversion, blkKeyInfo.pKeyTuple->skey, pLastKeyTuple->skey); - } - - int32_t len = pBlocks->numOfRows * - (isRawPayload ? (pOneTableBlock->rowSize + expandSize) : getExtendedRowSize(pOneTableBlock)) + - sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); - - pBlocks->tid = htonl(pBlocks->tid); - pBlocks->uid = htobe64(pBlocks->uid); - pBlocks->sversion = htonl(pBlocks->sversion); - pBlocks->numOfRows = htons(pBlocks->numOfRows); - pBlocks->schemaLen = 0; - - // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pInsertParam, blkKeyInfo.pKeyTuple); - assert(finalLen <= len); - - dataBuf->size += (finalLen + sizeof(SSubmitBlk)); - assert(dataBuf->size <= dataBuf->nAllocSize); - - // the length does not include the SSubmitBlk structure - pBlocks->dataLen = htonl(finalLen); - dataBuf->numOfTables += 1; - - pBlocks->numOfRows = 0; - }else { - tscDebug("0x%"PRIx64" table %s data block is empty", pInsertParam->objectId, pOneTableBlock->tableName.tname); - } - - p = taosHashIterate(pInsertParam->pTableBlockHashList, p); - if (p == NULL) { - break; - } - - pOneTableBlock = *p; - } - - extractTableNameList(pInsertParam, freeBlockMap); - - // free the table data blocks; - pInsertParam->pDataBlocks = pVnodeDataBlockList; - taosHashCleanup(pVnodeDataBlockHashList); - tfree(blkKeyInfo.pKeyTuple); - - return TSDB_CODE_SUCCESS; -} - -// TODO: all subqueries should be freed correctly before close this connection. -void tscCloseTscObj(void *param) { - STscObj *pObj = param; - - pObj->signature = NULL; - taosTmrStopA(&(pObj->pTimer)); - - tfree(pObj->tscCorMgmtEpSet); - tscReleaseRpc(pObj->pRpcObj); - pthread_mutex_destroy(&pObj->mutex); - - tfree(pObj); -} - -bool tscIsInsertData(char* sqlstr) { - int32_t index = 0; - - do { - SStrToken t0 = tStrGetToken(sqlstr, &index, false); - if (t0.type != TK_LP) { - return t0.type == TK_INSERT || t0.type == TK_IMPORT; - } - } while (1); -} - -int32_t tscAllocPayloadFast(SSqlCmd *pCmd, size_t size) { - if (pCmd->payload == NULL) { - assert(pCmd->allocSize == 0); - - pCmd->payload = malloc(size); - pCmd->allocSize = (uint32_t) size; - } else if (pCmd->allocSize < size) { - char* tmp = realloc(pCmd->payload, size); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pCmd->payload = tmp; - pCmd->allocSize = (uint32_t) size; - } - - assert(pCmd->allocSize >= size); - return TSDB_CODE_SUCCESS; -} - -int32_t tscAllocPayload(SSqlCmd* pCmd, int size) { - assert(size > 0); - - int32_t code = tscAllocPayloadFast(pCmd, (size_t) size); - if (code == TSDB_CODE_SUCCESS) { - memset(pCmd->payload, 0, pCmd->allocSize); - } - - return code; -} - -TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes) { - TAOS_FIELD f = { .type = type, .bytes = bytes, }; - tstrncpy(f.name, name, sizeof(f.name)); - return f; -} - -int32_t tscGetFirstInvisibleFieldPos(SQueryInfo* pQueryInfo) { - if (pQueryInfo->fieldsInfo.numOfOutput <= 0 || pQueryInfo->fieldsInfo.internalField == NULL) { - return 0; - } - - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - SInternalField* pField = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); - if (!pField->visible) { - return i; - } - } - - return pQueryInfo->fieldsInfo.numOfOutput; -} - - -SInternalField* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) { - assert(pFieldInfo != NULL); - pFieldInfo->numOfOutput++; - - struct SInternalField info = { .pExpr = NULL, .visible = true }; - - info.field = *pField; - return taosArrayPush(pFieldInfo->internalField, &info); -} - -SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field) { - pFieldInfo->numOfOutput++; - struct SInternalField info = { .pExpr = NULL, .visible = true }; - - info.field = *field; - return taosArrayInsert(pFieldInfo->internalField, index, &info); -} - -void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) { - int32_t offset = 0; - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* p = taosArrayGetP(pQueryInfo->exprList, i); - - p->base.offset = offset; - offset += p->base.resBytes; - } -} - -SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index) { - assert(index < pFieldInfo->numOfOutput); - return TARRAY_GET_ELEM(pFieldInfo->internalField, index); -} - -TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) { - assert(index < pFieldInfo->numOfOutput); - return &((SInternalField*)TARRAY_GET_ELEM(pFieldInfo->internalField, index))->field; -} - -int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) { - SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, index); - assert(pInfo != NULL && pInfo->pExpr->pExpr == NULL); - - return pInfo->pExpr->base.offset; -} - -int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2, int32_t *diffSize) { - assert(pFieldInfo1 != NULL && pFieldInfo2 != NULL); - - if (pFieldInfo1->numOfOutput != pFieldInfo2->numOfOutput) { - return pFieldInfo1->numOfOutput - pFieldInfo2->numOfOutput; - } - - for (int32_t i = 0; i < pFieldInfo1->numOfOutput; ++i) { - TAOS_FIELD* pField1 = tscFieldInfoGetField((SFieldInfo*) pFieldInfo1, i); - TAOS_FIELD* pField2 = tscFieldInfoGetField((SFieldInfo*) pFieldInfo2, i); - - if (pField1->type != pField2->type || - strcasecmp(pField1->name, pField2->name) != 0) { - return 1; - } - - if (pField1->bytes != pField2->bytes) { - *diffSize = 1; - - if (pField2->bytes > pField1->bytes) { - assert(IS_VAR_DATA_TYPE(pField1->type)); - pField1->bytes = pField2->bytes; - } - } - } - - return 0; -} - -int32_t tscFieldInfoSetSize(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2) { - assert(pFieldInfo1 != NULL && pFieldInfo2 != NULL); - - for (int32_t i = 0; i < pFieldInfo1->numOfOutput; ++i) { - TAOS_FIELD* pField1 = tscFieldInfoGetField((SFieldInfo*) pFieldInfo1, i); - TAOS_FIELD* pField2 = tscFieldInfoGetField((SFieldInfo*) pFieldInfo2, i); - - pField2->bytes = pField1->bytes; - } - - return 0; -} - - -int32_t tscGetResRowLength(SArray* pExprList) { - size_t num = taosArrayGetSize(pExprList); - if (num == 0) { - return 0; - } - - int32_t size = 0; - for(int32_t i = 0; i < num; ++i) { - SExprInfo* pExpr = taosArrayGetP(pExprList, i); - size += pExpr->base.resBytes; - } - - return size; -} - -static void destroyFilterInfo(SColumnFilterList* pFilterList) { - if (pFilterList->filterInfo == NULL) { - pFilterList->numOfFilters = 0; - return; - } - - for(int32_t i = 0; i < pFilterList->numOfFilters; ++i) { - if (pFilterList->filterInfo[i].filterstr) { - tfree(pFilterList->filterInfo[i].pz); - } - } - - tfree(pFilterList->filterInfo); - pFilterList->numOfFilters = 0; -} - -void* sqlExprDestroy(SExprInfo* pExpr) { - if (pExpr == NULL) { - return NULL; - } - - SSqlExpr* p = &pExpr->base; - for(int32_t i = 0; i < tListLen(p->param); ++i) { - tVariantDestroy(&p->param[i]); - } - - if (p->flist.numOfFilters > 0) { - tfree(p->flist.filterInfo); - } - - if (pExpr->pExpr != NULL) { - tExprTreeDestroy(pExpr->pExpr, NULL); - } - - tfree(pExpr); - return NULL; -} - -void tscFieldInfoClear(SFieldInfo* pFieldInfo) { - if (pFieldInfo == NULL) { - return; - } - - if (pFieldInfo->internalField != NULL) { - size_t num = taosArrayGetSize(pFieldInfo->internalField); - for (int32_t i = 0; i < num; ++i) { - SInternalField* pfield = taosArrayGet(pFieldInfo->internalField, i); - if (pfield->pExpr != NULL && pfield->pExpr->pExpr != NULL) { - sqlExprDestroy(pfield->pExpr); - } - } - } - - taosArrayDestroy(pFieldInfo->internalField); - tfree(pFieldInfo->final); - - memset(pFieldInfo, 0, sizeof(SFieldInfo)); -} - -void tscFieldInfoCopy(SFieldInfo* pFieldInfo, const SFieldInfo* pSrc, const SArray* pExprList) { - assert(pFieldInfo != NULL && pSrc != NULL && pExprList != NULL); - pFieldInfo->numOfOutput = pSrc->numOfOutput; - - if (pSrc->final != NULL) { - pFieldInfo->final = calloc(pSrc->numOfOutput, sizeof(TAOS_FIELD)); - memcpy(pFieldInfo->final, pSrc->final, sizeof(TAOS_FIELD) * pSrc->numOfOutput); - } - - if (pSrc->internalField != NULL) { - size_t num = taosArrayGetSize(pSrc->internalField); - size_t numOfExpr = taosArrayGetSize(pExprList); - - for (int32_t i = 0; i < num; ++i) { - SInternalField* pfield = taosArrayGet(pSrc->internalField, i); - - SInternalField p = {.visible = pfield->visible, .field = pfield->field}; - - bool found = false; - int32_t resColId = pfield->pExpr->base.resColId; - for(int32_t j = 0; j < numOfExpr; ++j) { - SExprInfo* pExpr = taosArrayGetP(pExprList, j); - if (pExpr->base.resColId == resColId) { - p.pExpr = pExpr; - found = true; - break; - } - } - - if (!found) { - assert(pfield->pExpr->pExpr != NULL); - p.pExpr = calloc(1, sizeof(SExprInfo)); - tscExprAssign(p.pExpr, pfield->pExpr); - } - - taosArrayPush(pFieldInfo->internalField, &p); - } - } -} - - -SExprInfo* tscExprCreate(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t resColId, int16_t interSize, int32_t colType) { - SExprInfo* pExpr = calloc(1, sizeof(SExprInfo)); - if (pExpr == NULL) { - return NULL; - } - - SSqlExpr* p = &pExpr->base; - p->functionId = functionId; - - // set the correct columnIndex index - if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema* s = tGetTbnameColumnSchema(); - p->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; - p->colBytes = s->bytes; - p->colType = s->type; - } else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX) { - p->colInfo.colId = pColIndex->columnIndex; - p->colBytes = size; - p->colType = type; - } else if (functionId == TSDB_FUNC_BLKINFO) { - p->colInfo.colId = pColIndex->columnIndex; - p->colBytes = TSDB_MAX_BINARY_LEN; - p->colType = TSDB_DATA_TYPE_BINARY; - } else { - int32_t len = tListLen(p->colInfo.name); - if (TSDB_COL_IS_TAG(colType)) { - SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - p->colInfo.colId = pSchema[pColIndex->columnIndex].colId; - p->colBytes = pSchema[pColIndex->columnIndex].bytes; - p->colType = pSchema[pColIndex->columnIndex].type; - snprintf(p->colInfo.name, len, "%s.%s", pTableMetaInfo->aliasName, pSchema[pColIndex->columnIndex].name); - } else if (pTableMetaInfo->pTableMeta != NULL) { - // in handling select database/version/server_status(), the pTableMeta is NULL - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex); - p->colInfo.colId = pSchema->colId; - p->colBytes = pSchema->bytes; - p->colType = pSchema->type; - snprintf(p->colInfo.name, len, "%s.%s", pTableMetaInfo->aliasName, pSchema->name); - } - } - - p->colInfo.flag = colType; - p->colInfo.colIndex = pColIndex->columnIndex; - - p->resType = type; - p->resBytes = size; - p->resColId = resColId; - p->interBytes = interSize; - - if (pTableMetaInfo->pTableMeta) { - p->uid = pTableMetaInfo->pTableMeta->id.uid; - } - - return pExpr; -} - -SExprInfo* tscExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) { - int32_t num = (int32_t)taosArrayGetSize(pQueryInfo->exprList); - if (index == num) { - return tscExprAppend(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); - SExprInfo* pExpr = tscExprCreate(pTableMetaInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); - taosArrayInsert(pQueryInfo->exprList, index, &pExpr); - return pExpr; -} - -SExprInfo* tscExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); - SExprInfo* pExpr = tscExprCreate(pTableMetaInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); - taosArrayPush(pQueryInfo->exprList, &pExpr); - return pExpr; -} - -SExprInfo* tscExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, - int16_t type, int16_t size) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SExprInfo* pExpr = tscExprGet(pQueryInfo, index); - if (pExpr == NULL) { - return NULL; - } - - SSqlExpr* pse = &pExpr->base; - pse->functionId = functionId; - - pse->colInfo.colIndex = srcColumnIndex; - pse->colInfo.colId = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, srcColumnIndex)->colId; - - pse->resType = type; - pse->resBytes = size; - - return pExpr; -} - -bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t index) { - if (!UTIL_TABLE_IS_SUPER_TABLE(pQueryInfo->pTableMetaInfo[index])) { - return false; - } - - int32_t numOfExprs = (int32_t) tscNumOfExprs(pQueryInfo); - for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr->base.functionId == TSDB_FUNC_STDDEV_DST) { - return true; - } - } - - return false; -} - -size_t tscNumOfExprs(SQueryInfo* pQueryInfo) { - return taosArrayGetSize(pQueryInfo->exprList); -} - -int32_t tscExprTopBottomIndex(SQueryInfo* pQueryInfo){ - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - if (pExpr == NULL) - continue; - if (pExpr->base.functionId == TSDB_FUNC_TOP || pExpr->base.functionId == TSDB_FUNC_BOTTOM) { - return i; - } - } - return -1; -} - -// todo REFACTOR -void tscExprAddParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes) { - assert (pExpr != NULL || argument != NULL || bytes != 0); - - // set parameter value - // transfer to tVariant from byte data/no ascii data - tVariantCreateFromBinary(&pExpr->param[pExpr->numOfParams], argument, bytes, type); - pExpr->numOfParams += 1; - - assert(pExpr->numOfParams <= 3); -} - -SExprInfo* tscExprGet(SQueryInfo* pQueryInfo, int32_t index) { - return taosArrayGetP(pQueryInfo->exprList, index); -} - -/* - * NOTE: Does not release SExprInfo here. - */ -void tscExprDestroy(SArray* pExprInfo) { - size_t size = taosArrayGetSize(pExprInfo); - - for(int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); - sqlExprDestroy(pExpr); - } - - taosArrayDestroy(pExprInfo); -} - -int32_t tscExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) { - assert(src != NULL && dst != NULL); - - size_t size = taosArrayGetSize(src); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = taosArrayGetP(src, i); - - if (pExpr->base.uid == uid) { - if (deepcopy) { - SExprInfo* p1 = calloc(1, sizeof(SExprInfo)); - tscExprAssign(p1, pExpr); - - taosArrayPush(dst, &p1); - } else { - taosArrayPush(dst, &pExpr); - } - } - } - - return 0; -} - -int32_t tscExprCopyAll(SArray* dst, const SArray* src, bool deepcopy) { - assert(src != NULL && dst != NULL); - - size_t size = taosArrayGetSize(src); - for (int32_t i = 0; i < size; ++i) { - SExprInfo* pExpr = taosArrayGetP(src, i); - - if (deepcopy) { - SExprInfo* p1 = calloc(1, sizeof(SExprInfo)); - tscExprAssign(p1, pExpr); - - taosArrayPush(dst, &p1); - } else { - taosArrayPush(dst, &pExpr); - } - } - - return 0; -} - -// ignore the tbname columnIndex to be inserted into source list -int32_t tscColumnExists(SArray* pColumnList, int32_t columnId, uint64_t uid) { - size_t numOfCols = taosArrayGetSize(pColumnList); - - int32_t i = 0; - while (i < numOfCols) { - SColumn* pCol = taosArrayGetP(pColumnList, i); - if ((pCol->info.colId != columnId) || (pCol->tableUid != uid)) { - ++i; - continue; - } else { - break; - } - } - - if (i >= numOfCols || numOfCols == 0) { - return -1; - } - - return i; -} - -void tscExprAssign(SExprInfo* dst, const SExprInfo* src) { - assert(dst != NULL && src != NULL); - - *dst = *src; - - if (src->base.flist.numOfFilters > 0) { - dst->base.flist.filterInfo = calloc(src->base.flist.numOfFilters, sizeof(SColumnFilterInfo)); - memcpy(dst->base.flist.filterInfo, src->base.flist.filterInfo, sizeof(SColumnFilterInfo) * src->base.flist.numOfFilters); - } - - dst->pExpr = exprdup(src->pExpr); - - memset(dst->base.param, 0, sizeof(tVariant) * tListLen(dst->base.param)); - for (int32_t j = 0; j < src->base.numOfParams; ++j) { - tVariantAssign(&dst->base.param[j], &src->base.param[j]); - } -} - -SColumn* tscColumnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema) { - // ignore the tbname columnIndex to be inserted into source list - if (columnIndex < 0) { - return NULL; - } - - size_t numOfCols = taosArrayGetSize(pColumnList); - - int32_t i = 0; - while (i < numOfCols) { - SColumn* pCol = taosArrayGetP(pColumnList, i); - if (pCol->columnIndex < columnIndex) { - i++; - } else if (pCol->tableUid < uid) { - i++; - } else { - break; - } - } - - if (i >= numOfCols || numOfCols == 0) { - SColumn* b = calloc(1, sizeof(SColumn)); - if (b == NULL) { - return NULL; - } - - b->columnIndex = columnIndex; - b->tableUid = uid; - b->info.colId = pSchema->colId; - b->info.bytes = pSchema->bytes; - b->info.type = pSchema->type; - - taosArrayInsert(pColumnList, i, &b); - } else { - SColumn* pCol = taosArrayGetP(pColumnList, i); - - if (i < numOfCols && (pCol->columnIndex > columnIndex || pCol->tableUid != uid)) { - SColumn* b = calloc(1, sizeof(SColumn)); - if (b == NULL) { - return NULL; - } - - b->columnIndex = columnIndex; - b->tableUid = uid; - b->info.colId = pSchema->colId; - b->info.bytes = pSchema->bytes; - b->info.type = pSchema->type; - - taosArrayInsert(pColumnList, i, &b); - } - } - - return taosArrayGetP(pColumnList, i); -} - - - -SColumn* tscColumnClone(const SColumn* src) { - assert(src != NULL); - - SColumn* dst = calloc(1, sizeof(SColumn)); - if (dst == NULL) { - return NULL; - } - - tscColumnCopy(dst, src); - return dst; -} - -static void tscColumnDestroy(SColumn* pCol) { - destroyFilterInfo(&pCol->info.flist); - free(pCol); -} - -void tscColumnCopy(SColumn* pDest, const SColumn* pSrc) { - destroyFilterInfo(&pDest->info.flist); - - pDest->columnIndex = pSrc->columnIndex; - pDest->tableUid = pSrc->tableUid; - pDest->info.flist.numOfFilters = pSrc->info.flist.numOfFilters; - pDest->info.flist.filterInfo = tFilterInfoDup(pSrc->info.flist.filterInfo, pSrc->info.flist.numOfFilters); - pDest->info.type = pSrc->info.type; - pDest->info.colId = pSrc->info.colId; - pDest->info.bytes = pSrc->info.bytes; -} - - - -void tscColumnListCopyAll(SArray* dst, const SArray* src) { - assert(src != NULL && dst != NULL); - - size_t num = taosArrayGetSize(src); - for (int32_t i = 0; i < num; ++i) { - SColumn* pCol = taosArrayGetP(src, i); - SColumn* p = tscColumnClone(pCol); - taosArrayPush(dst, &p); - } -} - - -void tscColumnListDestroy(SArray* pColumnList) { - if (pColumnList == NULL) { - return; - } - - size_t num = taosArrayGetSize(pColumnList); - for (int32_t i = 0; i < num; ++i) { - SColumn* pCol = taosArrayGetP(pColumnList, i); - tscColumnDestroy(pCol); - } - - taosArrayDestroy(pColumnList); -} - -/* - * 1. normal name, not a keyword or number - * 2. name with quote - * 3. string with only one delimiter '.'. - * - * only_one_part - * 'only_one_part' - * first_part.second_part - * first_part.'second_part' - * 'first_part'.second_part - * 'first_part'.'second_part' - * 'first_part.second_part' - * - */ -static int32_t validateQuoteToken(SStrToken* pToken) { - tscDequoteAndTrimToken(pToken); - - int32_t k = tGetToken(pToken->z, &pToken->type); - - if (pToken->type == TK_STRING) { - return tscValidateName(pToken); - } - - if (k != pToken->n || pToken->type != TK_ID) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - return TSDB_CODE_SUCCESS; -} - -void tscDequoteAndTrimToken(SStrToken* pToken) { - uint32_t first = 0, last = pToken->n; - - // trim leading spaces - while (first < last) { - char c = pToken->z[first]; - if (c != ' ' && c != '\t') { - break; - } - first++; - } - - // trim ending spaces - while (first < last) { - char c = pToken->z[last - 1]; - if (c != ' ' && c != '\t') { - break; - } - last--; - } - - // there are still at least two characters - if (first < last - 1) { - char c = pToken->z[first]; - // dequote - if ((c == '\'' || c == '"') && c == pToken->z[last - 1]) { - first++; - last--; - } - } - - // left shift the string and pad spaces - for (uint32_t i = 0; i + first < last; i++) { - pToken->z[i] = pToken->z[first + i]; - } - for (uint32_t i = last - first; i < pToken->n; i++) { - pToken->z[i] = ' '; - } - - // adjust token length - pToken->n = last - first; -} - -int32_t tscValidateName(SStrToken* pToken) { - if (pToken == NULL || pToken->z == NULL || - (pToken->type != TK_STRING && pToken->type != TK_ID)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - char* sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); - if (sep == NULL) { // single part - if (pToken->type == TK_STRING) { - - tscDequoteAndTrimToken(pToken); - tscStrToLower(pToken->z, pToken->n); - //pToken->n = (uint32_t)strtrim(pToken->z); - - int len = tGetToken(pToken->z, &pToken->type); - - // single token, validate it - if (len == pToken->n) { - return validateQuoteToken(pToken); - } else { - sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); - if (sep == NULL) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return tscValidateName(pToken); - } - } else { - if (isNumber(pToken)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - } else { // two part - int32_t oldLen = pToken->n; - char* pStr = pToken->z; - - if (pToken->type == TK_SPACE) { - pToken->n = (uint32_t)strtrim(pToken->z); - } - - pToken->n = tGetToken(pToken->z, &pToken->type); - if (pToken->z[pToken->n] != TS_PATH_DELIMITER[0]) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pToken->type != TK_STRING && pToken->type != TK_ID) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pToken->type == TK_STRING && validateQuoteToken(pToken) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int32_t firstPartLen = pToken->n; - - pToken->z = sep + 1; - pToken->n = (uint32_t)(oldLen - (sep - pStr) - 1); - int32_t len = tGetToken(pToken->z, &pToken->type); - if (len != pToken->n || (pToken->type != TK_STRING && pToken->type != TK_ID)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pToken->type == TK_STRING && validateQuoteToken(pToken) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // re-build the whole name string - if (pStr[firstPartLen] == TS_PATH_DELIMITER[0]) { - // first part do not have quote do nothing - } else { - pStr[firstPartLen] = TS_PATH_DELIMITER[0]; - memmove(&pStr[firstPartLen + 1], pToken->z, pToken->n); - uint32_t offset = (uint32_t)(pToken->z - (pStr + firstPartLen + 1)); - memset(pToken->z + pToken->n - offset, ' ', offset); - } - pToken->n += (firstPartLen + sizeof(TS_PATH_DELIMITER[0])); - pToken->z = pStr; - - tscStrToLower(pToken->z,pToken->n); - } - - return TSDB_CODE_SUCCESS; -} - -void tscIncStreamExecutionCount(void* pStream) { - if (pStream == NULL) { - return; - } - - SSqlStream* ps = (SSqlStream*)pStream; - ps->num += 1; -} - -bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams) { - if (pTableMetaInfo->pTableMeta == NULL) { - return false; - } - - if (colId == TSDB_TBNAME_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) { - return true; - } - - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - int32_t numOfTotal = tinfo.numOfTags + tinfo.numOfColumns; - - for (int32_t i = 0; i < numOfTotal; ++i) { - if (pSchema[i].colId == colId) { - return true; - } - } - - return false; -} - -int32_t tscTagCondCopy(STagCond* dest, const STagCond* src) { - memset(dest, 0, sizeof(STagCond)); - - if (src->tbnameCond.cond != NULL) { - dest->tbnameCond.cond = strdup(src->tbnameCond.cond); - if (dest->tbnameCond.cond == NULL) { - return -1; - } - } - - dest->tbnameCond.uid = src->tbnameCond.uid; - dest->tbnameCond.len = src->tbnameCond.len; - - dest->joinInfo.hasJoin = src->joinInfo.hasJoin; - - for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) { - if (src->joinInfo.joinTables[i]) { - dest->joinInfo.joinTables[i] = calloc(1, sizeof(SJoinNode)); - - memcpy(dest->joinInfo.joinTables[i], src->joinInfo.joinTables[i], sizeof(SJoinNode)); - - if (src->joinInfo.joinTables[i]->tsJoin) { - dest->joinInfo.joinTables[i]->tsJoin = taosArrayDup(src->joinInfo.joinTables[i]->tsJoin); - } - - if (src->joinInfo.joinTables[i]->tagJoin) { - dest->joinInfo.joinTables[i]->tagJoin = taosArrayDup(src->joinInfo.joinTables[i]->tagJoin); - } - } - } - - - dest->relType = src->relType; - - if (src->pCond == NULL) { - return 0; - } - - size_t s = taosArrayGetSize(src->pCond); - dest->pCond = taosArrayInit(s, sizeof(SCond)); - - for (int32_t i = 0; i < s; ++i) { - SCond* pCond = taosArrayGet(src->pCond, i); - - SCond c = {0}; - c.len = pCond->len; - c.uid = pCond->uid; - - if (pCond->len > 0) { - assert(pCond->cond != NULL); - c.cond = malloc(c.len); - if (c.cond == NULL) { - return -1; - } - - memcpy(c.cond, pCond->cond, c.len); - } - - taosArrayPush(dest->pCond, &c); - } - - return 0; -} - -int32_t tscColCondCopy(SArray** dest, const SArray* src, uint64_t uid, int16_t tidx) { - if (src == NULL) { - return 0; - } - - size_t s = taosArrayGetSize(src); - *dest = taosArrayInit(s, sizeof(SCond)); - - for (int32_t i = 0; i < s; ++i) { - STblCond* pCond = taosArrayGet(src, i); - STblCond c = {0}; - - if (tidx > 0) { - if (!(pCond->uid == uid && pCond->idx == tidx)) { - continue; - } - - c.idx = 0; - } else { - c.idx = pCond->idx; - } - - c.len = pCond->len; - c.uid = pCond->uid; - - if (pCond->len > 0) { - assert(pCond->cond != NULL); - c.cond = malloc(c.len); - if (c.cond == NULL) { - return -1; - } - - memcpy(c.cond, pCond->cond, c.len); - } - - taosArrayPush(*dest, &c); - } - - return 0; -} - -void tscColCondRelease(SArray** pCond) { - if (*pCond == NULL) { - return; - } - - size_t s = taosArrayGetSize(*pCond); - for (int32_t i = 0; i < s; ++i) { - STblCond* p = taosArrayGet(*pCond, i); - tfree(p->cond); - } - - taosArrayDestroy(*pCond); - - *pCond = NULL; -} - - -void tscTagCondRelease(STagCond* pTagCond) { - free(pTagCond->tbnameCond.cond); - - if (pTagCond->pCond != NULL) { - size_t s = taosArrayGetSize(pTagCond->pCond); - for (int32_t i = 0; i < s; ++i) { - SCond* p = taosArrayGet(pTagCond->pCond, i); - tfree(p->cond); - } - - taosArrayDestroy(pTagCond->pCond); - } - - for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) { - SJoinNode *node = pTagCond->joinInfo.joinTables[i]; - if (node == NULL) { - continue; - } - - if (node->tsJoin != NULL) { - taosArrayDestroy(node->tsJoin); - } - - if (node->tagJoin != NULL) { - taosArrayDestroy(node->tagJoin); - } - - tfree(node); - } - - memset(pTagCond, 0, sizeof(STagCond)); -} - -void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - pColInfo[i].functionId = pExpr->base.functionId; - - if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag)) { - SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - - int16_t index = pExpr->base.colInfo.colIndex; - pColInfo[i].type = (index != -1) ? pTagSchema[index].type : TSDB_DATA_TYPE_BINARY; - } else { - pColInfo[i].type = pSchema[pExpr->base.colInfo.colIndex].type; - } - } -} - -/* - * the following four kinds of SqlObj should not be freed - * 1. SqlObj for stream computing - * 2. main SqlObj - * 3. heartbeat SqlObj - * 4. SqlObj for subscription - * - * If res code is error and SqlObj does not belong to above types, it should be - * automatically freed for async query, ignoring that connection should be kept. - * - * If connection need to be recycled, the SqlObj also should be freed. - */ -bool tscShouldBeFreed(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) { - return false; - } - - STscObj* pTscObj = pSql->pTscObj; - if (pSql->pStream != NULL || pTscObj->hbrid == pSql->self || pSql->pSubscription != NULL) { - return false; - } - - // only the table meta and super table vgroup query will free resource automatically - int32_t command = pSql->cmd.command; - if (command == TSDB_SQL_META || command == TSDB_SQL_STABLEVGROUP) { - return true; - } - - return false; -} - -/** - * - * @param pCmd - * @param clauseIndex denote the index of the union sub clause, usually are 0, if no union query exists. - * @param tableIndex denote the table index for join query, where more than one table exists - * @return - */ -STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd* pCmd, int32_t tableIndex) { - assert(pCmd != NULL); - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - return tscGetMetaInfo(pQueryInfo, tableIndex); -} - -STableMetaInfo* tscGetMetaInfo(SQueryInfo* pQueryInfo, int32_t tableIndex) { - assert(pQueryInfo != NULL); - - if (pQueryInfo->pTableMetaInfo == NULL) { - assert(pQueryInfo->numOfTables == 0); - return NULL; - } - - assert(tableIndex >= 0 && tableIndex <= pQueryInfo->numOfTables && pQueryInfo->pTableMetaInfo != NULL); - - return pQueryInfo->pTableMetaInfo[tableIndex]; -} - -SQueryInfo* tscGetQueryInfoS(SSqlCmd* pCmd) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - int32_t ret = TSDB_CODE_SUCCESS; - - while ((pQueryInfo) == NULL) { - if ((ret = tscAddQueryInfo(pCmd)) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - pQueryInfo = tscGetQueryInfo(pCmd); - } - - return pQueryInfo; -} - -STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { - int32_t k = -1; - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - if (pQueryInfo->pTableMetaInfo[i]->pTableMeta->id.uid == uid) { - k = i; - break; - } - } - - if (index != NULL) { - *index = k; - } - - assert(k != -1); - return tscGetMetaInfo(pQueryInfo, k); -} - -// todo refactor -void tscInitQueryInfo(SQueryInfo* pQueryInfo) { - assert(pQueryInfo->fieldsInfo.internalField == NULL); - pQueryInfo->fieldsInfo.internalField = taosArrayInit(4, sizeof(SInternalField)); - - assert(pQueryInfo->exprList == NULL); - - pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; - pQueryInfo->limit.limit = -1; - pQueryInfo->limit.offset = 0; - - pQueryInfo->slimit.limit = -1; - pQueryInfo->slimit.offset = 0; - pQueryInfo->pUpstream = taosArrayInit(4, POINTER_BYTES); - pQueryInfo->window = TSWINDOW_INITIALIZER; - pQueryInfo->multigroupResult = true; -} - -int32_t tscAddQueryInfo(SSqlCmd* pCmd) { - assert(pCmd != NULL); - SQueryInfo* pQueryInfo = calloc(1, sizeof(SQueryInfo)); - - if (pQueryInfo == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - tscInitQueryInfo(pQueryInfo); - pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer - - if (pCmd->pQueryInfo == NULL) { - pCmd->pQueryInfo = pQueryInfo; - } else { - SQueryInfo* p = pCmd->pQueryInfo; - while(p->sibling != NULL) { - p = p->sibling; - } - - p->sibling = pQueryInfo; - } - - pCmd->active = pQueryInfo; - return TSDB_CODE_SUCCESS; -} - -static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { - tscTagCondRelease(&pQueryInfo->tagCond); - tscColCondRelease(&pQueryInfo->colCond); - tscFieldInfoClear(&pQueryInfo->fieldsInfo); - - tscExprDestroy(pQueryInfo->exprList); - pQueryInfo->exprList = NULL; - - if (pQueryInfo->exprList1 != NULL) { - tscExprDestroy(pQueryInfo->exprList1); - pQueryInfo->exprList1 = NULL; - } - - tscColumnListDestroy(pQueryInfo->colList); - pQueryInfo->colList = NULL; - - if (pQueryInfo->groupbyExpr.columnInfo != NULL) { - taosArrayDestroy(pQueryInfo->groupbyExpr.columnInfo); - pQueryInfo->groupbyExpr.columnInfo = NULL; - pQueryInfo->groupbyExpr.numOfGroupCols = 0; - } - - pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf); - pQueryInfo->fillType = 0; - - tfree(pQueryInfo->fillVal); - pQueryInfo->fillType = 0; - tfree(pQueryInfo->buf); - - taosArrayDestroy(pQueryInfo->pUpstream); - pQueryInfo->pUpstream = NULL; - pQueryInfo->bufLen = 0; -} - -void tscClearSubqueryInfo(SSqlCmd* pCmd) { - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - while (pQueryInfo != NULL) { - SQueryInfo* p = pQueryInfo->sibling; - freeQueryInfoImpl(pQueryInfo); - pQueryInfo = p; - } -} - -int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) { - assert(pQueryInfo != NULL && pSrc != NULL); - int32_t code = TSDB_CODE_SUCCESS; - - memcpy(&pQueryInfo->interval, &pSrc->interval, sizeof(pQueryInfo->interval)); - - pQueryInfo->command = pSrc->command; - pQueryInfo->type = pSrc->type; - pQueryInfo->window = pSrc->window; - pQueryInfo->limit = pSrc->limit; - pQueryInfo->slimit = pSrc->slimit; - pQueryInfo->order = pSrc->order; - pQueryInfo->vgroupLimit = pSrc->vgroupLimit; - pQueryInfo->tsBuf = NULL; - pQueryInfo->fillType = pSrc->fillType; - pQueryInfo->fillVal = NULL; - pQueryInfo->numOfFillVal = 0;; - pQueryInfo->clauseLimit = pSrc->clauseLimit; - pQueryInfo->prjOffset = pSrc->prjOffset; - pQueryInfo->numOfTables = 0; - pQueryInfo->window = pSrc->window; - pQueryInfo->sessionWindow = pSrc->sessionWindow; - pQueryInfo->pTableMetaInfo = NULL; - pQueryInfo->multigroupResult = pSrc->multigroupResult; - - pQueryInfo->bufLen = pSrc->bufLen; - pQueryInfo->orderProjectQuery = pSrc->orderProjectQuery; - pQueryInfo->arithmeticOnAgg = pSrc->arithmeticOnAgg; - pQueryInfo->buf = malloc(pSrc->bufLen); - if (pQueryInfo->buf == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (pSrc->bufLen > 0) { - memcpy(pQueryInfo->buf, pSrc->buf, pSrc->bufLen); - } - - pQueryInfo->groupbyExpr = pSrc->groupbyExpr; - if (pSrc->groupbyExpr.columnInfo != NULL) { - pQueryInfo->groupbyExpr.columnInfo = taosArrayDup(pSrc->groupbyExpr.columnInfo); - if (pQueryInfo->groupbyExpr.columnInfo == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - } - - if (tscTagCondCopy(&pQueryInfo->tagCond, &pSrc->tagCond) != 0) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (tscColCondCopy(&pQueryInfo->colCond, pSrc->colCond, 0, -1) != 0) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (pSrc->fillType != TSDB_FILL_NONE) { - pQueryInfo->fillVal = calloc(1, pSrc->fieldsInfo.numOfOutput * sizeof(int64_t)); - if (pQueryInfo->fillVal == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - pQueryInfo->numOfFillVal = pSrc->fieldsInfo.numOfOutput; - - memcpy(pQueryInfo->fillVal, pSrc->fillVal, pSrc->fieldsInfo.numOfOutput * sizeof(int64_t)); - } - - if (tscExprCopyAll(pQueryInfo->exprList, pSrc->exprList, true) != 0) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (pQueryInfo->arithmeticOnAgg) { - pQueryInfo->exprList1 = taosArrayInit(4, POINTER_BYTES); - if (tscExprCopyAll(pQueryInfo->exprList1, pSrc->exprList1, true) != 0) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - } - - tscColumnListCopyAll(pQueryInfo->colList, pSrc->colList); - tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pSrc->fieldsInfo, pQueryInfo->exprList); - - for(int32_t i = 0; i < pSrc->numOfTables; ++i) { - STableMetaInfo* p1 = tscGetMetaInfo((SQueryInfo*) pSrc, i); - - STableMeta* pMeta = tscTableMetaDup(p1->pTableMeta); - if (pMeta == NULL) { - // todo handle the error - } - - tscAddTableMetaInfo(pQueryInfo, &p1->name, pMeta, p1->vgroupList, p1->tagColList, p1->pVgroupTables); - } - - SArray *pUdfInfo = NULL; - if (pSrc->pUdfInfo) { - pUdfInfo = taosArrayDup(pSrc->pUdfInfo); - } - - pQueryInfo->pUdfInfo = pUdfInfo; - pQueryInfo->udfCopy = true; - - _error: - return code; -} - -void tscFreeVgroupTableInfo(SArray* pVgroupTables) { - if (pVgroupTables == NULL) { - return; - } - - size_t num = taosArrayGetSize(pVgroupTables); - for (size_t i = 0; i < num; i++) { - SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); -#if 0 - for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { - tfree(pInfo->vgInfo.epAddr[j].fqdn); - } -#endif - taosArrayDestroy(pInfo->itemList); - } - - taosArrayDestroy(pVgroupTables); -} - -void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) { - assert(pVgroupTable != NULL && index >= 0); - - size_t size = taosArrayGetSize(pVgroupTable); - assert(size > index); - - SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTable, index); -// for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { -// tfree(pInfo->vgInfo.epAddr[j].fqdn); -// } - - taosArrayDestroy(pInfo->itemList); - taosArrayRemove(pVgroupTable, index); -} - -void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) { - memset(info, 0, sizeof(SVgroupTableInfo)); - - info->vgInfo = pInfo->vgInfo; - -#if 0 - for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { - info->vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn); - } -#endif - - if (pInfo->itemList) { - info->itemList = taosArrayDup(pInfo->itemList); - } -} - -SArray* tscVgroupTableInfoDup(SArray* pVgroupTables) { - if (pVgroupTables == NULL) { - return NULL; - } - - size_t num = taosArrayGetSize(pVgroupTables); - SArray* pa = taosArrayInit(num, sizeof(SVgroupTableInfo)); - - SVgroupTableInfo info; - for (size_t i = 0; i < num; i++) { - SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); - tscVgroupTableCopy(&info, pInfo); - - taosArrayPush(pa, &info); - } - - return pa; -} - -void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta, uint64_t id) { - for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - if (removeMeta) { - tscRemoveCachedTableMeta(pTableMetaInfo, id); - } - - tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); - tscClearTableMetaInfo(pTableMetaInfo); - } - - tfree(pQueryInfo->pTableMetaInfo); -} - -STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableMeta* pTableMeta, - SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables) { - void* tmp = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); - if (tmp == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - pQueryInfo->pTableMetaInfo = tmp; - STableMetaInfo* pTableMetaInfo = calloc(1, sizeof(STableMetaInfo)); - - if (pTableMetaInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = pTableMetaInfo; - - if (name != NULL) { - tNameAssign(&pTableMetaInfo->name, name); - } - - pTableMetaInfo->pTableMeta = pTableMeta; - pTableMetaInfo->tableMetaSize = (pTableMetaInfo->pTableMeta == NULL)? 0:tscGetTableMetaSize(pTableMeta); - - pTableMetaInfo->tableMetaCapacity = (size_t)(pTableMetaInfo->tableMetaSize); - - if (vgroupList != NULL) { - pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList); - } - - // TODO handle malloc failure - pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); - if (pTableMetaInfo->tagColList == NULL) { - return NULL; - } - - if (pTagCols != NULL && pTableMetaInfo->pTableMeta != NULL) { - tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, pTableMetaInfo->pTableMeta->id.uid); - } - - pTableMetaInfo->pVgroupTables = tscVgroupTableInfoDup(pVgroupTables); - - pQueryInfo->numOfTables += 1; - return pTableMetaInfo; -} - -STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) { - return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL, NULL); -} - -void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo) { - if (pTableMetaInfo == NULL) { - return; - } - - tfree(pTableMetaInfo->pTableMeta); - pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); - - tscColumnListDestroy(pTableMetaInfo->tagColList); - pTableMetaInfo->tagColList = NULL; - - free(pTableMetaInfo); -} - -void tscResetForNextRetrieve(SSqlRes* pRes) { - if (pRes == NULL) { - return; - } - - pRes->row = 0; - pRes->numOfRows = 0; -} - -void tscInitResForMerge(SSqlRes* pRes) { - pRes->qId = 1; // hack to pass the safety check in fetch_row function - pRes->rspType = 0; // used as a flag to denote if taos_retrieved() has been called yet - tscResetForNextRetrieve(pRes); - - assert(pRes->pMerger != NULL); - pRes->data = pRes->pMerger->buf; -} - -void registerSqlObj(SSqlObj* pSql) { - taosAcquireRef(tscRefId, pSql->pTscObj->rid); - pSql->self = taosAddRef(tscObjRef, pSql); - - int32_t num = atomic_add_fetch_32(&pSql->pTscObj->numOfObj, 1); - int32_t total = atomic_add_fetch_32(&tscNumOfObj, 1); - tscDebug("0x%"PRIx64" new SqlObj from %p, total in tscObj:%d, total:%d", pSql->self, pSql->pTscObj, num, total); -} - -SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, int32_t cmd) { - SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); - if (pNew == NULL) { - tscError("0x%"PRIx64" new subquery failed, tableIndex:%d", pSql->self, 0); - return NULL; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - - SSqlCmd* pCmd = &pNew->cmd; - pCmd->command = cmd; - tsem_init(&pNew->rspSem, 0 ,0); - - if (tscAddQueryInfo(pCmd) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pNew); - return NULL; - } - - pNew->fp = fp; - pNew->fetchFp = fp; - pNew->param = param; - pNew->sqlstr = NULL; - pNew->maxRetry = TSDB_MAX_REPLICA; - - SQueryInfo* pQueryInfo = tscGetQueryInfoS(pCmd); - STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - - tscAddTableMetaInfo(pQueryInfo, &pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); - registerSqlObj(pNew); - - return pNew; -} - -static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pNewQueryInfo, int64_t uid) { - int32_t numOfOutput = (int32_t)tscNumOfExprs(pNewQueryInfo); - if (numOfOutput == 0) { - return; - } - - // set the field info in pNewQueryInfo object according to sqlExpr information - for (int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExpr = tscExprGet(pNewQueryInfo, i); - - TAOS_FIELD f = tscCreateField((int8_t) pExpr->base.resType, pExpr->base.aliasName, pExpr->base.resBytes); - SInternalField* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, &f); - pInfo1->pExpr = pExpr; - } - - // update the pSqlExpr pointer in SInternalField according the field name - // make sure the pSqlExpr point to the correct SqlExpr in pNewQueryInfo, not SqlExpr in pQueryInfo - for (int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) { - TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f); - - bool matched = false; - for (int32_t k1 = 0; k1 < numOfOutput; ++k1) { - SExprInfo* pExpr1 = tscExprGet(pNewQueryInfo, k1); - - if (strcmp(field->name, pExpr1->base.aliasName) == 0) { // establish link according to the result field name - SInternalField* pInfo = tscFieldInfoGetInternalField(&pNewQueryInfo->fieldsInfo, f); - pInfo->pExpr = pExpr1; - - matched = true; - break; - } - } - - assert(matched); - (void)matched; - } - - tscFieldInfoUpdateOffset(pNewQueryInfo); -} - -SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t fp, void* param, int32_t cmd, SSqlObj* pPrevSql) { - SSqlCmd* pCmd = &pSql->cmd; - - SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); - if (pNew == NULL) { - tscError("0x%"PRIx64" new subquery failed, tableIndex:%d", pSql->self, tableIndex); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[tableIndex]; - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->sqlstr = strdup(pSql->sqlstr); - tsem_init(&pNew->rspSem, 0, 0); - - SSqlCmd* pnCmd = &pNew->cmd; - memcpy(pnCmd, pCmd, sizeof(SSqlCmd)); - - pnCmd->command = cmd; - pnCmd->payload = NULL; - pnCmd->allocSize = 0; - pnCmd->pTableMetaMap = NULL; - - pnCmd->pQueryInfo = NULL; - pnCmd->insertParam.pDataBlocks = NULL; - - pnCmd->insertParam.numOfTables = 0; - pnCmd->insertParam.pTableNameList = NULL; - pnCmd->insertParam.pTableBlockHashList = NULL; - pnCmd->insertParam.objectId = pNew->self; - - memset(&pnCmd->insertParam.tagData, 0, sizeof(STagData)); - - if (tscAddQueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pnCmd); - - if (pQueryInfo->pUdfInfo) { - pNewQueryInfo->pUdfInfo = taosArrayDup(pQueryInfo->pUdfInfo); - pNewQueryInfo->udfCopy = true; - } - - pNewQueryInfo->command = pQueryInfo->command; - pnCmd->active = pNewQueryInfo; - - memcpy(&pNewQueryInfo->interval, &pQueryInfo->interval, sizeof(pNewQueryInfo->interval)); - pNewQueryInfo->type = pQueryInfo->type; - pNewQueryInfo->window = pQueryInfo->window; - pNewQueryInfo->limit = pQueryInfo->limit; - pNewQueryInfo->slimit = pQueryInfo->slimit; - pNewQueryInfo->order = pQueryInfo->order; - pNewQueryInfo->tsBuf = NULL; - pNewQueryInfo->fillType = pQueryInfo->fillType; - pNewQueryInfo->fillVal = NULL; - pNewQueryInfo->clauseLimit = pQueryInfo->clauseLimit; - pNewQueryInfo->prjOffset = pQueryInfo->prjOffset; - pNewQueryInfo->numOfFillVal = 0; - pNewQueryInfo->numOfTables = 0; - pNewQueryInfo->pTableMetaInfo = NULL; - pNewQueryInfo->bufLen = pQueryInfo->bufLen; - pNewQueryInfo->vgroupLimit = pQueryInfo->vgroupLimit; - pNewQueryInfo->distinct = pQueryInfo->distinct; - pNewQueryInfo->multigroupResult = pQueryInfo->multigroupResult; - - pNewQueryInfo->buf = malloc(pQueryInfo->bufLen); - if (pNewQueryInfo->buf == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (pQueryInfo->bufLen > 0) { - memcpy(pNewQueryInfo->buf, pQueryInfo->buf, pQueryInfo->bufLen); - } - - pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; - if (pQueryInfo->groupbyExpr.columnInfo != NULL) { - pNewQueryInfo->groupbyExpr.columnInfo = taosArrayDup(pQueryInfo->groupbyExpr.columnInfo); - if (pNewQueryInfo->groupbyExpr.columnInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - } - - if (tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (tscColCondCopy(&pNewQueryInfo->colCond, pQueryInfo->colCond, pTableMetaInfo->pTableMeta->id.uid, tableIndex) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - if (pQueryInfo->fillType != TSDB_FILL_NONE) { - //just make memory memory sanitizer happy - //refactor later - pNewQueryInfo->fillVal = calloc(1, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); - if (pNewQueryInfo->fillVal == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - pNewQueryInfo->numOfFillVal = pQueryInfo->fieldsInfo.numOfOutput; - memcpy(pNewQueryInfo->fillVal, pQueryInfo->fillVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); - } - - if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql->self, tableIndex, pTableMetaInfo->vgroupIndex); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; - tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, uid); - - // set the correct query type - if (pPrevSql != NULL) { - SQueryInfo* pPrevQueryInfo = tscGetQueryInfo(&pPrevSql->cmd); - pNewQueryInfo->type = pPrevQueryInfo->type; - } else { - TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY);// it must be the subquery - } - - if (tscExprCopy(pNewQueryInfo->exprList, pQueryInfo->exprList, uid, true) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - doSetSqlExprAndResultFieldInfo(pNewQueryInfo, uid); - - pNew->fp = fp; - pNew->fetchFp = fp; - pNew->param = param; - pNew->maxRetry = TSDB_MAX_REPLICA; - - STableMetaInfo* pFinalInfo = NULL; - - if (pPrevSql == NULL) { - STableMeta* pTableMeta = tscTableMetaDup(pTableMetaInfo->pTableMeta); - assert(pTableMeta != NULL); - - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, &pTableMetaInfo->name, pTableMeta, pTableMetaInfo->vgroupList, - pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); - } else { // transfer the ownership of pTableMeta to the newly create sql object. - STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, 0); - if (pPrevInfo->pTableMeta && pPrevInfo->pTableMeta->tableType < 0) { - terrno = TSDB_CODE_TSC_APP_ERROR; - goto _error; - } - - STableMeta* pPrevTableMeta = tscTableMetaDup(pPrevInfo->pTableMeta); - SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, &pTableMetaInfo->name, pPrevTableMeta, pVgroupsInfo, - pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); - } - - // this case cannot be happened - if (pFinalInfo->pTableMeta == NULL) { - tscError("0x%"PRIx64" new subquery failed since no tableMeta, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); - - if (pPrevSql != NULL) { // pass the previous error to client - assert(pPrevSql->res.code != TSDB_CODE_SUCCESS); - terrno = pPrevSql->res.code; - } else { - terrno = TSDB_CODE_TSC_APP_ERROR; - } - - goto _error; - } - - assert(pNewQueryInfo->numOfTables == 1); - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - assert(pFinalInfo->vgroupList != NULL); - } - - registerSqlObj(pNew); - - if (cmd == TSDB_SQL_SELECT) { - size_t size = taosArrayGetSize(pNewQueryInfo->colList); - - tscDebug("0x%"PRIx64" new subquery:0x%"PRIx64", tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu "," - "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64, - pSql->self, pNew->self, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscNumOfExprs(pNewQueryInfo), - size, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pFinalInfo->name), pNewQueryInfo->window.skey, - pNewQueryInfo->window.ekey, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit); - - tscPrintSelNodeList(pNew, 0); - } else { - tscDebug("0x%"PRIx64" new sub insertion: %p, vnodeIdx:%d", pSql->self, pNew, pTableMetaInfo->vgroupIndex); - } - - return pNew; - -_error: - tscFreeSqlObj(pNew); - return NULL; -} - -void doExecuteQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { - uint16_t type = pQueryInfo->type; - if (QUERY_IS_JOIN_QUERY(type) && !TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_SUBQUERY)) { - tscHandleMasterJoinQuery(pSql); - } else if (tscMultiRoundQuery(pQueryInfo, 0) && pQueryInfo->round == 0) { - tscHandleFirstRoundStableQuery(pSql); // todo lock? - } else if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { // super table query - tscLockByThread(&pSql->squeryLock); - tscHandleMasterSTableQuery(pSql); - tscUnlockByThread(&pSql->squeryLock); - } else if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT)) { - if (TSDB_QUERY_HAS_TYPE(pSql->cmd.insertParam.insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { - tscImportDataFromFile(pSql); - } else { - tscHandleMultivnodeInsert(pSql); - } - } else if (pSql->cmd.command > TSDB_SQL_LOCAL) { - tscProcessLocalCmd(pSql); - } else { // send request to server directly - tscBuildAndSendRequest(pSql, pQueryInfo); - } -} - -void doRetrieveSubqueryData(SSchedMsg *pMsg) { - SSqlObj* pSql = (SSqlObj*) pMsg->ahandle; - if (pSql == NULL || pSql->signature != pSql) { - tscDebug("%p SqlObj is freed, not add into queue async res", pMsg->ahandle); - return; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - handleDownstreamOperator(pSql->pSubs, pSql->subState.numOfSub, pQueryInfo, pSql); - - pSql->res.qId = -1; - if (pSql->res.code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); - } else { - tscAsyncResultOnError(pSql); - } -} - -// NOTE: the blocking query can not be executed in the rpc message handler thread -static void tscSubqueryRetrieveCallback(void* param, TAOS_RES* tres, int code) { - // handle the pDownStream process - SRetrieveSupport* ps = param; - SSqlObj* pParentSql = ps->pParentSql; - SSqlObj* pSql = tres; - - int32_t index = ps->subqueryIndex; - bool ret = subAndCheckDone(pSql, pParentSql, index); - - // TODO refactor - tfree(ps); - pSql->param = NULL; - - if (!ret) { - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" orderOfSub:%d completed, not all subquery finished", pParentSql->self, pSql->self, index); - return; - } - - pParentSql->cmd.active = pParentSql->cmd.pQueryInfo; - pParentSql->res.qId = -1; - if (pSql->res.code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pParentSql->param, pParentSql, pParentSql->res.numOfRows); - } else { - pParentSql->res.code = pSql->res.code; - tscAsyncResultOnError(pParentSql); - } -} - -static void tscSubqueryCompleteCallback(void* param, TAOS_RES* tres, int code) { - SSqlObj* pSql = tres; - SRetrieveSupport* ps = param; - - if (pSql->res.code != TSDB_CODE_SUCCESS) { - SSqlObj* pParentSql = ps->pParentSql; - - int32_t index = ps->subqueryIndex; - bool ret = subAndCheckDone(pSql, pParentSql, index); - - tscFreeRetrieveSup(pSql); - - if (!ret) { - tscDebug("0x%"PRIx64" sub:0x%"PRIx64" orderOfSub:%d completed, not all subquery finished", pParentSql->self, pSql->self, index); - return; - } - - // todo refactor - tscDebug("0x%"PRIx64" all subquery response received, retry", pParentSql->self); - if (code && !((code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) && pParentSql->retry < pParentSql->maxRetry)) { - pParentSql->res.code = code; - - tscAsyncResultOnError(pParentSql); - return; - } - - tscFreeSubobj(pParentSql); - tfree(pParentSql->pSubs); - - pParentSql->res.code = TSDB_CODE_SUCCESS; - pParentSql->retry++; - - tscDebug("0x%"PRIx64" retry parse sql and send query, prev error: %s, retry:%d", pParentSql->self, - tstrerror(code), pParentSql->retry); - - - tscResetSqlCmd(&pParentSql->cmd, true, pParentSql->self); - - code = tsParseSql(pParentSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - return; - } - - if (code != TSDB_CODE_SUCCESS) { - pParentSql->res.code = code; - tscAsyncResultOnError(pParentSql); - return; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pParentSql->cmd); - executeQuery(pParentSql, pQueryInfo); - return; - } - - taos_fetch_rows_a(tres, tscSubqueryRetrieveCallback, param); -} - -int32_t doInitSubState(SSqlObj* pSql, int32_t numOfSubqueries) { - assert(pSql->subState.numOfSub == 0 && pSql->pSubs == NULL && pSql->subState.states == NULL); - pSql->subState.numOfSub = numOfSubqueries; - - pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES); - pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(int8_t)); - - int32_t code = pthread_mutex_init(&pSql->subState.mutex, NULL); - if (pSql->pSubs == NULL || pSql->subState.states == NULL || code != 0) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - return TSDB_CODE_SUCCESS; -} - -// do execute the query according to the query execution plan -void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t numOfInit = 0; - - if (pSql->cmd.command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - (*pSql->fp)(pSql->param, pSql, 0); - return; - } - - if (pSql->cmd.command == TSDB_SQL_SELECT) { - tscAddIntoSqlList(pSql); - } - - if (taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // nest query. do execute it firstly - code = doInitSubState(pSql, (int32_t) taosArrayGetSize(pQueryInfo->pUpstream)); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SQueryInfo* pSub = taosArrayGetP(pQueryInfo->pUpstream, i); - - pSql->cmd.active = pSub; - pSql->cmd.command = TSDB_SQL_SELECT; - - SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); - if (pNew == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->sqlstr = strdup(pSql->sqlstr); - pNew->fp = tscSubqueryCompleteCallback; - pNew->fetchFp = tscSubqueryCompleteCallback; - pNew->maxRetry = pSql->maxRetry; - - pNew->cmd.resColumnId = TSDB_RES_COL_ID; - - tsem_init(&pNew->rspSem, 0, 0); - - SRetrieveSupport* ps = calloc(1, sizeof(SRetrieveSupport)); // todo use object id - if (ps == NULL) { - tscFreeSqlObj(pNew); - goto _error; - } - - ps->pParentSql = pSql; - ps->subqueryIndex = i; - - pNew->param = ps; - pSql->pSubs[i] = pNew; - - SSqlCmd* pCmd = &pNew->cmd; - pCmd->command = TSDB_SQL_SELECT; - if ((code = tscAddQueryInfo(pCmd)) != TSDB_CODE_SUCCESS) { - goto _error; - } - - SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pCmd); - tscQueryInfoCopy(pNewQueryInfo, pSub); - - TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY); - numOfInit++; - } - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - SSqlObj* psub = pSql->pSubs[i]; - registerSqlObj(psub); - - // create sub query to handle the sub query. - SQueryInfo* pq = tscGetQueryInfo(&psub->cmd); - executeQuery(psub, pq); - } - - return; - } - - pSql->cmd.active = pQueryInfo; - doExecuteQuery(pSql, pQueryInfo); - return; - - _error: - for(int32_t i = 0; i < numOfInit; ++i) { - SSqlObj* p = pSql->pSubs[i]; - tscFreeSqlObj(p); - } - - pSql->res.code = code; - pSql->subState.numOfSub = 0; // not initialized sub query object will not be freed - tfree(pSql->subState.states); - tfree(pSql->pSubs); - tscAsyncResultOnError(pSql); -} - -int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) { - int32_t i = 0; - while (i < TSDB_MAX_JOIN_TABLE_NUM) { - SJoinNode* node = pTagCond->joinInfo.joinTables[i]; - if (node && node->uid == uid) { - return node->tagColId; - } - - i++; - } - - assert(0); - return -1; -} - - -int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId) { - int32_t numOfTags = tscGetNumOfTags(pTableMeta); - - SSchema* pSchema = tscGetTableTagSchema(pTableMeta); - for(int32_t i = 0; i < numOfTags; ++i) { - if (pSchema[i].colId == colId) { - return i; - } - } - - // can not reach here - assert(0); - return INT16_MIN; -} - -bool tscIsUpdateQuery(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - SSqlCmd* pCmd = &pSql->cmd; - return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) || TSDB_SQL_RESET_CACHE == pCmd->command || TSDB_SQL_USE_DB == pCmd->command); -} - -char* tscGetSqlStr(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) { - return NULL; - } - - return pSql->sqlstr; -} - -bool tscIsQueryWithLimit(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) { - return false; - } - - SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo* pqi = tscGetQueryInfo(pCmd); - while(pqi != NULL) { - if (pqi->limit.limit > 0) { - return true; - } - - pqi = pqi->sibling; - } - - return false; -} - - -int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql) { - const char* msgFormat1 = "syntax error near \'%s\'"; - const char* msgFormat2 = "syntax error near \'%s\' (%s)"; - const char* msgFormat3 = "%s"; - - const char* prefix = "syntax error"; - const int32_t BACKWARD_CHAR_STEP = 0; - - if (sql == NULL) { - assert(additionalInfo != NULL); - sprintf(msg, msgFormat1, additionalInfo); - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - - char buf[64] = {0}; // only extract part of sql string - strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); - - if (additionalInfo != NULL) { - sprintf(msg, msgFormat2, buf, additionalInfo); - } else { - const char* msgFormat = (0 == strncmp(sql, prefix, strlen(prefix))) ? msgFormat3 : msgFormat1; - sprintf(msg, msgFormat, buf); - } - - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; -} - -int32_t tscInvalidOperationMsg(char* msg, const char* additionalInfo, const char* sql) { - const char* msgFormat1 = "invalid operation: %s"; - const char* msgFormat2 = "invalid operation: \'%s\' (%s)"; - const char* msgFormat3 = "invalid operation: \'%s\'"; - - const int32_t BACKWARD_CHAR_STEP = 0; - - if (sql == NULL) { - assert(additionalInfo != NULL); - sprintf(msg, msgFormat1, additionalInfo); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - char buf[64] = {0}; // only extract part of sql string - strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); - - if (additionalInfo != NULL) { - sprintf(msg, msgFormat2, buf, additionalInfo); - } else { - sprintf(msg, msgFormat3, buf); // no additional information for invalid sql error - } - - return TSDB_CODE_TSC_INVALID_OPERATION; -} - -int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql) { - const char* msgFormat1 = "%s:%s"; - const char* msgFormat2 = "%s:\'%s\' (%s)"; - const char* msgFormat3 = "%s:\'%s\'"; - - const int32_t BACKWARD_CHAR_STEP = 0; - - if (sql == NULL) { - assert(errMsg != NULL); - sprintf(dstBuffer, msgFormat1, tstrerror(code), errMsg); - return code; - } - - char buf[64] = {0}; // only extract part of sql string - strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); - - if (errMsg != NULL) { - sprintf(dstBuffer, msgFormat2, tstrerror(code), buf, errMsg); - } else { - sprintf(dstBuffer, msgFormat3, tstrerror(code), buf); // no additional information for invalid sql error - } - - return code; -} - -bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) { - assert(pQueryInfo != NULL && pQueryInfo->clauseLimit != 0); - return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit); -} - -char* tscGetErrorMsgPayload(SSqlCmd* pCmd) { return pCmd->payload; } - -/** - * If current vnode query does not return results anymore (pRes->numOfRows == 0), try the next vnode if exists, - * while multi-vnode super table projection query and the result does not reach the limitation. - */ -bool hasMoreVnodesToTry(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - if (pCmd->command != TSDB_SQL_FETCH) { - return false; - } - - assert(pRes->completed); - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - // for normal table, no need to try any more if results are all retrieved from one vnode - if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) || (pTableMetaInfo->vgroupList == NULL)) { - return false; - } - - int32_t numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; - if (pTableMetaInfo->pVgroupTables != NULL) { - numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - } - - return tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && - (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vgroupIndex < numOfVgroups - 1); -} - -bool hasMoreClauseToTry(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - return pCmd->active->sibling != NULL; -} - -void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - /* - * no result returned from the current virtual node anymore, try the next vnode if exists - * if case of: multi-vnode super table projection query - */ - assert(pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && !tscHasReachLimitation(pQueryInfo, pRes)); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups; - if (++pTableMetaInfo->vgroupIndex < totalVgroups) { - tscDebug("0x%"PRIx64" results from vgroup index:%d completed, try next:%d. total vgroups:%d. current numOfRes:%" PRId64, pSql->self, - pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfClauseTotal); - - /* - * update the limit and offset value for the query on the next vnode, - * according to current retrieval results - * - * NOTE: - * if the pRes->offset is larger than 0, the start returned position has not reached yet. - * Therefore, the pRes->numOfRows, as well as pRes->numOfClauseTotal, must be 0. - * The pRes->offset value will be updated by virtual node, during query execution. - */ - if (pQueryInfo->clauseLimit >= 0) { - pQueryInfo->limit.limit = pQueryInfo->clauseLimit - pRes->numOfClauseTotal; - } - - pQueryInfo->limit.offset = pRes->offset; - assert((pRes->offset >= 0 && pRes->numOfRows == 0) || (pRes->offset == 0 && pRes->numOfRows >= 0)); - - tscDebug("0x%"PRIx64" new query to next vgroup, index:%d, limit:%" PRId64 ", offset:%" PRId64 ", glimit:%" PRId64, - pSql->self, pTableMetaInfo->vgroupIndex, pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->clauseLimit); - - /* - * For project query with super table join, the numOfSub is equalled to the number of all subqueries. - * Therefore, we need to reset the value of numOfSubs to be 0. - * - * For super table join with projection query, if anyone of the subquery is exhausted, the query completed. - */ - pSql->subState.numOfSub = 0; - pCmd->command = TSDB_SQL_SELECT; - - tscResetForNextRetrieve(pRes); - - // set the callback function - pSql->fp = fp; - tscBuildAndSendRequest(pSql, NULL); - } else { - tscDebug("0x%"PRIx64" try all %d vnodes, query complete. current numOfRes:%" PRId64, pSql->self, totalVgroups, pRes->numOfClauseTotal); - } -} - -void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - pSql->cmd.command = pQueryInfo->command; - - //backup the total number of result first - int64_t num = pRes->numOfTotal + pRes->numOfClauseTotal; - - // DON't free final since it may be recoreded and used later in APP - TAOS_FIELD* finalBk = pRes->final; - pRes->final = NULL; - tscFreeSqlResult(pSql); - - pRes->final = finalBk; - pRes->numOfTotal = num; - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - taos_free_result(pSql->pSubs[i]); - } - - tfree(pSql->pSubs); - tfree(pSql->subState.states); - pSql->subState.numOfSub = 0; - pthread_mutex_destroy(&pSql->subState.mutex); - - pSql->fp = fp; - - tscDebug("0x%"PRIx64" try data in the next subclause", pSql->self); - if (pCmd->command > TSDB_SQL_LOCAL) { - tscProcessLocalCmd(pSql); - } else { - executeQuery(pSql, pQueryInfo); - } -} - -void* malloc_throw(size_t size) { - void* p = malloc(size); - if (p == NULL) { - THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); - } - return p; -} - -void* calloc_throw(size_t nmemb, size_t size) { - void* p = calloc(nmemb, size); - if (p == NULL) { - THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); - } - return p; -} - -char* strdup_throw(const char* str) { - char* p = strdup(str); - if (p == NULL) { - THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); - } - return p; -} - -int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corMgmtEpSet) { - corMgmtEpSet->version = 0; - // init mgmt ip set - SRpcEpSet *mgmtEpSet = &(corMgmtEpSet->epSet); - mgmtEpSet->numOfEps = 0; - mgmtEpSet->inUse = 0; - - if (first && first[0] != 0) { - if (strlen(first) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - taosGetFqdnPortFromEp(first, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps])); - mgmtEpSet->numOfEps++; - } - - if (second && second[0] != 0) { - if (strlen(second) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - taosGetFqdnPortFromEp(second, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps])); - mgmtEpSet->numOfEps++; - } - - if (mgmtEpSet->numOfEps == 0) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - - return 0; -} - -bool tscSetSqlOwner(SSqlObj* pSql) { - SSqlRes* pRes = &pSql->res; - - // set the sql object owner - int64_t threadId = taosGetSelfPthreadId(); - if (atomic_val_compare_exchange_64(&pSql->owner, 0, threadId) != 0) { - pRes->code = TSDB_CODE_QRY_IN_EXEC; - return false; - } - - return true; -} - -void tscClearSqlOwner(SSqlObj* pSql) { - atomic_store_64(&pSql->owner, 0); -} - -SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) { - if (vgroupList == NULL) { - return NULL; - } - - size_t size = sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * vgroupList->numOfVgroups; - SVgroupsInfo* pNew = malloc(size); - if (pNew == NULL) { - return NULL; - } - - pNew->numOfVgroups = vgroupList->numOfVgroups; - - for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) { - SVgroupMsg* pNewVInfo = &pNew->vgroups[i]; - - SVgroupMsg* pvInfo = &vgroupList->vgroups[i]; - pNewVInfo->vgId = pvInfo->vgId; - pNewVInfo->numOfEps = pvInfo->numOfEps; - - for(int32_t j = 0; j < pvInfo->numOfEps; ++j) { - pNewVInfo->epAddr[j].port = pvInfo->epAddr[j].port; - tstrncpy(pNewVInfo->epAddr[j].fqdn, pvInfo->epAddr[j].fqdn, TSDB_FQDN_LEN); - } - } - - return pNew; -} - -void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) { - if (vgroupList == NULL) { - return NULL; - } - -#if 0 - for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) { - SVgroupMsg* pVgroupInfo = &vgroupList->vgroups[i]; - - for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) { - tfree(pVgroupInfo->epAddr[j].fqdn); - } - - for(int32_t j = pVgroupInfo->numOfEps; j < TSDB_MAX_REPLICA; j++) { - assert( pVgroupInfo->epAddr[j].fqdn == NULL ); - } - } - -#endif - tfree(vgroupList); - return NULL; -} -# if 0 -void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src) { - dst->vgId = src->vgId; - dst->numOfEps = src->numOfEps; - for(int32_t i = 0; i < dst->numOfEps; ++i) { - tfree(dst->epAddr[i].fqdn); - dst->epAddr[i].port = src->epAddr[i].port; - assert(dst->epAddr[i].fqdn == NULL); - - dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn); - } -} - -#endif - -char* serializeTagData(STagData* pTagData, char* pMsg) { - int32_t n = (int32_t) strlen(pTagData->name); - *(int32_t*) pMsg = htonl(n); - pMsg += sizeof(n); - - memcpy(pMsg, pTagData->name, n); - pMsg += n; - - *(int32_t*)pMsg = htonl(pTagData->dataLen); - pMsg += sizeof(int32_t); - - memcpy(pMsg, pTagData->data, pTagData->dataLen); - pMsg += pTagData->dataLen; - - return pMsg; -} - -int32_t copyTagData(STagData* dst, const STagData* src) { - dst->dataLen = src->dataLen; - tstrncpy(dst->name, src->name, tListLen(dst->name)); - - if (dst->dataLen > 0) { - dst->data = malloc(dst->dataLen); - if (dst->data == NULL) { - return -1; - } - - memcpy(dst->data, src->data, dst->dataLen); - } - - return 0; -} - -STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { - assert(pChild != NULL); - int32_t total = pChild->numOfColumns + pChild->numOfTags; - - STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total); - pTableMeta->tableType = TSDB_SUPER_TABLE; - pTableMeta->tableInfo.numOfTags = pChild->numOfTags; - pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns; - pTableMeta->tableInfo.precision = pChild->precision; - - pTableMeta->id.tid = 0; - pTableMeta->id.uid = pChild->suid; - pTableMeta->tversion = pChild->tversion; - pTableMeta->sversion = pChild->sversion; - - memcpy(pTableMeta->schema, pChild->schema, sizeof(SSchema) * total); - - int32_t num = pTableMeta->tableInfo.numOfColumns; - for(int32_t i = 0; i < num; ++i) { - pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; - } - - return pTableMeta; -} - -uint32_t tscGetTableMetaSize(STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - - int32_t totalCols = 0; - if (pTableMeta->tableInfo.numOfColumns >= 0) { - totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; - } - - return sizeof(STableMeta) + totalCols * sizeof(SSchema); -} - -CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - - CChildTableMeta* cMeta = calloc(1, sizeof(CChildTableMeta)); - - cMeta->tableType = TSDB_CHILD_TABLE; - cMeta->vgId = pTableMeta->vgId; - cMeta->id = pTableMeta->id; - cMeta->suid = pTableMeta->suid; - tstrncpy(cMeta->sTableName, pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); - - return cMeta; -} - -int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity, STableMeta**ppSTable) { - assert(*ppChild != NULL); - STableMeta* p = *ppSTable; - STableMeta* pChild = *ppChild; - - size_t sz = (p != NULL) ? tscGetTableMetaSize(p) : 0; //ppSTableBuf actually capacity may larger than sz, dont care - if (p != NULL && sz != 0) { - memset((char *)p, 0, sz); - } - - if (NULL == taosHashGetCloneExt(tscTableMetaMap, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, (void **)&p, &sz)) { - tfree(p); - } else { - *ppSTable = p; - } - - // tableMeta exists, build child table meta according to the super table meta - // the uid need to be checked in addition to the general name of the super table. - if (p && p->id.uid > 0 && pChild->suid == p->id.uid) { - int32_t totalBytes = (p->tableInfo.numOfColumns + p->tableInfo.numOfTags) * sizeof(SSchema); - int32_t tableMetaSize = sizeof(STableMeta) + totalBytes; - if (*tableMetaCapacity < tableMetaSize) { - STableMeta* pChild1 = realloc(pChild, tableMetaSize); - if(pChild1 == NULL) return -1; - pChild = pChild1; - *tableMetaCapacity = (size_t)tableMetaSize; - } - - pChild->sversion = p->sversion; - pChild->tversion = p->tversion; - memcpy(&pChild->tableInfo, &p->tableInfo, sizeof(STableComInfo)); - memcpy(pChild->schema, p->schema, totalBytes); - - *ppChild = pChild; - return TSDB_CODE_SUCCESS; - } else { // super table has been removed, current tableMeta is also expired. remove it here - taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - return -1; - } -} - -uint32_t tscGetTableMetaMaxSize() { - return sizeof(STableMeta) + TSDB_MAX_COLUMNS * sizeof(SSchema); -} - -STableMeta* tscTableMetaDup(STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - size_t size = tscGetTableMetaSize(pTableMeta); - - STableMeta* p = malloc(size); - memcpy(p, pTableMeta, size); - return p; -} - -SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo) { - assert(pVgroupsInfo != NULL); - - size_t size = sizeof(SVgroupMsg) * pVgroupsInfo->numOfVgroups + sizeof(SVgroupsInfo); - SVgroupsInfo* pInfo = calloc(1, size); - pInfo->numOfVgroups = pVgroupsInfo->numOfVgroups; - for (int32_t m = 0; m < pVgroupsInfo->numOfVgroups; ++m) { - memcpy(&pInfo->vgroups[m], &pVgroupsInfo->vgroups[m], sizeof(SVgroupMsg)); -// tscSVgroupInfoCopy(&pInfo->vgroups[m], &pVgroupsInfo->vgroups[m]); - } - return pInfo; -} - -int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num) { - if (!pQueryInfo->arithmeticOnAgg) { - return TSDB_CODE_SUCCESS; - } - - *num = tscNumOfFields(pQueryInfo); - *pExpr = calloc(*(num), POINTER_BYTES); - if ((*pExpr) == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < (*num); ++i) { - SInternalField* pField = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i); - SExprInfo* pSource = pField->pExpr; - - SExprInfo* px = calloc(1, sizeof(SExprInfo)); - (*pExpr)[i] = px; - - SSqlExpr *pse = &px->base; - pse->uid = pTableMetaInfo->pTableMeta->id.uid; - pse->resColId = pSource->base.resColId; - strncpy(pse->aliasName, pSource->base.aliasName, tListLen(pse->aliasName)); - strncpy(pse->token, pSource->base.token, tListLen(pse->token)); - - if (pSource->base.functionId != TSDB_FUNC_ARITHM) { // this should be switched to projection query - pse->numOfParams = 0; // no params for projection query - pse->functionId = TSDB_FUNC_PRJ; - pse->colInfo.colId = pSource->base.resColId; - - int32_t numOfOutput = (int32_t) taosArrayGetSize(pQueryInfo->exprList); - for (int32_t j = 0; j < numOfOutput; ++j) { - SExprInfo* p = taosArrayGetP(pQueryInfo->exprList, j); - if (p->base.resColId == pse->colInfo.colId) { - pse->colInfo.colIndex = j; - break; - } - } - - pse->colInfo.flag = TSDB_COL_NORMAL; - pse->resType = pSource->base.resType; - pse->resBytes = pSource->base.resBytes; - strncpy(pse->colInfo.name, pSource->base.aliasName, tListLen(pse->colInfo.name)); - - // TODO restore refactor - int32_t functionId = pSource->base.functionId; - if (pSource->base.functionId == TSDB_FUNC_FIRST_DST) { - functionId = TSDB_FUNC_FIRST; - } else if (pSource->base.functionId == TSDB_FUNC_LAST_DST) { - functionId = TSDB_FUNC_LAST; - } else if (pSource->base.functionId == TSDB_FUNC_STDDEV_DST) { - functionId = TSDB_FUNC_STDDEV; - } - - int32_t inter = 0; - getResultDataInfo(pSource->base.colType, pSource->base.colBytes, functionId, 0, &pse->resType, - &pse->resBytes, &inter, 0, false, NULL); - pse->colType = pse->resType; - pse->colBytes = pse->resBytes; - - } else { // arithmetic expression - pse->colInfo.colId = pSource->base.colInfo.colId; - pse->colType = pSource->base.colType; - pse->colBytes = pSource->base.colBytes; - pse->resBytes = sizeof(double); - pse->resType = TSDB_DATA_TYPE_DOUBLE; - - pse->functionId = pSource->base.functionId; - pse->numOfParams = pSource->base.numOfParams; - - for (int32_t j = 0; j < pSource->base.numOfParams; ++j) { - tVariantAssign(&pse->param[j], &pSource->base.param[j]); - buildArithmeticExprFromMsg(px, NULL); - } - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t createGlobalAggregateExpr(SQueryAttr* pQueryAttr, SQueryInfo* pQueryInfo) { - assert(tscIsTwoStageSTableQuery(pQueryInfo, 0)); - - pQueryAttr->numOfExpr3 = (int32_t) tscNumOfExprs(pQueryInfo); - pQueryAttr->pExpr3 = calloc(pQueryAttr->numOfExpr3, sizeof(SExprInfo)); - if (pQueryAttr->pExpr3 == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pQueryAttr->numOfExpr3; ++i) { - SExprInfo* pExpr = &pQueryAttr->pExpr1[i]; - SSqlExpr* pse = &pQueryAttr->pExpr3[i].base; - - tscExprAssign(&pQueryAttr->pExpr3[i], pExpr); - pse->colInfo.colId = pExpr->base.resColId; - pse->colInfo.colIndex = i; - - pse->colType = pExpr->base.resType; - pse->colBytes = pExpr->base.resBytes; - } - - { - for (int32_t i = 0; i < pQueryAttr->numOfExpr3; ++i) { - SExprInfo* pExpr = &pQueryAttr->pExpr1[i]; - SSqlExpr* pse = &pQueryAttr->pExpr3[i].base; - - // the final result size and type in the same as query on single table. - // so here, set the flag to be false; - int32_t inter = 0; - - int32_t functionId = pExpr->base.functionId; - if (functionId >= TSDB_FUNC_TS && functionId <= TSDB_FUNC_DIFF) { - continue; - } - - if (functionId == TSDB_FUNC_FIRST_DST) { - functionId = TSDB_FUNC_FIRST; - } else if (functionId == TSDB_FUNC_LAST_DST) { - functionId = TSDB_FUNC_LAST; - } else if (functionId == TSDB_FUNC_STDDEV_DST) { - functionId = TSDB_FUNC_STDDEV; - } - - SUdfInfo* pUdfInfo = NULL; - - if (functionId < 0) { - pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); - } - - getResultDataInfo(pExpr->base.colType, pExpr->base.colBytes, functionId, 0, &pse->resType, &pse->resBytes, &inter, - 0, false, pUdfInfo); - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t createTagColumnInfo(SQueryAttr* pQueryAttr, SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo) { - if (pTableMetaInfo->tagColList == NULL) { - return TSDB_CODE_SUCCESS; - } - - pQueryAttr->numOfTags = (int16_t)taosArrayGetSize(pTableMetaInfo->tagColList); - if (pQueryAttr->numOfTags == 0) { - return TSDB_CODE_SUCCESS; - } - - STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[0]->pTableMeta; - - int32_t numOfTagColumns = tscGetNumOfTags(pTableMeta); - - pQueryAttr->tagColList = calloc(pQueryAttr->numOfTags, sizeof(SColumnInfo)); - if (pQueryAttr->tagColList == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SSchema* pSchema = tscGetTableTagSchema(pTableMeta); - for (int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { - SColumn* pCol = taosArrayGetP(pTableMetaInfo->tagColList, i); - SSchema* pColSchema = &pSchema[pCol->columnIndex]; - - if ((pCol->columnIndex >= numOfTagColumns || pCol->columnIndex < TSDB_TBNAME_COLUMN_INDEX) || - (!isValidDataType(pColSchema->type))) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - SColumnInfo* pTagCol = &pQueryAttr->tagColList[i]; - - pTagCol->colId = pColSchema->colId; - pTagCol->bytes = pColSchema->bytes; - pTagCol->type = pColSchema->type; - pTagCol->flist.numOfFilters = 0; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tscGetColFilterSerializeLen(SQueryInfo* pQueryInfo) { - int16_t numOfCols = (int16_t)taosArrayGetSize(pQueryInfo->colList); - int32_t len = 0; - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i); - for (int32_t j = 0; j < pCol->info.flist.numOfFilters; ++j) { - len += sizeof(SColumnFilterInfo); - if (pCol->info.flist.filterInfo[j].filterstr) { - len += (int32_t)pCol->info.flist.filterInfo[j].len + 1 * TSDB_NCHAR_SIZE; - } - } - } - return len; -} - -int32_t tscGetTagFilterSerializeLen(SQueryInfo* pQueryInfo) { - // serialize tag column query condition - if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) { - STagCond* pTagCond = &pQueryInfo->tagCond; - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid); - if (pCond != NULL && pCond->cond != NULL) { - return pCond->len; - } - } - return 0; -} - -int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr) { - memset(pQueryAttr, 0, sizeof(SQueryAttr)); - - int16_t numOfCols = (int16_t) taosArrayGetSize(pQueryInfo->colList); - int16_t numOfOutput = (int16_t) tscNumOfExprs(pQueryInfo); - - pQueryAttr->topBotQuery = tscIsTopBotQuery(pQueryInfo); - pQueryAttr->hasTagResults = hasTagValOutput(pQueryInfo); - pQueryAttr->stabledev = isStabledev(pQueryInfo); - pQueryAttr->tsCompQuery = isTsCompQuery(pQueryInfo); - pQueryAttr->diffQuery = tscIsDiffDerivQuery(pQueryInfo); - pQueryAttr->simpleAgg = isSimpleAggregateRv(pQueryInfo); - pQueryAttr->needReverseScan = tscNeedReverseScan(pQueryInfo); - pQueryAttr->stableQuery = QUERY_IS_STABLE_QUERY(pQueryInfo->type); - pQueryAttr->groupbyColumn = (!pQueryInfo->stateWindow) && tscGroupbyColumn(pQueryInfo); - pQueryAttr->queryBlockDist = isBlockDistQuery(pQueryInfo); - pQueryAttr->pointInterpQuery = tscIsPointInterpQuery(pQueryInfo); - pQueryAttr->timeWindowInterpo = timeWindowInterpoRequired(pQueryInfo); - pQueryAttr->distinct = pQueryInfo->distinct; - pQueryAttr->sw = pQueryInfo->sessionWindow; - pQueryAttr->stateWindow = pQueryInfo->stateWindow; - pQueryAttr->multigroupResult = pQueryInfo->multigroupResult; - - pQueryAttr->numOfCols = numOfCols; - pQueryAttr->numOfOutput = numOfOutput; - pQueryAttr->limit = pQueryInfo->limit; - pQueryAttr->slimit = pQueryInfo->slimit; - pQueryAttr->order = pQueryInfo->order; - pQueryAttr->fillType = pQueryInfo->fillType; - pQueryAttr->havingNum = pQueryInfo->havingFieldNum; - pQueryAttr->pUdfInfo = pQueryInfo->pUdfInfo; - - if (pQueryInfo->order.order == TSDB_ORDER_ASC) { // TODO refactor - pQueryAttr->window = pQueryInfo->window; - } else { - pQueryAttr->window.skey = pQueryInfo->window.ekey; - pQueryAttr->window.ekey = pQueryInfo->window.skey; - } - - memcpy(&pQueryAttr->interval, &pQueryInfo->interval, sizeof(pQueryAttr->interval)); - - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { - pQueryAttr->pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr)); - *(pQueryAttr->pGroupbyExpr) = pQueryInfo->groupbyExpr; - pQueryAttr->pGroupbyExpr->columnInfo = taosArrayDup(pQueryInfo->groupbyExpr.columnInfo); - } else { - assert(pQueryInfo->groupbyExpr.columnInfo == NULL); - } - - pQueryAttr->pExpr1 = calloc(pQueryAttr->numOfOutput, sizeof(SExprInfo)); - for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - SExprInfo* pExpr = tscExprGet(pQueryInfo, i); - tscExprAssign(&pQueryAttr->pExpr1[i], pExpr); - - if (pQueryAttr->pExpr1[i].base.functionId == TSDB_FUNC_ARITHM) { - for (int32_t j = 0; j < pQueryAttr->pExpr1[i].base.numOfParams; ++j) { - buildArithmeticExprFromMsg(&pQueryAttr->pExpr1[i], NULL); - } - } - } - - pQueryAttr->tableCols = calloc(numOfCols, sizeof(SColumnInfo)); - for(int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i); - if (!isValidDataType(pCol->info.type) || pCol->info.type == TSDB_DATA_TYPE_NULL) { - assert(0); - } - - pQueryAttr->tableCols[i] = pCol->info; - pQueryAttr->tableCols[i].flist.filterInfo = tFilterInfoDup(pCol->info.flist.filterInfo, pQueryAttr->tableCols[i].flist.numOfFilters); - } - - // global aggregate query - if (pQueryAttr->stableQuery && (pQueryAttr->simpleAgg || pQueryAttr->interval.interval > 0) && tscIsTwoStageSTableQuery(pQueryInfo, 0)) { - createGlobalAggregateExpr(pQueryAttr, pQueryInfo); - } - - // for simple table, not for super table - if (pQueryInfo->arithmeticOnAgg) { - pQueryAttr->numOfExpr2 = (int32_t) taosArrayGetSize(pQueryInfo->exprList1); - pQueryAttr->pExpr2 = calloc(pQueryAttr->numOfExpr2, sizeof(SExprInfo)); - for(int32_t i = 0; i < pQueryAttr->numOfExpr2; ++i) { - SExprInfo* p = taosArrayGetP(pQueryInfo->exprList1, i); - tscExprAssign(&pQueryAttr->pExpr2[i], p); - } - } - - // tag column info - int32_t code = createTagColumnInfo(pQueryAttr, pQueryInfo, pTableMetaInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (pQueryAttr->fillType != TSDB_FILL_NONE) { - pQueryAttr->fillVal = calloc(pQueryAttr->numOfOutput, sizeof(int64_t)); - memcpy(pQueryAttr->fillVal, pQueryInfo->fillVal, pQueryInfo->numOfFillVal * sizeof(int64_t)); - } - - pQueryAttr->srcRowSize = 0; - pQueryAttr->maxTableColumnWidth = 0; - for (int16_t i = 0; i < numOfCols; ++i) { - pQueryAttr->srcRowSize += pQueryAttr->tableCols[i].bytes; - if (pQueryAttr->maxTableColumnWidth < pQueryAttr->tableCols[i].bytes) { - pQueryAttr->maxTableColumnWidth = pQueryAttr->tableCols[i].bytes; - } - } - - pQueryAttr->interBufSize = getOutputInterResultBufSize(pQueryAttr); - - if (pQueryAttr->numOfCols <= 0 && !tscQueryTags(pQueryInfo) && !pQueryAttr->queryBlockDist) { - tscError("%p illegal value of numOfCols in query msg: %" PRIu64 ", table cols:%d", addr, - (uint64_t)pQueryAttr->numOfCols, numOfCols); - - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pQueryAttr->interval.interval < 0) { - tscError("%p illegal value of aggregation time interval in query msg: %" PRId64, addr, - (int64_t)pQueryInfo->interval.interval); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pQueryAttr->pGroupbyExpr != NULL && pQueryAttr->pGroupbyExpr->numOfGroupCols < 0) { - tscError("%p illegal value of numOfGroupCols in query msg: %d", addr, pQueryInfo->groupbyExpr.numOfGroupCols); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t doAddTableName(char* nextStr, char** str, SArray* pNameArray, SSqlObj* pSql) { - int32_t code = TSDB_CODE_SUCCESS; - SSqlCmd* pCmd = &pSql->cmd; - - char tablename[TSDB_TABLE_FNAME_LEN] = {0}; - int32_t len = 0; - - if (nextStr == NULL) { - tstrncpy(tablename, *str, TSDB_TABLE_FNAME_LEN); - len = (int32_t) strlen(tablename); - } else { - len = (int32_t)(nextStr - (*str)); - if (len >= TSDB_TABLE_NAME_LEN) { - sprintf(pCmd->payload, "table name too long"); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - memcpy(tablename, *str, nextStr - (*str)); - tablename[len] = '\0'; - } - - (*str) = nextStr + 1; - len = (int32_t)strtrim(tablename); - - SStrToken sToken = {.n = len, .type = TK_ID, .z = tablename}; - tGetToken(tablename, &sToken.type); - - // Check if the table name available or not - if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) { - sprintf(pCmd->payload, "table name is invalid"); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - SName name = {0}; - if ((code = tscSetTableFullName(&name, &sToken, pSql)) != TSDB_CODE_SUCCESS) { - return code; - } - - memset(tablename, 0, tListLen(tablename)); - tNameExtractFullName(&name, tablename); - - char* p = strdup(tablename); - taosArrayPush(pNameArray, &p); - return TSDB_CODE_SUCCESS; -} - -int32_t nameComparFn(const void* n1, const void* n2) { - int32_t ret = strcmp(*(char**)n1, *(char**)n2); - if (ret == 0) { - return 0; - } else { - return ret > 0? 1:-1; - } -} - -static void freeContent(void* p) { - char* ptr = *(char**)p; - tfree(ptr); -} - -int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray) { - SSqlCmd *pCmd = &pSql->cmd; - - pCmd->command = TSDB_SQL_MULTI_META; - pCmd->msgType = TDMT_VND_TABLES_META; - - int code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - char *str = (char *)pNameList; - - SQueryInfo *pQueryInfo = tscGetQueryInfoS(pCmd); - if (pQueryInfo == NULL) { - pSql->res.code = terrno; - return terrno; - } - - char *nextStr; - while (1) { - nextStr = strchr(str, ','); - if (nextStr == NULL) { - code = doAddTableName(nextStr, &str, pNameArray, pSql); - break; - } - - code = doAddTableName(nextStr, &str, pNameArray, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (taosArrayGetSize(pNameArray) > TSDB_MULTI_TABLEMETA_MAX_NUM) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pCmd->payload, "tables over the max number"); - return code; - } - } - - size_t len = taosArrayGetSize(pNameArray); - if (len == 1) { - return TSDB_CODE_SUCCESS; - } - - if (len > TSDB_MULTI_TABLEMETA_MAX_NUM) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pCmd->payload, "tables over the max number"); - return code; - } - - taosArraySort(pNameArray, nameComparFn); - taosArrayRemoveDuplicate(pNameArray, nameComparFn, freeContent); - return TSDB_CODE_SUCCESS; -} - -bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src) { - assert(pExisted != NULL && src != NULL); - if (pExisted->numOfEps != src->numOfEps) { - return false; - } - - for(int32_t i = 0; i < pExisted->numOfEps; ++i) { - if (pExisted->ep[i].port != src->epAddr[i].port) { - return false; - } - - if (strncmp(pExisted->ep[i].fqdn, src->epAddr[i].fqdn, tListLen(pExisted->ep[i].fqdn)) != 0) { - return false; - } - } - - return true; -} - -SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg) { - assert(pVgroupMsg != NULL); - - SNewVgroupInfo info = {0}; - info.numOfEps = pVgroupMsg->numOfEps; - info.vgId = pVgroupMsg->vgId; - info.inUse = 0; // 0 is the default value of inUse in case of multiple replica - - assert(info.numOfEps >= 1 && info.vgId >= 1); - for(int32_t i = 0; i < pVgroupMsg->numOfEps; ++i) { - tstrncpy(info.ep[i].fqdn, pVgroupMsg->epAddr[i].fqdn, TSDB_FQDN_LEN); - info.ep[i].port = pVgroupMsg->epAddr[i].port; - } - - return info; -} - -void tscRemoveCachedTableMeta(STableMetaInfo* pTableMetaInfo, uint64_t id) { - char fname[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, fname); - - int32_t len = (int32_t) strnlen(fname, TSDB_TABLE_FNAME_LEN); - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - void* pv = taosCacheAcquireByKey(tscVgroupListBuf, fname, len); - if (pv != NULL) { - taosCacheRelease(tscVgroupListBuf, &pv, true); - } - } - - taosHashRemove(tscTableMetaMap, fname, len); - tscDebug("0x%"PRIx64" remove table meta %s, numOfRemain:%d", id, fname, (int32_t) taosHashGetSize(tscTableMetaMap)); -} - -char* cloneCurrentDBName(SSqlObj* pSql) { - char *p = NULL; - HttpContext *pCtx = NULL; - - pthread_mutex_lock(&pSql->pTscObj->mutex); - STscObj *pTscObj = pSql->pTscObj; - switch (pTscObj->from) { - case TAOS_REQ_FROM_HTTP: - pCtx = pSql->param; - if (pCtx && pCtx->db[0] != '\0') { - char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN] = {0}; - int32_t len = sprintf(db, "%s%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pCtx->db); - assert(len <= sizeof(db)); - - p = strdup(db); - } - break; - default: - break; - } - if (p == NULL) { - p = strdup(pSql->pTscObj->db); - } - pthread_mutex_unlock(&pSql->pTscObj->mutex); - - return p; -} diff --git a/2.0/src/client/tests/CMakeLists.txt b/2.0/src/client/tests/CMakeLists.txt deleted file mode 100644 index 24bfb44ac90e11e01ba99423aa68bd5a9511f746..0000000000000000000000000000000000000000 --- a/2.0/src/client/tests/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) -FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64) -FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64) - -IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) - MESSAGE(STATUS "gTest library found, build unit test") - - # GoogleTest requires at least C++11 - SET(CMAKE_CXX_STANDARD 11) - - INCLUDE_DIRECTORIES(/usr/include /usr/local/include) - LINK_DIRECTORIES(/usr/lib /usr/local/lib) - - AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - - ADD_EXECUTABLE(cliTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(cliTest taos tutil common gtest pthread) -ENDIF() diff --git a/2.0/src/client/tests/cliTest.cpp b/2.0/src/client/tests/cliTest.cpp deleted file mode 100644 index c2799c7ff6cf75017fc4aacfca04caddf6189389..0000000000000000000000000000000000000000 --- a/2.0/src/client/tests/cliTest.cpp +++ /dev/null @@ -1,267 +0,0 @@ -#include -#include -#include - -#include "taos.h" -#include "tglobal.h" - -namespace { -static int64_t start_ts = 1433955661000; - -void stmtInsertTest() { - TAOS* conn = taos_connect("ubuntu", "root", "taosdata", 0, 0); - if (conn == NULL) { - printf("Failed to connect to DB, reason:%s", taos_errstr(conn)); - exit(-1); - } - - TAOS_RES* res = taos_query(conn, "use test"); - taos_free_result(res); - - const char* sql = "insert into t1 values(?, ?, ?, ?)"; - TAOS_STMT* stmt = taos_stmt_init(conn); - - int32_t ret = taos_stmt_prepare(stmt, sql, 0); - ASSERT_EQ(ret, 0); - - //ts timestamp, k int, a binary(11), b nchar(4) - struct { - int64_t ts; - int k; - char* a; - char* b; - } v = {0}; - - TAOS_BIND params[4]; - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts); - params[0].buffer = &v.ts; - params[0].length = ¶ms[0].buffer_length; - params[0].is_null = NULL; - - params[1].buffer_type = TSDB_DATA_TYPE_INT; - params[1].buffer_length = sizeof(v.k); - params[1].buffer = &v.k; - params[1].length = ¶ms[1].buffer_length; - params[1].is_null = NULL; - - params[2].buffer_type = TSDB_DATA_TYPE_BINARY; - params[2].buffer_length = sizeof(v.a); - params[2].buffer = &v.a; - params[2].is_null = NULL; - - params[3].buffer_type = TSDB_DATA_TYPE_NCHAR; - params[3].buffer_length = sizeof(v.b); - params[3].buffer = &v.b; - params[3].is_null = NULL; - - v.ts = start_ts + 20; - v.k = 123; - - char str[] = "abc"; - uintptr_t len = strlen(str); - - v.a = str; - params[2].length = &len; - params[2].buffer_length = len; - params[2].buffer = str; - - char nstr[] = "999"; - uintptr_t len1 = strlen(nstr); - - v.b = nstr; - params[3].buffer_length = len1; - params[3].buffer = nstr; - params[3].length = &len1; - - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("\033[31mfailed to execute insert statement.\033[0m\n"); - return; - } - - v.ts = start_ts + 30; - v.k = 911; - - char str1[] = "92"; - len = strlen(str1); - - params[2].length = &len; - params[2].buffer_length = len; - params[2].buffer = str1; - - char nstr1[] = "1920"; - len1 = strlen(nstr1); - - params[3].buffer_length = len1; - params[3].buffer = nstr1; - params[3].length = &len1; - - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); - - ret = taos_stmt_execute(stmt); - if (ret != 0) { - printf("%d\n", ret); - printf("\033[31mfailed to execute insert statement.\033[0m\n"); - return; - } - - taos_stmt_close(stmt); - taos_close(conn); -} - -void validateResultFields() { - TAOS* conn = taos_connect("ubuntu", "root", "taosdata", 0, 0); - if (conn == NULL) { - printf("Failed to connect to DB, reason:%s", taos_errstr(conn)); - exit(-1); - } - - TAOS_RES* res = taos_query(conn, "create database if not exists test"); - ASSERT_EQ(taos_errno(res), 0); - taos_free_result(res); - - res = taos_query(conn, "use test"); - ASSERT_EQ(taos_errno(res), 0); - taos_free_result(res); - - res = taos_query(conn, "create table if not exists t1(ts timestamp, k int, a binary(11), b nchar(4))"); - ASSERT_EQ(taos_errno(res), 0); - taos_free_result(res); - - char sql[512] = {0}; - sprintf(sql, "insert into t1 values(%" PRId64 ", 99, 'abc', 'test')", start_ts); - - res = taos_query(conn, sql); - ASSERT_EQ(taos_errno(res), 0); - taos_free_result(res); - - res = taos_query(conn, "select count(*), spread(ts)/(1000 * 3600 * 24), first(a), last(b) from t1"); - ASSERT_EQ(taos_num_fields(res), 4); - - TAOS_FIELD* fields = taos_fetch_fields(res); - ASSERT_EQ(fields[0].bytes, 8); - ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_BIGINT); - ASSERT_STREQ(fields[0].name, "count(*)"); - - ASSERT_EQ(fields[1].bytes, 8); - ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_STREQ(fields[1].name, "spread(ts)/(1000 * 3600 * 24)"); - - ASSERT_EQ(fields[2].bytes, 11); - ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY); - ASSERT_STREQ(fields[2].name, "first(a)"); - - ASSERT_EQ(fields[3].bytes, 4); - ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR); - ASSERT_STREQ(fields[3].name, "last(b)"); - - taos_free_result(res); - - res = taos_query(conn, "select last_row(*) from t1"); - ASSERT_EQ(taos_num_fields(res), 4); - - fields = taos_fetch_fields(res); - ASSERT_EQ(fields[0].bytes, 8); - ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); - ASSERT_STREQ(fields[0].name, "last_row(ts)"); - - ASSERT_EQ(fields[1].bytes, 4); - ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_INT); - ASSERT_STREQ(fields[1].name, "last_row(k)"); - - ASSERT_EQ(fields[2].bytes, 11); - ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY); - ASSERT_STREQ(fields[2].name, "last_row(a)"); - - ASSERT_EQ(fields[3].bytes, 4); - ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR); - ASSERT_STREQ(fields[3].name, "last_row(b)"); - - taos_free_result(res); - res = taos_query(conn, "select first(*), last(*) from t1"); - ASSERT_EQ(taos_num_fields(res), 8); - - fields = taos_fetch_fields(res); - ASSERT_EQ(fields[0].bytes, 8); - ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); - ASSERT_STREQ(fields[0].name, "first(ts)"); - - ASSERT_EQ(fields[1].bytes, 4); - ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_INT); - ASSERT_STREQ(fields[1].name, "first(k)"); - - ASSERT_EQ(fields[2].bytes, 11); - ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY); - ASSERT_STREQ(fields[2].name, "first(a)"); - - ASSERT_EQ(fields[3].bytes, 4); - ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR); - ASSERT_STREQ(fields[3].name, "first(b)"); - - taos_free_result(res); - - res = taos_query(conn, "select first(ts, a, k, k, b, b, ts) from t1"); - ASSERT_EQ(taos_num_fields(res), 7); - - fields = taos_fetch_fields(res); - ASSERT_EQ(fields[0].bytes, 8); - ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); - ASSERT_STREQ(fields[0].name, "first(ts)"); - - ASSERT_EQ(fields[1].bytes, 11); - ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_BINARY); - ASSERT_STREQ(fields[1].name, "first(a)"); - - ASSERT_EQ(fields[2].bytes, 4); - ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_INT); - ASSERT_STREQ(fields[2].name, "first(k)"); - - ASSERT_EQ(fields[3].bytes, 4); - ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_INT); - ASSERT_STREQ(fields[3].name, "first(k)"); - - ASSERT_EQ(fields[4].bytes, 4); - ASSERT_EQ(fields[4].type, TSDB_DATA_TYPE_NCHAR); - ASSERT_STREQ(fields[4].name, "first(b)"); - - ASSERT_EQ(fields[5].bytes, 4); - ASSERT_EQ(fields[5].type, TSDB_DATA_TYPE_NCHAR); - ASSERT_STREQ(fields[5].name, "first(b)"); - - ASSERT_EQ(fields[6].bytes, 8); - ASSERT_EQ(fields[6].type, TSDB_DATA_TYPE_TIMESTAMP); - ASSERT_STREQ(fields[6].name, "first(ts)"); - - taos_free_result(res); - - // update the configure parameter, the result field name will be changed - tsKeepOriginalColumnName = 1; - res = taos_query(conn, "select first(ts, a, k, k, b, b, ts) from t1"); - ASSERT_EQ(taos_num_fields(res), 7); - - fields = taos_fetch_fields(res); - ASSERT_EQ(fields[0].bytes, 8); - ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); - ASSERT_STREQ(fields[0].name, "ts"); - - ASSERT_EQ(fields[2].bytes, 4); - ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_INT); - ASSERT_STREQ(fields[2].name, "k"); - - taos_free_result(res); - - taos_close(conn); -} -} -/* test parse time function */ -TEST(testCase, result_field_test) { - taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); - taos_init(); - - validateResultFields(); - stmtInsertTest(); -} diff --git a/2.0/src/client/tests/timeParseTest.cpp b/2.0/src/client/tests/timeParseTest.cpp deleted file mode 100644 index f6de5d46a7c569ae5abe38f7e9cd2b6dc56a5586..0000000000000000000000000000000000000000 --- a/2.0/src/client/tests/timeParseTest.cpp +++ /dev/null @@ -1,173 +0,0 @@ - -#include -#include -#include -#include - -#include "os.h" -#include "taos.h" -#include "ttoken.h" -#include "tutil.h" - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -extern void deltaToUtcInitOnce(); -/* test parse time function */ -TEST(testCase, parse_time) { - - taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); - deltaToUtcInitOnce(); - - char t1[] = "2018-1-1 1:1:1.952798"; - char t13[] = "1970-1-1 0:0:0"; - - int64_t time = 0, time1 = 0; - - taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 1514739661952); - - taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND); - - char t2[] = "2018-1-1T1:1:1.952Z"; - taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI, 0); - - EXPECT_EQ(time, 1514739661952 + 28800000); - - char t3[] = "2018-1-1 1:01:01.952"; - taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 1514739661952); - - char t4[] = "2018-1-1 1:01:01.9"; - char t5[] = "2018-1-1 1:01:1.900"; - char t6[] = "2018-01-01 1:1:1.90"; - char t7[] = "2018-01-01 01:01:01.9"; - char t8[] = "2018-01-01 01:01:01.9007865"; - - taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t9[] = "2017-4-3 1:1:2.980"; - char t10[] = "2017-4-3T2:1:2.98+9:00"; - taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t11[] = "2017-4-3T2:1:2.98+09:00"; - taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t12[] = "2017-4-3T2:1:2.98+0900"; - taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taos_options(TSDB_OPTION_TIMEZONE, "UTC"); - deltaToUtcInitOnce(); - - taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 0); - - taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); - deltaToUtcInitOnce(); - - char t14[] = "1970-1-1T0:0:0Z"; - taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 0); - - char t40[] = "1970-1-1 0:0:0.999999999"; - taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND); - - char t41[] = "1997-1-1 0:0:0.999999999"; - taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 852048000999); - - // int64_t k = timezone; - char t42[] = "1997-1-1T0:0:0.999999999Z"; - taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND); - - //////////////////////////////////////////////////////////////////// - // illegal timestamp format - char t15[] = "2017-12-33 0:0:0"; - EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t16[] = "2017-12-31 99:0:0"; - EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t17[] = "2017-12-31T9:0:0"; - EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t18[] = "2017-12-31T9:0:0.Z"; - EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t19[] = "2017-12-31 9:0:0.-1"; - EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t20[] = "2017-12-31 9:0:0.1+12:99"; - EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI, 0), 0); - EXPECT_EQ(time, 1514682000100); - - char t21[] = "2017-12-31T9:0:0.1+12:99"; - EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t22[] = "2017-12-31 9:0:0.1+13:1"; - EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI, 0), 0); - - char t23[] = "2017-12-31T9:0:0.1+13:1"; - EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI, 0), 0); - - - //======================== add some case ============================// - - char b1[] = "9999-12-31 23:59:59.999"; - taosParseTime(b1, &time, strlen(b1), TSDB_TIME_PRECISION_MILLI,0); - EXPECT_EQ(time, 253402271999999); - - - char b2[] = "2020-01-01 01:01:01.321"; - taosParseTime(b2, &time, strlen(b2), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 1577811661321); - - taos_options(TSDB_OPTION_TIMEZONE, "America/New_York"); - deltaToUtcInitOnce(); - - taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 18000 * MILLISECOND_PER_SECOND); - - taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo"); - deltaToUtcInitOnce(); - - taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, -32400 * MILLISECOND_PER_SECOND); - - taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); - deltaToUtcInitOnce(); - - taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND); - - char t[] = "2021-01-08T02:11:40.000+00:00"; - taosParseTime(t, &time, strlen(t), TSDB_TIME_PRECISION_MILLI, 0); - printf("%" PRId64 "\n", time); -} - - diff --git a/2.0/src/query/CMakeLists.txt b/2.0/src/query/CMakeLists.txt deleted file mode 100644 index fd730adee56c3d5edddb943303f5b6b24d9f019c..0000000000000000000000000000000000000000 --- a/2.0/src/query/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(inc) - -AUX_SOURCE_DIRECTORY(src SRC) -ADD_LIBRARY(query ${SRC}) -SET_SOURCE_FILES_PROPERTIES(src/sql.c PROPERTIES COMPILE_FLAGS -w) -TARGET_LINK_LIBRARIES(query tsdb tutil lua) - -IF (TD_LINUX) - TARGET_LINK_LIBRARIES(query m rt lua) - ADD_SUBDIRECTORY(tests) -ENDIF () - -IF (TD_DARWIN) - TARGET_LINK_LIBRARIES(query m lua) - ADD_SUBDIRECTORY(tests) -ENDIF () diff --git a/2.0/src/query/inc/qAggMain.h b/2.0/src/query/inc/qAggMain.h deleted file mode 100644 index db165aa2060b032a970e3a0c536ad115f39d6333..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qAggMain.h +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QAGGMAIN_H -#define TDENGINE_QAGGMAIN_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" - -#include "tname.h" -#include "taosdef.h" -#include "trpc.h" -#include "tvariant.h" -#include "tsdb.h" -#include "qUdf.h" - -#define TSDB_FUNC_INVALID_ID -1 -#define TSDB_FUNC_COUNT 0 -#define TSDB_FUNC_SUM 1 -#define TSDB_FUNC_AVG 2 -#define TSDB_FUNC_MIN 3 -#define TSDB_FUNC_MAX 4 -#define TSDB_FUNC_STDDEV 5 -#define TSDB_FUNC_PERCT 6 -#define TSDB_FUNC_APERCT 7 -#define TSDB_FUNC_FIRST 8 -#define TSDB_FUNC_LAST 9 -#define TSDB_FUNC_LAST_ROW 10 -#define TSDB_FUNC_TOP 11 -#define TSDB_FUNC_BOTTOM 12 -#define TSDB_FUNC_SPREAD 13 -#define TSDB_FUNC_TWA 14 -#define TSDB_FUNC_LEASTSQR 15 - -#define TSDB_FUNC_TS 16 -#define TSDB_FUNC_TS_DUMMY 17 -#define TSDB_FUNC_TAG_DUMMY 18 -#define TSDB_FUNC_TS_COMP 19 - -#define TSDB_FUNC_TAG 20 -#define TSDB_FUNC_PRJ 21 - -#define TSDB_FUNC_TAGPRJ 22 -#define TSDB_FUNC_ARITHM 23 -#define TSDB_FUNC_DIFF 24 - -#define TSDB_FUNC_FIRST_DST 25 -#define TSDB_FUNC_LAST_DST 26 -#define TSDB_FUNC_STDDEV_DST 27 -#define TSDB_FUNC_INTERP 28 - -#define TSDB_FUNC_RATE 29 -#define TSDB_FUNC_IRATE 30 -#define TSDB_FUNC_TID_TAG 31 -#define TSDB_FUNC_DERIVATIVE 32 -#define TSDB_FUNC_BLKINFO 33 - - -#define TSDB_FUNC_HISTOGRAM 34 -#define TSDB_FUNC_HLL 35 -#define TSDB_FUNC_MODE 36 -#define TSDB_FUNC_SAMPLE 37 -#define TSDB_FUNC_CEIL 38 -#define TSDB_FUNC_FLOOR 39 -#define TSDB_FUNC_ROUND 40 -#define TSDB_FUNC_MAVG 41 -#define TSDB_FUNC_CSUM 42 - -#define TSDB_FUNCSTATE_SO 0x1u // single output -#define TSDB_FUNCSTATE_MO 0x2u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM -#define TSDB_FUNCSTATE_STREAM 0x4u // function avail for stream -#define TSDB_FUNCSTATE_STABLE 0x8u // function avail for super table -#define TSDB_FUNCSTATE_OF 0x10u // outer forward -#define TSDB_FUNCSTATE_NEED_TS 0x20u // timestamp is required during query processing -#define TSDB_FUNCSTATE_SELECTIVITY 0x40u // selectivity functions, can exists along with tag columns - -#define TSDB_BASE_FUNC_SO TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF -#define TSDB_BASE_FUNC_MO TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF - -#define TSDB_FUNCTIONS_NAME_MAX_LENGTH 16 -#define TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE 50 - -#define DATA_SET_FLAG ',' // to denote the output area has data, not null value -#define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG) - -#define QUERY_ASC_FORWARD_STEP 1 -#define QUERY_DESC_FORWARD_STEP -1 - -#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) - -#define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results -#define TOP_BOTTOM_QUERY_LIMIT 100 - -enum { - MASTER_SCAN = 0x0u, - REVERSE_SCAN = 0x1u, - REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan - MERGE_STAGE = 0x20u, -}; - -#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) -#define QUERY_IS_JOIN_QUERY(type) (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_QUERY)) -#define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) -#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0) - -typedef struct SArithmeticSupport { - SExprInfo *pExprInfo; - int32_t numOfCols; - SColumnInfo *colList; - void *exprList; // client side used - int32_t offset; - char** data; -} SArithmeticSupport; - -typedef struct SQLPreAggVal { - bool isSet; // statistics info set or not - bool dataBlockLoaded; // data block is loaded or not - SDataStatis statis; -} SQLPreAggVal; - -typedef struct SInterpInfoDetail { - TSKEY ts; // interp specified timestamp - int8_t type; - int8_t primaryCol; -} SInterpInfoDetail; - -typedef struct SResultRowCellInfo { - int8_t hasResult; // result generated, not NULL value - bool initialized; // output buffer has been initialized - bool complete; // query has completed - uint32_t numOfRes; // num of output result in current buffer -} SResultRowCellInfo; - -typedef struct SPoint1 { - int64_t key; - union{double val; char* ptr;}; -} SPoint1; - -#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo))) - -struct SQLFunctionCtx; - -/** - * for selectivity query, the corresponding tag value is assigned if the data is qualified - */ -typedef struct SExtTagsInfo { - int16_t tagsLen; // keep the tags data for top/bottom query result - int16_t numOfTagCols; - struct SQLFunctionCtx **pTagCtxList; -} SExtTagsInfo; - -// sql function runtime context -typedef struct SQLFunctionCtx { - int32_t size; // number of rows - void * pInput; // input data buffer - uint32_t order; // asc|desc - int16_t inputType; - int16_t inputBytes; - - int16_t outputType; - int16_t outputBytes; // size of results, determined by function and input column data type - int32_t interBufBytes; // internal buffer size - bool hasNull; // null value exist in current block - bool requireNull; // require null in some function - bool stableQuery; - int16_t functionId; // function id - char * pOutput; // final result output buffer, point to sdata->data - uint8_t currentStage; // record current running step, default: 0 - int64_t startTs; // timestamp range of current query when function is executed on a specific data block - int32_t numOfParams; - tVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param - int64_t *ptsList; // corresponding timestamp array list - void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ - SQLPreAggVal preAggVals; - tVariant tag; - - SResultRowCellInfo *resultInfo; - - SExtTagsInfo tagInfo; - SPoint1 start; - SPoint1 end; -} SQLFunctionCtx; - -typedef struct SAggFunctionInfo { - char name[TSDB_FUNCTIONS_NAME_MAX_LENGTH]; - uint8_t index; // index of function in aAggs - int8_t stableFuncId; // transfer function for super table query - uint16_t status; - - bool (*init)(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultCellInfo); // setup the execute environment - void (*xFunction)(SQLFunctionCtx *pCtx); // blocks version function - - // finalizer must be called after all xFunction has been executed to generated final result. - void (*xFinalize)(SQLFunctionCtx *pCtx); - void (*mergeFunc)(SQLFunctionCtx *pCtx); - - int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId); -} SAggFunctionInfo; - -#define GET_RES_INFO(ctx) ((ctx)->resultInfo) - -int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, - int16_t *len, int32_t *interBytes, int16_t extLength, bool isSuperTable, SUdfInfo* pUdfInfo); -int32_t isValidFunction(const char* name, int32_t len); - -#define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0) -#define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0) -#define IS_SINGLEOUTPUT(x) (((x)&TSDB_FUNCSTATE_SO) != 0) -#define IS_OUTER_FORWARD(x) (((x)&TSDB_FUNCSTATE_OF) != 0) - -// determine the real data need to calculated the result -enum { - BLK_DATA_NO_NEEDED = 0x0, - BLK_DATA_STATIS_NEEDED = 0x1, - BLK_DATA_ALL_NEEDED = 0x3, - BLK_DATA_DISCARD = 0x4, // discard current data block since it is not qualified for filter -}; - -typedef struct STwaInfo { - int8_t hasResult; // flag to denote has value - double dOutput; - SPoint1 p; - STimeWindow win; -} STwaInfo; - -struct SBufferWriter; -void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw); -void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist); - -/* global sql function array */ -extern struct SAggFunctionInfo aAggs[]; - -extern int32_t functionCompatList[]; // compatible check array list - -bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval); - -/** - * the numOfRes should be kept, since it may be used later - * and allow the ResultInfo to be re initialized - */ -#define RESET_RESULT_INFO(_r) \ - do { \ - (_r)->initialized = false; \ - } while (0) - -static FORCE_INLINE void initResultRowEntry(SResultRowCellInfo *pResInfo, int32_t bufLen) { - pResInfo->initialized = true; // the this struct has been initialized flag - - pResInfo->complete = false; - pResInfo->hasResult = false; - pResInfo->numOfRes = 0; - - memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen); -} - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_QAGGMAIN_H diff --git a/2.0/src/query/inc/qExecutor.h b/2.0/src/query/inc/qExecutor.h deleted file mode 100644 index 34b306b129ab4231e44ecabb098c750b81f07ba4..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qExecutor.h +++ /dev/null @@ -1,688 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef TDENGINE_QEXECUTOR_H -#define TDENGINE_QEXECUTOR_H - -#include "os.h" - -#include "hash.h" -#include "qAggMain.h" -#include "qFill.h" -#include "qResultbuf.h" -#include "qSqlparser.h" -#include "qTableMeta.h" -#include "qTsbuf.h" -#include "query.h" -#include "taosdef.h" -#include "tarray.h" -#include "tlockfree.h" -#include "tsdb.h" -#include "qUdf.h" - -struct SColumnFilterElem; -typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, const char* val1, const char* val2, int16_t type); -typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); - -#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) -#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u) -#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) - -#define GET_TABLEGROUP(q, _index) ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index))) - -#define GET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL? 0:((_r)->outputBuf)->info.rows) - -#define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData? 1 : 0) - -enum { - // when query starts to execute, this status will set - QUERY_NOT_COMPLETED = 0x1u, - - /* query is over - * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. - * 2. when all data within queried time window, it is also denoted as query_completed - */ - QUERY_COMPLETED = 0x2u, - - /* when the result is not completed return to client, this status will be - * usually used in case of interval query with interpolation option - */ - QUERY_OVER = 0x4u, -}; - -typedef struct SResultRowPool { - int32_t elemSize; - int32_t blockSize; - int32_t numOfElemPerBlock; - - struct { - int32_t blockIndex; - int32_t pos; - } position; - - SArray* pData; // SArray -} SResultRowPool; - -typedef struct SResultRow { - int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer - int32_t offset:29; // row index in buffer page - bool startInterp; // the time window start timestamp has done the interpolation already. - bool endInterp; // the time window end timestamp has done the interpolation already. - bool closed; // this result status: closed or opened - uint32_t numOfRows; // number of rows of current time window - SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo - STimeWindow win; - char *key; // start key of current result row -} SResultRow; - -typedef struct SResultRowCell { - uint64_t groupId; - SResultRow *pRow; -} SResultRowCell; - -typedef struct SGroupResInfo { - int32_t totalGroup; - int32_t currentGroup; - int32_t index; - SArray* pRows; // SArray - bool ordered; - int32_t position; -} SGroupResInfo; - -/** - * If the number of generated results is greater than this value, - * query query will be halt and return results to client immediate. - */ -typedef struct SRspResultInfo { - int64_t total; // total generated result size in rows - int32_t capacity; // capacity of current result output buffer - int32_t threshold; // result size threshold in rows. -} SRspResultInfo; - -typedef struct SResultRowInfo { - SResultRow** pResult; // result list - int16_t type:8; // data type for hash key - int32_t size:24; // number of result set - int32_t capacity; // max capacity - int32_t curPos; // current active result row index of pResult list -} SResultRowInfo; - -typedef struct SColumnFilterElem { - int16_t bytes; // column length - __filter_func_t fp; - SColumnFilterInfo filterInfo; - void *q; -} SColumnFilterElem; - -typedef struct SSingleColumnFilterInfo { - void* pData; - void* pData2; //used for nchar column - int32_t numOfFilters; - SColumnInfo info; - SColumnFilterElem* pFilters; -} SSingleColumnFilterInfo; - -typedef struct STableQueryInfo { - TSKEY lastKey; - int32_t groupIndex; // group id in table list - tVariant tag; - STimeWindow win; - STSCursor cur; - void* pTable; // for retrieve the page id list - SResultRowInfo resInfo; -} STableQueryInfo; - -typedef enum { - QUERY_PROF_BEFORE_OPERATOR_EXEC = 0, - QUERY_PROF_AFTER_OPERATOR_EXEC, - QUERY_PROF_QUERY_ABORT -} EQueryProfEventType; - -typedef struct { - EQueryProfEventType eventType; - int64_t eventTime; - - union { - uint8_t operatorType; //for operator event - int32_t abortCode; //for query abort event - }; -} SQueryProfEvent; - -typedef struct { - uint8_t operatorType; - int64_t sumSelfTime; - int64_t sumRunTimes; -} SOperatorProfResult; - -typedef struct SQueryCostInfo { - uint64_t loadStatisTime; - uint64_t loadFileBlockTime; - uint64_t loadDataInCacheTime; - uint64_t loadStatisSize; - uint64_t loadFileBlockSize; - uint64_t loadDataInCacheSize; - - uint64_t loadDataTime; - uint64_t totalRows; - uint64_t totalCheckedRows; - uint32_t totalBlocks; - uint32_t loadBlocks; - uint32_t loadBlockStatis; - uint32_t skipBlocks; - uint64_t elapsedTime; - uint64_t firstStageMergeTime; - uint64_t winInfoSize; - uint64_t tableInfoSize; - uint64_t hashSize; - uint64_t numOfTimeWindows; - - SArray* queryProfEvents; //SArray - SHashObj* operatorProfResults; //map -} SQueryCostInfo; - -typedef struct { - int64_t vgroupLimit; - int64_t ts; -} SOrderedPrjQueryInfo; - -typedef struct { - char* tags; - SArray* pResult; // SArray -} SInterResult; - -typedef struct SSDataBlock { - SDataStatis *pBlockStatis; - SArray *pDataBlock; - SDataBlockInfo info; -} SSDataBlock; - -// The basic query information extracted from the SQueryInfo tree to support the -// execution of query in a data node. -typedef struct SQueryAttr { - SLimitVal limit; - SLimitVal slimit; - - bool stableQuery; // super table query or not - bool topBotQuery; // TODO used bitwise flag - bool groupbyColumn; // denote if this is a groupby normal column query - bool hasTagResults; // if there are tag values in final result or not - bool timeWindowInterpo;// if the time window start/end required interpolation - bool queryBlockDist; // if query data block distribution - bool stabledev; // super table stddev query - bool tsCompQuery; // is tscomp query - bool diffQuery; // is diff query - bool simpleAgg; - bool pointInterpQuery; // point interpolation query - bool needReverseScan; // need reverse scan - bool distinct; // distinct query or not - bool stateWindow; // window State on sub/normal table - bool createFilterOperator; // if filter operator is needed - bool multigroupResult; // multigroup result can exist in one SSDataBlock - int32_t interBufSize; // intermediate buffer sizse - - int32_t havingNum; // having expr number - - SOrderVal order; - int16_t numOfCols; - int16_t numOfTags; - - STimeWindow window; - SInterval interval; - SSessionWindow sw; - int16_t precision; - int16_t numOfOutput; - int16_t fillType; - - int32_t srcRowSize; // todo extract struct - int32_t resultRowSize; - int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query. - int32_t maxTableColumnWidth; - int32_t tagLen; // tag value length of current query - SGroupbyExpr *pGroupbyExpr; - - SExprInfo* pExpr1; - SExprInfo* pExpr2; - int32_t numOfExpr2; - SExprInfo* pExpr3; - int32_t numOfExpr3; - - SColumnInfo* tableCols; - SColumnInfo* tagColList; - int32_t numOfFilterCols; - int64_t* fillVal; - SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query. - - SSingleColumnFilterInfo* pFilterInfo; - SFilterInfo *pFilters; - - void* tsdb; - SMemRef memRef; - STableGroupInfo tableGroupInfo; // table list SArray - int32_t vgId; - SArray *pUdfInfo; // no need to free -} SQueryAttr; - -typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup); -typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); - -struct SOperatorInfo; - -typedef struct SQueryRuntimeEnv { - jmp_buf env; - SQueryAttr* pQueryAttr; - uint32_t status; // query status - void* qinfo; - uint8_t scanFlag; // denotes reversed scan of data or not - void* pTsdbReadHandle; - - int32_t prevGroupId; // previous executed group id - bool enableGroupData; - SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file - SHashObj* pResultRowHashTable; // quick locate the window object for each result - SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not - SArray* pResultRowArrayList; // The array list that contains the Result rows - char* keyBuf; // window key buffer - SResultRowPool* pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object. - char** prevRow; - - SArray* prevResult; // intermediate result, SArray - STSBuf* pTsBuf; // timestamp filter list - STSCursor cur; - - char* tagVal; // tag value of current data block - SArithmeticSupport *sasArray; - - SSDataBlock *outputBuf; - STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure - struct SOperatorInfo *proot; - SGroupResInfo groupResInfo; - int64_t currentOffset; // dynamic offset value - - STableQueryInfo *current; - SRspResultInfo resultInfo; - SHashObj *pTableRetrieveTsMap; - SUdfInfo *pUdfInfo; -} SQueryRuntimeEnv; - -enum { - OP_IN_EXECUTING = 1, - OP_RES_TO_RETURN = 2, - OP_EXEC_DONE = 3, -}; - -enum OPERATOR_TYPE_E { - OP_TableScan = 1, - OP_DataBlocksOptScan = 2, - OP_TableSeqScan = 3, - OP_TagScan = 4, - OP_TableBlockInfoScan= 5, - OP_Aggregate = 6, - OP_Project = 7, - OP_Groupby = 8, - OP_Limit = 9, - OP_SLimit = 10, - OP_TimeWindow = 11, - OP_SessionWindow = 12, - OP_Fill = 13, - OP_MultiTableAggregate = 14, - OP_MultiTableTimeInterval = 15, - OP_DummyInput = 16, //TODO remove it after fully refactor. - OP_MultiwayMergeSort = 17, // multi-way data merge into one input stream. - OP_GlobalAggregate = 18, // global merge for the multi-way data sources. - OP_Filter = 19, - OP_Distinct = 20, - OP_Join = 21, - OP_StateWindow = 22, - OP_AllTimeWindow = 23, - OP_AllMultiTableTimeInterval = 24, - OP_Order = 25, -}; - -typedef struct SOperatorInfo { - uint8_t operatorType; - bool blockingOptr; // block operator or not - uint8_t status; // denote if current operator is completed - int32_t numOfOutput; // number of columns of the current operator results - char *name; // name, used to show the query execution plan - void *info; // extension attribution - SExprInfo *pExpr; - SQueryRuntimeEnv *pRuntimeEnv; - - struct SOperatorInfo **upstream; // upstream pointer list - int32_t numOfUpstream; // number of upstream. The value is always ONE expect for join operator - __operator_fn_t exec; - __optr_cleanup_fn_t cleanup; -} SOperatorInfo; - -enum { - QUERY_RESULT_NOT_READY = 1, - QUERY_RESULT_READY = 2, -}; - -typedef struct { - int32_t numOfTags; - int32_t numOfCols; - SColumnInfo *colList; -} SQueriedTableInfo; - -typedef struct SQInfo { - void* signature; - uint64_t qId; - int32_t code; // error code to returned to client - int64_t owner; // if it is in execution - - SQueryRuntimeEnv runtimeEnv; - SQueryAttr query; - void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; - - pthread_mutex_t lock; // used to synchronize the rsp/query threads - tsem_t ready; - int32_t dataReady; // denote if query result is ready or not - void* rspContext; // response context - int64_t startExecTs; // start to exec timestamp - char* sql; // query sql string - SQueryCostInfo summary; -} SQInfo; - -typedef struct SQueryParam { - char *sql; - char *tagCond; - char *colCond; - char *tbnameCond; - char *prevResult; - SArray *pTableIdList; - SSqlExpr **pExpr; - SSqlExpr **pSecExpr; - SExprInfo *pExprs; - SExprInfo *pSecExprs; - - SFilterInfo *pFilters; - - SColIndex *pGroupColIndex; - SColumnInfo *pTagColumnInfo; - SGroupbyExpr *pGroupbyExpr; - int32_t tableScanOperator; - SArray *pOperator; - SUdfInfo *pUdfInfo; -} SQueryParam; - -typedef struct STableScanInfo { - void *pTsdbReadHandle; - int32_t numOfBlocks; - int32_t numOfSkipped; - int32_t numOfBlockStatis; - int64_t numOfRows; - - int32_t order; // scan order - int32_t times; // repeat counts - int32_t current; - int32_t reverseTimes; // 0 by default - - SQLFunctionCtx *pCtx; // next operator query context - SResultRowInfo *pResultRowInfo; - int32_t *rowCellInfoOffset; - SExprInfo *pExpr; - SSDataBlock block; - int32_t numOfOutput; - int64_t elapsedTime; - - int32_t tableIndex; - int32_t prevGroupId; // previous table group id -} STableScanInfo; - -typedef struct STagScanInfo { - SColumnInfo* pCols; - SSDataBlock* pRes; - int32_t totalTables; - int32_t curPos; -} STagScanInfo; - -typedef struct SOptrBasicInfo { - SResultRowInfo resultRowInfo; - int32_t *rowCellInfoOffset; // offset value for each row result cell info - SQLFunctionCtx *pCtx; - SSDataBlock *pRes; -} SOptrBasicInfo; - -typedef struct SOptrBasicInfo STableIntervalOperatorInfo; - -typedef struct SAggOperatorInfo { - SOptrBasicInfo binfo; - uint32_t seed; -} SAggOperatorInfo; - -typedef struct SProjectOperatorInfo { - SOptrBasicInfo binfo; - int32_t bufCapacity; - uint32_t seed; - - SSDataBlock *existDataBlock; -} SProjectOperatorInfo; - -typedef struct SLimitOperatorInfo { - int64_t limit; - int64_t total; -} SLimitOperatorInfo; - -typedef struct SSLimitOperatorInfo { - int64_t groupTotal; - int64_t currentGroupOffset; - - int64_t rowsTotal; - int64_t currentOffset; - SLimitVal limit; - SLimitVal slimit; - - char **prevRow; - SArray *orderColumnList; - bool hasPrev; - bool ignoreCurrentGroup; - bool multigroupResult; - SSDataBlock *pRes; // result buffer - SSDataBlock *pPrevBlock; - int64_t capacity; - int64_t threshold; -} SSLimitOperatorInfo; - -typedef struct SFilterOperatorInfo { - SSingleColumnFilterInfo *pFilterInfo; - int32_t numOfFilterCols; -} SFilterOperatorInfo; - -typedef struct SFillOperatorInfo { - SFillInfo *pFillInfo; - SSDataBlock *pRes; - int64_t totalInputRows; - void **p; - SSDataBlock *existNewGroupBlock; - bool multigroupResult; -} SFillOperatorInfo; - -typedef struct SGroupbyOperatorInfo { - SOptrBasicInfo binfo; - int32_t colIndex; - char *prevData; // previous group by value -} SGroupbyOperatorInfo; - -typedef struct SSWindowOperatorInfo { - SOptrBasicInfo binfo; - STimeWindow curWindow; // current time window - TSKEY prevTs; // previous timestamp - int32_t numOfRows; // number of rows - int32_t start; // start row index - bool reptScan; // next round scan -} SSWindowOperatorInfo; - -typedef struct SStateWindowOperatorInfo { - SOptrBasicInfo binfo; - STimeWindow curWindow; // current time window - int32_t numOfRows; // number of rows - int32_t colIndex; // start row index - int32_t start; - char* prevData; // previous data - bool reptScan; -} SStateWindowOperatorInfo; - -typedef struct SDistinctDataInfo { - int32_t index; - int32_t type; - int32_t bytes; -} SDistinctDataInfo; - -typedef struct SDistinctOperatorInfo { - SHashObj *pSet; - SSDataBlock *pRes; - bool recordNullVal; //has already record the null value, no need to try again - int64_t threshold; - int64_t outputCapacity; - int32_t totalBytes; - char* buf; - SArray* pDistinctDataInfo; -} SDistinctOperatorInfo; - -struct SGlobalMerger; - -typedef struct SMultiwayMergeInfo { - struct SGlobalMerger *pMerge; - SOptrBasicInfo binfo; - int32_t bufCapacity; - int64_t seed; - char **prevRow; - SArray *orderColumnList; - int32_t resultRowFactor; - - bool hasGroupColData; - char **currentGroupColData; - SArray *groupColumnList; - bool hasDataBlockForNewGroup; - SSDataBlock *pExistBlock; - - SArray *udfInfo; - bool hasPrev; - bool multiGroupResults; -} SMultiwayMergeInfo; - -// todo support the disk-based sort -typedef struct SSortOperatorInfo { - int32_t colIndex; - int32_t order; - SSDataBlock *pDataBlock; -} SSortOperatorInfo; - -void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); - -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime); -SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); - -SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); -SOperatorInfo* createIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); -SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); -SOperatorInfo* createMultiwaySortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, - int32_t numOfRows, void* merger); -SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp); -SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult); -SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, - int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter); - -SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); -SOperatorInfo* createSortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal); - -SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); -SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); -SSDataBlock* doSLimit(void* param, bool* newgroup); - -int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); -void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); -bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p); -void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); - -SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows); - -void* blockDataDestroy(SSDataBlock* pBlock); -void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); - -void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); -int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput); -void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); -void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows); -void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity); -void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); - -void freeParam(SQueryParam *param); -int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param); -int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, - SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, SUdfInfo* pUdfInfo); - -int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, - SSqlExpr **pExpr, SExprInfo *prevExpr, SUdfInfo *pUdfInfo); - -int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters); - -SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code); -SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, - SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, SUdfInfo* pUdfInfo); - -int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, - int32_t prevResultLen, void* merger); - -int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId); -void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters); - -STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf); -STableQueryInfo* createTmpTableQueryInfo(STimeWindow win); - -int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg); - -bool isQueryKilled(SQInfo *pQInfo); -int32_t checkForQueryBuf(size_t numOfTables); -bool checkNeedToCompressQueryCol(SQInfo *pQInfo); -bool doBuildResCheck(SQInfo* pQInfo); -void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status); - -bool onlyQueryTags(SQueryAttr* pQueryAttr); -void destroyUdfInfo(SUdfInfo* pUdfInfo); - -bool isValidQInfo(void *param); - -int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen); - -size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows); -void setQueryKilled(SQInfo *pQInfo); - -void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType); -void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code); -void calculateOperatorProfResults(SQInfo* pQInfo); -void queryCostStatis(SQInfo *pQInfo); - -void freeQInfo(SQInfo *pQInfo); -void freeQueryAttr(SQueryAttr *pQuery); - -int32_t getMaximumIdleDurationSec(); - -void doInvokeUdf(SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); - -#endif // TDENGINE_QEXECUTOR_H diff --git a/2.0/src/query/inc/qExtbuffer.h b/2.0/src/query/inc/qExtbuffer.h deleted file mode 100644 index 1238348237332f70064a4b771ee32adc62fef334..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qExtbuffer.h +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef TDENGINE_TEXTBUFFER_H -#define TDENGINE_TEXTBUFFER_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" -#include "tmsg.h" - -#include "tarray.h" -#include "tutil.h" -#include "tdataformat.h" -#include "talgo.h" - -#define MAX_TMPFILE_PATH_LENGTH PATH_MAX -#define INITIAL_ALLOCATION_BUFFER_SIZE 64 -#define DEFAULT_PAGE_SIZE (1024L) // 16k larger than the SHistoInfo - -typedef enum EXT_BUFFER_FLUSH_MODEL { - /* - * all data that have been flushed to disk is belonged to the same group - * which means, all data in disk are sorted, or order is not matter in this case - */ - SINGLE_APPEND_MODEL, - - /* - * each flush operation to disk is completely independant to any other flush operation - * we simply merge several set of data in one file, to reduce the count of flat files - * in disk. So in this case, we need to keep the flush-out information in tFlushoutInfo - * structure. - */ - MULTIPLE_APPEND_MODEL, -} EXT_BUFFER_FLUSH_MODEL; - -typedef struct tFlushoutInfo { - uint32_t startPageId; - uint32_t numOfPages; -} tFlushoutInfo; - -typedef struct tFlushoutData { - uint32_t nAllocSize; - uint32_t nLength; - tFlushoutInfo *pFlushoutInfo; -} tFlushoutData; - -typedef struct SExtFileInfo { - uint32_t nFileSize; // in pages - uint32_t pageSize; - uint32_t numOfElemsInFile; - tFlushoutData flushoutData; -} SExtFileInfo; - -typedef struct tFilePage { - uint64_t num; - char data[]; -} tFilePage; - -typedef struct tFilePagesItem { - struct tFilePagesItem *pNext; - tFilePage item; -} tFilePagesItem; - -typedef struct SSchemaEx { - struct SSchema field; - int32_t offset; -} SSchemaEx; - -typedef struct SColumnModel { - int32_t capacity; - int32_t numOfCols; - int32_t rowSize; - SSchemaEx *pFields; -} SColumnModel; - -typedef struct SColumnOrderInfo { - int32_t numOfCols; - int16_t colIndex[]; -} SColumnOrderInfo; - -typedef struct tOrderDescriptor { - SColumnModel * pColumnModel; - int32_t tsOrder; // timestamp order type if exists - SColumnOrderInfo orderInfo; -} tOrderDescriptor; - -typedef struct tExtMemBuffer { - int32_t inMemCapacity; - int32_t nElemSize; - int32_t pageSize; - int32_t numOfTotalElems; - int32_t numOfElemsInBuffer; - int32_t numOfElemsPerPage; - int16_t numOfInMemPages; - - tFilePagesItem *pHead; - tFilePagesItem *pTail; - - char * path; - FILE * file; - SExtFileInfo fileMeta; - - SColumnModel * pColumnModel; - EXT_BUFFER_FLUSH_MODEL flushModel; -} tExtMemBuffer; - -/** - * - * @param inMemSize - * @param elemSize - * @param pModel - * @return - */ -tExtMemBuffer *createExtMemBuffer(int32_t inMemSize, int32_t elemSize, int32_t pagesize, SColumnModel *pModel); - -/** - * - * @param pMemBuffer - * @return - */ -void *destoryExtMemBuffer(tExtMemBuffer *pMemBuffer); - -/** - * @param pMemBuffer - * @param data input data pointer - * @param numOfRows number of rows in data - * @param pModel column format model - * @return number of pages in memory - */ -int16_t tExtMemBufferPut(tExtMemBuffer *pMemBuffer, void *data, int32_t numOfRows); - -/** - * - * @param pMemBuffer - * @return - */ -int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer); - -/** - * - * remove all data that has been put into buffer, including in buffer or - * ext-buffer(disk) - */ -void tExtMemBufferClear(tExtMemBuffer *pMemBuffer); - -/* - * this function should be removed. - * since the flush to disk operation is transparent to client this structure should provide stream operation for data, - * and there is an internal cursor point to the data. - */ -bool tExtMemBufferLoadData(tExtMemBuffer *pMemBuffer, tFilePage *pFilePage, int32_t flushIdx, int32_t pageIdx); - -/** - * - * @param pMemBuffer - * @return - */ -bool tExtMemBufferIsAllDataInMem(tExtMemBuffer *pMemBuffer); - -/** - * - * @param fields - * @param numOfCols - * @param blockCapacity - * @return - */ -SColumnModel *createColumnModel(SSchema *fields, int32_t numOfCols, int32_t blockCapacity); - -/** - * - * @param pSrc - * @return - */ -SColumnModel *cloneColumnModel(SColumnModel *pSrc); - -/** - * - * @param pModel - */ -void destroyColumnModel(SColumnModel *pModel); - -/* - * compress data into consecutive block without hole in data - */ -void tColModelCompact(SColumnModel *pModel, tFilePage *inputBuffer, int32_t maxElemsCapacity); - -void tColModelErase(SColumnModel *pModel, tFilePage *inputBuffer, int32_t maxCapacity, int32_t s, int32_t e); -SSchema *getColumnModelSchema(SColumnModel *pColumnModel, int32_t index); - -int16_t getColumnModelOffset(SColumnModel *pColumnModel, int32_t index); - -typedef struct SSrcColumnInfo { - int32_t functionId; - int32_t type; -} SSrcColumnInfo; - -/* - * display data in column format model for debug purpose only - */ -void tColModelDisplay(SColumnModel *pModel, void *pData, int32_t numOfRows, int32_t maxCount); - -void tColModelDisplayEx(SColumnModel *pModel, void *pData, int32_t numOfRows, int32_t maxCount, SSrcColumnInfo *pInfo); - -tOrderDescriptor *tOrderDesCreate(const int32_t *orderColIdx, int32_t numOfOrderCols, SColumnModel *pModel, - int32_t tsOrderType); - -void tOrderDescDestroy(tOrderDescriptor *pDesc); - -void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn); - -void tColModelAppend(SColumnModel *dstModel, tFilePage *dstPage, void *srcData, int32_t srcStartRows, - int32_t numOfRowsToWrite, int32_t srcCapacity); - -typedef int (*__col_compar_fn_t)(tOrderDescriptor *, int32_t numOfRows, int32_t idx1, int32_t idx2, char *data); - -void tColDataQSort(tOrderDescriptor *, int32_t numOfRows, int32_t start, int32_t end, char *data, int32_t orderType); - -void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn); - -int32_t compare_sa(tOrderDescriptor *, int32_t numOfRows, int32_t idx1, int32_t idx2, char *data); - -int32_t compare_sd(tOrderDescriptor *, int32_t numOfRows, int32_t idx1, int32_t idx2, char *data); - -int32_t compare_a(tOrderDescriptor *, int32_t numOfRow1, int32_t s1, char *data1, int32_t numOfRow2, int32_t s2, - char *data2); - -int32_t compare_d(tOrderDescriptor *, int32_t numOfRow1, int32_t s1, char *data1, int32_t numOfRow2, int32_t s2, - char *data2); - -struct SSDataBlock; -int32_t compare_aRv(struct SSDataBlock* pBlock, SArray* colIndex, int32_t numOfCols, int32_t rowIndex, char** buffer, int32_t order); - -int32_t columnValueAscendingComparator(char *f1, char *f2, int32_t type, int32_t bytes); - -#ifdef __cplusplus -} -#endif - -#endif // TBASE_SORT_H diff --git a/2.0/src/query/inc/qFill.h b/2.0/src/query/inc/qFill.h deleted file mode 100644 index caa0c55b3ff1a805975a6258ca360e5099f4894e..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qFill.h +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QFILL_H -#define TDENGINE_QFILL_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" -#include "qExtbuffer.h" -#include "taosdef.h" - -struct SSDataBlock; - -typedef struct { - STColumn col; // column info - int16_t functionId; // sql function id - int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN - int16_t tagIndex; // index of current tag in SFillTagColInfo array list - union {int64_t i; double d;} fillVal; -} SFillColInfo; - -typedef struct { - SSchema col; - char* tagVal; -} SFillTagColInfo; - -typedef struct SFillInfo { - TSKEY start; // start timestamp - TSKEY end; // endKey for fill - TSKEY currentKey; // current active timestamp, the value may be changed during the fill procedure. - int32_t order; // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC] - int32_t type; // fill type - int32_t numOfRows; // number of rows in the input data block - int32_t index; // active row index - int32_t numOfTotal; // number of filled rows in one round - int32_t numOfCurrent; // number of filled rows in current results - - int32_t numOfTags; // number of tags - int32_t numOfCols; // number of columns, including the tags columns - int32_t rowSize; // size of each row - SInterval interval; - char * prevValues; // previous row of data, to generate the interpolation results - char * nextValues; // next row of data - char** pData; // original result data block involved in filling data - int32_t alloc; // data buffer size in rows - int8_t precision; // time resoluation - - SFillColInfo* pFillCol; // column info for fill operations - SFillTagColInfo* pTags; // tags value for filling gap - void* handle; // for debug purpose -} SFillInfo; - -typedef struct SPoint { - int64_t key; - void * val; -} SPoint; - -SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, - int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - SFillColInfo* pFillCol, void* handle); - -void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp); - -void* taosDestroyFillInfo(SFillInfo *pFillInfo); - -void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); - -void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const struct SSDataBlock* pInput); - -bool taosFillHasMoreResults(SFillInfo* pFillInfo); - -int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); - -int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType); - -int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_QFILL_H diff --git a/2.0/src/query/inc/qHistogram.h b/2.0/src/query/inc/qHistogram.h deleted file mode 100644 index 3b5c2b4cfb9bac638c7d86988f8ac14d7419f83c..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qHistogram.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HISTOGRAM_H -#define TDENGINE_HISTOGRAM_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define USE_ARRAYLIST - -#define MAX_HISTOGRAM_BIN 500 - -typedef struct SHistBin { - double val; - int64_t num; - -#if !defined(USE_ARRAYLIST) - double delta; - int32_t index; // index in min-heap list -#endif -} SHistBin; - -typedef struct SHeapEntry { - void* pData; - double val; -} SHeapEntry; - -typedef struct SHistogramInfo { - int64_t numOfElems; - int32_t numOfEntries; - int32_t maxEntries; - double min; - double max; -#if defined(USE_ARRAYLIST) - SHistBin* elems; -#else - tSkipList* pList; - SLoserTreeInfo* pLoserTree; - int32_t maxIndex; - bool ordered; -#endif -} SHistogramInfo; - -SHistogramInfo* tHistogramCreate(int32_t numOfBins); -SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins); - -int32_t tHistogramAdd(SHistogramInfo** pHisto, double val); -int64_t tHistogramSum(SHistogramInfo* pHisto, double v); - -double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num); -SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries); -void tHistogramDestroy(SHistogramInfo** pHisto); - -void tHistogramPrint(SHistogramInfo* pHisto); - -int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val); - -SHeapEntry* tHeapCreate(int32_t numOfEntries); -void tHeapSort(SHeapEntry* pEntry, int32_t len); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_HISTOGRAM_H diff --git a/2.0/src/query/inc/qPercentile.h b/2.0/src/query/inc/qPercentile.h deleted file mode 100644 index f5b770593c1e30df8c3459676ffa42bbfc9cf0ae..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qPercentile.h +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QPERCENTILE_H -#define TDENGINE_QPERCENTILE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "qExtbuffer.h" -#include "qResultbuf.h" -#include "qTsbuf.h" - -typedef struct MinMaxEntry { - union { - double dMinVal; - int64_t i64MinVal; - uint64_t u64MinVal; - }; - union { - double dMaxVal; - int64_t i64MaxVal; - int64_t u64MaxVal; - }; -} MinMaxEntry; - -typedef struct { - int32_t size; - int32_t pageId; - tFilePage *data; -} SSlotInfo; - -typedef struct tMemBucketSlot { - SSlotInfo info; - MinMaxEntry range; -} tMemBucketSlot; - -struct tMemBucket; -typedef int32_t (*__perc_hash_func_t)(struct tMemBucket *pBucket, const void *value); - -typedef struct tMemBucket { - int16_t numOfSlots; - int16_t type; - int16_t bytes; - int32_t total; - int32_t elemPerPage; // number of elements for each object - int32_t maxCapacity; // maximum allowed number of elements that can be sort directly to get the result - int32_t bufPageSize; // disk page size - MinMaxEntry range; // value range - int32_t times; // count that has been checked for deciding the correct data value buckets. - __compar_fn_t comparFn; - - tMemBucketSlot * pSlots; - SDiskbasedResultBuf *pBuffer; - __perc_hash_func_t hashFunc; -} tMemBucket; - -tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval); - -void tMemBucketDestroy(tMemBucket *pBucket); - -int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size); - -double getPercentile(tMemBucket *pMemBucket, double percent); - -#endif // TDENGINE_QPERCENTILE_H - -#ifdef __cplusplus -} -#endif \ No newline at end of file diff --git a/2.0/src/query/inc/qPlan.h b/2.0/src/query/inc/qPlan.h deleted file mode 100644 index 60a50ca70c32b1aa3f10af4ed19231b77b50f677..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qPlan.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QPLAN_H -#define TDENGINE_QPLAN_H - -#include "qExecutor.h" - -struct SQueryInfo; - -typedef struct SQueryNodeBasicInfo { - int32_t type; - char *name; -} SQueryNodeBasicInfo; - -typedef struct SQueryTableInfo { - char *tableName; - STableId id; -} SQueryTableInfo; - -typedef struct SQueryNode { - SQueryNodeBasicInfo info; - SQueryTableInfo tableInfo; - SSchema *pSchema; // the schema of the input SSDatablock - int32_t numOfCols; // number of input columns - SExprInfo *pExpr; // the query functions or sql aggregations - int32_t numOfOutput; // number of result columns, which is also the number of pExprs - - void *pExtInfo; // additional information - // previous operator to generated result for current node to process - // in case of join, multiple prev nodes exist. - SArray *pPrevNodes;// upstream nodes - struct SQueryNode *nextNode; -} SQueryNode; - -SQueryNode* qCreateQueryPlan(struct SQueryInfo* pQueryInfo); -void* qDestroyQueryPlan(SQueryNode* pQueryNode); - -char* queryPlanToString(SQueryNode* pQueryNode); - -SArray* createTableScanPlan(SQueryAttr* pQueryAttr); -SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr); -SArray* createGlobalMergePlan(SQueryAttr* pQueryAttr); - -#endif // TDENGINE_QPLAN_H diff --git a/2.0/src/query/inc/qResultbuf.h b/2.0/src/query/inc/qResultbuf.h deleted file mode 100644 index ef43a9621a8298fff857a252a1b67a86988d786e..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qResultbuf.h +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QRESULTBUF_H -#define TDENGINE_QRESULTBUF_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include "hash.h" -#include "os.h" -#include "qExtbuffer.h" -#include "tlockfree.h" - -typedef struct SArray* SIDList; - -typedef struct SPageDiskInfo { - int32_t offset; - int32_t length; -} SPageDiskInfo; - -typedef struct SPageInfo { - SListNode* pn; // point to list node - int32_t pageId; - SPageDiskInfo info; - void* pData; - bool used; // set current page is in used -} SPageInfo; - -typedef struct SFreeListItem { - int32_t offset; - int32_t len; -} SFreeListItem; - -typedef struct SResultBufStatis { - int32_t flushBytes; - int32_t loadBytes; - int32_t getPages; - int32_t releasePages; - int32_t flushPages; -} SResultBufStatis; - -typedef struct SDiskbasedResultBuf { - int32_t numOfPages; - int64_t totalBufSize; - int64_t fileSize; // disk file size - FILE* file; - int32_t allocateId; // allocated page id - char* path; // file path - int32_t pageSize; // current used page size - int32_t inMemPages; // numOfPages that are allocated in memory - SHashObj* groupSet; // id hash table - SHashObj* all; - SList* lruList; - void* emptyDummyIdList; // dummy id list - void* assistBuf; // assistant buffer for compress/decompress data - SArray* pFree; // free area in file - bool comp; // compressed before flushed to disk - int32_t nextPos; // next page flush position - - uint64_t qId; // for debug purpose - SResultBufStatis statis; -} SDiskbasedResultBuf; - -#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes -#define PAGE_INFO_INITIALIZER (SPageDiskInfo){-1, -1} - -/** - * create disk-based result buffer - * @param pResultBuf - * @param rowSize - * @param pagesize - * @param inMemPages - * @param handle - * @return - */ -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId); - -/** - * - * @param pResultBuf - * @param groupId - * @param pageId - * @return - */ -tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId); - -/** - * - * @param pResultBuf - * @param groupId - * @return - */ -SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); - -/** - * get the specified buffer page by id - * @param pResultBuf - * @param id - * @return - */ -tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id); - -/** - * release the referenced buf pages - * @param pResultBuf - * @param page - */ -void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page); - -/** - * - * @param pResultBuf - * @param pi - */ -void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi); - - -/** - * get the total buffer size in the format of disk file - * @param pResultBuf - * @return - */ -size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf); - -/** - * get the number of groups in the result buffer - * @param pResultBuf - * @return - */ -size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf); - -/** - * destroy result buffer - * @param pResultBuf - */ -void destroyResultBuf(SDiskbasedResultBuf* pResultBuf); - -/** - * - * @param pList - * @return - */ -SPageInfo* getLastPageInfo(SIDList pList); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_QRESULTBUF_H diff --git a/2.0/src/query/inc/qSqlparser.h b/2.0/src/query/inc/qSqlparser.h deleted file mode 100644 index ecb5c664aaf81d0839539cba241fbd152557a638..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qSqlparser.h +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QSQLPARSER_H -#define TDENGINE_QSQLPARSER_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "../../../include/client/taos.h" -#include "tmsg.h" -#include "tstrbuild.h" -#include "ttoken.h" -#include "tvariant.h" - -#define ParseTOKENTYPE SStrToken - -#define NON_ARITHMEIC_EXPR 0 -#define NORMAL_ARITHMETIC 1 -#define AGG_ARIGHTMEIC 2 - -enum SQL_NODE_TYPE { - SQL_NODE_TABLE_COLUMN= 1, - SQL_NODE_SQLFUNCTION = 2, - SQL_NODE_VALUE = 3, - SQL_NODE_EXPR = 4, -}; - -enum SQL_NODE_FROM_TYPE { - SQL_FROM_NODE_SUBQUERY = 1, - SQL_FROM_NODE_TABLES = 2, -}; - -enum SQL_EXPR_FLAG { - EXPR_FLAG_TS_ERROR = 1, - EXPR_FLAG_NS_TIMESTAMP = 2, - EXPR_FLAG_TIMESTAMP_VAR = 3, -}; - -extern char tTokenTypeSwitcher[13]; - -#define toTSDBType(x) \ - do { \ - if ((x) >= tListLen(tTokenTypeSwitcher)) { \ - (x) = TSDB_DATA_TYPE_BINARY; \ - } else { \ - (x) = tTokenTypeSwitcher[(x)]; \ - } \ - } while (0) - -#define TPARSER_HAS_TOKEN(_t) ((_t).n > 0) -#define TPARSER_SET_NONE_TOKEN(_t) ((_t).n = 0) - -typedef struct SLimitVal { - int64_t limit; - int64_t offset; -} SLimitVal; - -typedef struct SOrderVal { - uint32_t order; - int32_t orderColId; -} SOrderVal; - -typedef struct tVariantListItem { - tVariant pVar; - uint8_t sortOrder; -} tVariantListItem; - -typedef struct SIntervalVal { - int32_t token; - SStrToken interval; - SStrToken offset; -} SIntervalVal; - -typedef struct SSessionWindowVal { - SStrToken col; - SStrToken gap; -} SSessionWindowVal; - -typedef struct SWindowStateVal { - SStrToken col; -} SWindowStateVal; - -struct SRelationInfo; - -typedef struct SSqlNode { - struct SArray *pSelNodeList; // select clause - struct SRelationInfo *from; // from clause SArray - struct tSqlExpr *pWhere; // where clause [optional] - SArray *pGroupby; // groupby clause, only for tags[optional], SArray - SArray *pSortOrder; // orderby [optional], SArray - SArray *fillType; // fill type[optional], SArray - SIntervalVal interval; // (interval, interval_offset) [optional] - SSessionWindowVal sessionVal; // session window [optional] - SWindowStateVal windowstateVal; // window_state(col) [optional] - SStrToken sliding; // sliding window [optional] - SLimitVal limit; // limit offset [optional] - SLimitVal slimit; // group limit offset [optional] - SStrToken sqlstr; // sql string in select clause - struct tSqlExpr *pHaving; // having clause [optional] -} SSqlNode; - -typedef struct SRelElement { - union { - SStrToken tableName; - SArray *pSubquery; - }; - - SStrToken aliasName; -} SRelElement; - -typedef struct SRelationInfo { - int32_t type; // nested query|table name list - SArray *list; // SArray -} SRelationInfo; - -typedef struct SCreatedTableInfo { - SStrToken name; // table name token - SStrToken stableName; // super table name token , for using clause - SArray *pTagNames; // create by using super table, tag name - SArray *pTagVals; // create by using super table, tag value - char *fullname; // table full name - STagData tagdata; // true tag data, super table full name is in STagData - int8_t igExist; // ignore if exists -} SCreatedTableInfo; - -typedef struct SCreateTableSql { - SStrToken name; // table name, create table [name] xxx - int8_t type; // create normal table/from super table/ stream - bool existCheck; - - struct { - SArray *pTagColumns; // SArray - SArray *pColumns; // SArray - } colInfo; - - SArray *childTableInfo; // SArray - SSqlNode *pSelect; -} SCreateTableSql; - -typedef struct SAlterTableInfo { - SStrToken name; - int16_t tableType; - int16_t type; - STagData tagData; - SArray *pAddColumns; // SArray - SArray *varList; // set t=val or: change src dst, SArray -} SAlterTableInfo; - -typedef struct SCreateDbInfo { - SStrToken dbname; - int32_t replica; - int32_t cacheBlockSize; - int32_t maxTablesPerVnode; - int32_t numOfBlocks; - int32_t daysPerFile; - int32_t minRowsPerBlock; - int32_t maxRowsPerBlock; - int32_t fsyncPeriod; - int64_t commitTime; - int32_t walLevel; - int32_t quorum; - int32_t compressionLevel; - SStrToken precision; - bool ignoreExists; - int8_t update; - int8_t cachelast; - SArray *keep; - int8_t dbType; - int16_t partitions; -} SCreateDbInfo; - -typedef struct SCreateFuncInfo { - SStrToken name; - SStrToken path; - int32_t type; - int32_t bufSize; - TAOS_FIELD output; -} SCreateFuncInfo; - - -typedef struct SCreateAcctInfo { - int32_t maxUsers; - int32_t maxDbs; - int32_t maxTimeSeries; - int32_t maxStreams; - int32_t maxPointsPerSecond; - int64_t maxStorage; - int64_t maxQueryTime; - int32_t maxConnections; - SStrToken stat; -} SCreateAcctInfo; - -typedef struct SShowInfo { - uint8_t showType; - SStrToken prefix; - SStrToken pattern; -} SShowInfo; - -typedef struct SUserInfo { - SStrToken user; - SStrToken passwd; - SStrToken privilege; - int16_t type; -} SUserInfo; - -typedef struct SMiscInfo { - SArray *a; // SArray - bool existsCheck; - int16_t dbType; - int16_t tableType; - SUserInfo user; - union { - SCreateDbInfo dbOpt; - SCreateAcctInfo acctOpt; - SCreateFuncInfo funcOpt; - SShowInfo showOpt; - SStrToken id; - }; -} SMiscInfo; - -typedef struct SSqlInfo { - int32_t type; - bool valid; - SArray *list; // todo refactor - char msg[256]; - SArray *funcs; - union { - SCreateTableSql *pCreateTableInfo; - SAlterTableInfo *pAlterInfo; - SMiscInfo *pMiscInfo; - }; -} SSqlInfo; - -typedef struct tSqlExpr { - uint16_t type; // sql node type - uint32_t tokenId; // TK_LE: less than(binary expr) - - // the whole string of the function(col, param), while the function name is kept in exprToken - struct { - SStrToken operand; - struct SArray *paramList; // function parameters list - } Expr; - - int32_t functionId; // function id, todo remove it - SStrToken columnName; // table column info - tVariant value; // the use input value - SStrToken exprToken; // original sql expr string - uint32_t flags; // todo remove it - - struct tSqlExpr *pLeft; // left child - struct tSqlExpr *pRight; // right child -} tSqlExpr; - -// used in select clause. select from xxx -typedef struct tSqlExprItem { - tSqlExpr *pNode; // The list of expressions - char *aliasName; // alias name, null-terminated string - bool distinct; -} tSqlExprItem; - -SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder); -SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index); -SArray *tVariantListAppendToken(SArray *pList, SStrToken *pAliasToken, uint8_t sortOrder); - -SRelationInfo *setTableNameList(SRelationInfo* pFromInfo, SStrToken *pName, SStrToken* pAlias); -void *destroyRelationInfo(SRelationInfo* pFromInfo); -SRelationInfo *addSubqueryElem(SRelationInfo* pRelationInfo, SArray* pSub, SStrToken* pAlias); - -// sql expr leaf node -tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType); -tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType); -SArray *tStrTokenAppend(SArray *pList, SStrToken *pToken); - -tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType); -tSqlExpr *tSqlExprClone(tSqlExpr *pSrc); -void tSqlExprCompact(tSqlExpr** pExpr); -bool tSqlExprIsLeaf(tSqlExpr* pExpr); -bool tSqlExprIsParentOfLeaf(tSqlExpr* pExpr); -void tSqlExprDestroy(tSqlExpr *pExpr); -SArray *tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SStrToken *pDistinct, SStrToken *pToken); -void tSqlExprListDestroy(SArray *pList); - -SSqlNode *tSetQuerySqlNode(SStrToken *pSelectToken, SArray *pSelNodeList, SRelationInfo *pFrom, tSqlExpr *pWhere, - SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *ps, SWindowStateVal *pw, - SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pgLimit, tSqlExpr *pHaving); -int32_t tSqlExprCompare(tSqlExpr *left, tSqlExpr *right); - -SCreateTableSql *tSetCreateTableInfo(SArray *pCols, SArray *pTags, SSqlNode *pSelect, int32_t type); - -SAlterTableInfo *tSetAlterTableInfo(SStrToken *pTableName, SArray *pCols, SArray *pVals, int32_t type, int16_t tableTable); -SCreatedTableInfo createNewChildTableInfo(SStrToken *pTableName, SArray *pTagNames, SArray *pTagVals, SStrToken *pToken, SStrToken* igExists); - -void destroyAllSqlNode(SArray *pSqlNode); -void destroySqlNode(SSqlNode *pSql); -void freeCreateTableInfo(void* p); - -SSqlInfo *setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pTableName, int32_t type); -SArray *setSubclause(SArray *pList, void *pSqlNode); -SArray *appendSelectClause(SArray *pList, void *pSubclause); - -void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pTableNameToken, SStrToken *pIfNotExists); - -void SqlInfoDestroy(SSqlInfo *pInfo); - -void setDCLSqlElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); -void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck,int16_t dbType,int16_t tableType); -void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns); - -void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDbInfo *pDB, SStrToken *pIgExists); - -void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctInfo *pAcctInfo); -void setCreateUserSql(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd); -void setKillSql(SSqlInfo *pInfo, int32_t type, SStrToken *ip); -void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege); - -void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam); - -void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo); -void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo); - -// prefix show db.tables; -void tSetDbName(SStrToken *pCpxName, SStrToken *pDb); - -void tSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType); -void tSetColumnType(TAOS_FIELD *pField, SStrToken *type); - -/** - * - * @param yyp The parser - * @param yymajor The major token code number - * @param yyminor The value for the token - */ -void Parse(void *yyp, int yymajor, ParseTOKENTYPE yyminor, SSqlInfo *); - -/** - * - * @param p The parser to be deleted - * @param freeProc Function used to reclaim memory - */ -void ParseFree(void *p, void (*freeProc)(void *)); - -/** - * - * @param mallocProc The parser allocator - * @return - */ -void *ParseAlloc(void *(*mallocProc)(size_t)); - -/** - * - * @param str sql string - * @return sql ast - */ -SSqlInfo qSqlParse(const char *str); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_QSQLPARSER_H diff --git a/2.0/src/query/inc/qTableMeta.h b/2.0/src/query/inc/qTableMeta.h deleted file mode 100644 index 5c18c5aef6a4c9855b653e89b021d692d6b3d2f5..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qTableMeta.h +++ /dev/null @@ -1,230 +0,0 @@ -#ifndef TDENGINE_QTABLEUTIL_H -#define TDENGINE_QTABLEUTIL_H - -#include "qSqlparser.h" -#include "tfilter.h" -#include "tsdb.h" //todo tsdb should not be here - -typedef struct SFieldInfo { - int16_t numOfOutput; // number of column in result - TAOS_FIELD* final; - SArray *internalField; // SArray -} SFieldInfo; - -typedef struct SCond { - uint64_t uid; - int32_t len; // length of tag query condition data - char * cond; -} SCond; - -typedef struct STblCond { - uint64_t uid; - int16_t idx; //table index - int32_t len; // length of tag query condition data - char * cond; -} STblCond; - - -typedef struct SJoinNode { - uint64_t uid; - int16_t tagColId; - SArray* tsJoin; - SArray* tagJoin; -} SJoinNode; - -typedef struct SJoinInfo { - bool hasJoin; - SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM]; -} SJoinInfo; - -typedef struct STagCond { - // relation between tbname list and query condition, including : TK_AND or TK_OR - int16_t relType; - - // tbname query condition, only support tbname query condition on one table - SCond tbnameCond; - - // join condition, only support two tables join currently - SJoinInfo joinInfo; - - // for different table, the query condition must be seperated - SArray *pCond; -} STagCond; - -typedef struct SGroupbyExpr { - int16_t tableIndex; - SArray* columnInfo; // SArray, group by columns information - int16_t numOfGroupCols; // todo remove it - int16_t orderIndex; // order by column index - int16_t orderType; // order by type: asc/desc -} SGroupbyExpr; - -typedef struct STableComInfo { - uint8_t numOfTags; - uint8_t precision; - int16_t numOfColumns; - int32_t rowSize; -} STableComInfo; - -typedef struct STableMeta { - int32_t vgId; - STableId id; - int8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; // super table name - uint64_t suid; // super table id - int16_t sversion; - int16_t tversion; - STableComInfo tableInfo; - SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info -} STableMeta; - -typedef struct STableMetaInfo { - STableMeta *pTableMeta; // table meta, cached in client side and acquired by name - uint32_t tableMetaSize; - size_t tableMetaCapacity; - SVgroupsInfo *vgroupList; - SArray *pVgroupTables; // SArray - - /* - * 1. keep the vgroup index during the multi-vnode super table projection query - * 2. keep the vgroup index for multi-vnode insertion - */ - int32_t vgroupIndex; - SName name; - char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql - SArray *tagColList; // SArray, involved tag columns -} STableMetaInfo; - -struct SQInfo; // global merge operator -struct SQueryAttr; // query object - -typedef struct STableFilter { - uint64_t uid; - SFilterInfo info; -} STableFilter; - -typedef struct SQueryInfo { - int16_t command; // the command may be different for each subclause, so keep it seperately. - uint32_t type; // query/insert type - STimeWindow window; // the whole query time window - - SInterval interval; // tumble time window - SSessionWindow sessionWindow; // session time window - - SGroupbyExpr groupbyExpr; // groupby tags info - SArray * colList; // SArray - SFieldInfo fieldsInfo; - SArray * exprList; // SArray - SArray * exprList1; // final exprlist in case of arithmetic expression exists - SLimitVal limit; - SLimitVal slimit; - STagCond tagCond; - - SArray * colCond; - - SOrderVal order; - int16_t numOfTables; - int16_t curTableIdx; - STableMetaInfo **pTableMetaInfo; - struct STSBuf *tsBuf; - - int16_t fillType; // final result fill type - int64_t * fillVal; // default value for fill - int32_t numOfFillVal; // fill value size - - char * msg; // pointer to the pCmd->payload to keep error message temporarily - int64_t clauseLimit; // limit for current sub clause - - int64_t prjOffset; // offset value in the original sql expression, only applied at client side - int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit - - int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX - bool distinct; // distinct tag or not - bool onlyHasTagCond; - int32_t round; // 0/1/.... - int32_t bufLen; - char* buf; - - bool udfCopy; - SArray *pUdfInfo; - - struct SQInfo *pQInfo; // global merge operator - struct SQueryAttr *pQueryAttr; // query object - - struct SQueryInfo *sibling; // sibling - SArray *pUpstream; // SArray - struct SQueryInfo *pDownstream; - int32_t havingFieldNum; - bool stableQuery; - bool groupbyColumn; - bool simpleAgg; - bool arithmeticOnAgg; - bool projectionQuery; - bool hasFilter; - bool onlyTagQuery; - bool orderProjectQuery; - bool stateWindow; - bool globalMerge; - bool multigroupResult; -} SQueryInfo; - -/** - * get the number of tags of this table - * @param pTableMeta - * @return - */ -int32_t tscGetNumOfTags(const STableMeta* pTableMeta); - -/** - * get the number of columns of this table - * @param pTableMeta - * @return - */ -int32_t tscGetNumOfColumns(const STableMeta* pTableMeta); - -/** - * get the basic info of this table - * @param pTableMeta - * @return - */ -STableComInfo tscGetTableInfo(const STableMeta* pTableMeta); - -/** - * get the schema - * @param pTableMeta - * @return - */ -SSchema* tscGetTableSchema(const STableMeta* pTableMeta); - -/** - * get the tag schema - * @param pMeta - * @return - */ -SSchema *tscGetTableTagSchema(const STableMeta *pMeta); - -/** - * get the column schema according to the column index - * @param pMeta - * @param colIndex - * @return - */ -SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex); - -/** - * get the column schema according to the column id - * @param pTableMeta - * @param colId - * @return - */ -SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId); - -/** - * create the table meta from the msg - * @param pTableMetaMsg - * @param size size of the table meta - * @return - */ -STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg); - -#endif // TDENGINE_QTABLEUTIL_H diff --git a/2.0/src/query/inc/qTsbuf.h b/2.0/src/query/inc/qTsbuf.h deleted file mode 100644 index 00cc4e897f130348b81a7d96419c1b292cacca8c..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qTsbuf.h +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_STSBUF_H -#define TDENGINE_STSBUF_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" -#include "taosdef.h" -#include "tvariant.h" - -#define MEM_BUF_SIZE (1 << 20) -#define TS_COMP_FILE_MAGIC 0x87F5EC4C -#define TS_COMP_FILE_GROUP_MAX 512 - -typedef struct STSList { - char* rawBuf; - int32_t allocSize; - int32_t threshold; - int32_t len; -} STSList; - -typedef struct STSElem { - TSKEY ts; - tVariant* tag; - int32_t id; -} STSElem; - -typedef struct STSCursor { - int32_t vgroupIndex; - int32_t blockIndex; - int32_t tsIndex; - uint32_t order; -} STSCursor; - -typedef struct STSBlock { - tVariant tag; // tag value - int32_t numOfElem; // number of elements - int32_t compLen; // size after compressed - int32_t padding; // 0xFFFFFFFF by default, after the payload - char* payload; // actual data that is compressed -} STSBlock; - -/* - * The size of buffer file should not be greater than 2G, - * and the offset of int32_t type is enough - */ -typedef struct STSGroupBlockInfo { - int32_t id; // group id - int32_t offset; // offset set value in file - int32_t numOfBlocks; // number of total blocks - int32_t compLen; // compressed size -} STSGroupBlockInfo; - -typedef struct STSGroupBlockInfoEx { - STSGroupBlockInfo info; - int32_t len; // length before compress -} STSGroupBlockInfoEx; - -typedef struct STSBuf { - FILE* f; - char path[PATH_MAX]; - uint32_t fileSize; - - // todo use array - STSGroupBlockInfoEx* pData; - uint32_t numOfAlloc; - uint32_t numOfGroups; - - char* assistBuf; - int32_t bufSize; - STSBlock block; - STSList tsData; // uncompressed raw ts data - uint64_t numOfTotal; - bool autoDelete; - bool remainOpen; - int32_t tsOrder; // order of timestamp in ts comp buffer - STSCursor cur; -} STSBuf; - -typedef struct STSBufFileHeader { - uint32_t magic; // file magic number - uint32_t numOfGroup; // number of group stored in current file - int32_t tsOrder; // timestamp order in current file -} STSBufFileHeader; - -STSBuf* tsBufCreate(bool autoDelete, int32_t order); -STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete); -STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder, int32_t id); - -void* tsBufDestroy(STSBuf* pTSBuf); - -void tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, int32_t len); -int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf); - -STSBuf* tsBufClone(STSBuf* pTSBuf); - -STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id); - -void tsBufFlush(STSBuf* pTSBuf); -void tsBufResetPos(STSBuf* pTSBuf); -bool tsBufNextPos(STSBuf* pTSBuf); - -STSElem tsBufGetElem(STSBuf* pTSBuf); -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag); - -STSCursor tsBufGetCursor(STSBuf* pTSBuf); -void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order); - -void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur); - -/** - * display all data in comp block file, for debug purpose only - * @param pTSBuf - */ -void tsBufDisplay(STSBuf* pTSBuf); - -int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf); - -void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id); - -int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t id, void* buf, int32_t* len, int32_t* numOfBlocks); - -STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag); - -bool tsBufIsValidElem(STSElem* pElem); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_STSBUF_H diff --git a/2.0/src/query/inc/qUdf.h b/2.0/src/query/inc/qUdf.h deleted file mode 100644 index 1083b1e698f7591aae4586c7722e5343cd9c4d86..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/qUdf.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QUDF_H -#define TDENGINE_QUDF_H - -enum { TSDB_UDF_FUNC_NORMAL = 0, TSDB_UDF_FUNC_INIT, TSDB_UDF_FUNC_FINALIZE, TSDB_UDF_FUNC_MERGE, TSDB_UDF_FUNC_DESTROY, TSDB_UDF_FUNC_MAX_NUM }; - - - -typedef struct SUdfInit{ - int32_t maybe_null; /* 1 if function can return NULL */ - uint32_t decimals; /* for real functions */ - uint64_t length; /* For string functions */ - char *ptr; /* free pointer for function data */ - int32_t const_item; /* 0 if result is independent of arguments */ - - // script like lua/javascript - void* script_ctx; - void (*destroyCtxFunc)(void *script_ctx); -} SUdfInit; - - -typedef struct SUdfInfo { - int32_t functionId; // system assigned function id - int32_t funcType; // scalar function or aggregate function - int8_t resType; // result type - int16_t resBytes; // result byte - int32_t contLen; // content length - int32_t bufSize; //interbuf size - char *name; // function name - void *handle; // handle loaded in mem - void *funcs[TSDB_UDF_FUNC_MAX_NUM]; // function ptr - - // for script like lua/javascript only - int isScript; - void *pScriptCtx; - - SUdfInit init; - char *content; - char *path; -} SUdfInfo; - -//script - -typedef int32_t (*scriptInitFunc)(void *pCtx); -typedef void (*scriptNormalFunc)(void *pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows, - int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput, int16_t oType, int16_t oBytes); -typedef void (*scriptFinalizeFunc)(void *pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput); -typedef void (*scriptMergeFunc)(void *pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput); -typedef void (*scriptDestroyFunc)(void* pCtx); - -// dynamic lib -typedef void (*udfNormalFunc)(char* data, int16_t itype, int16_t iBytes, int32_t numOfRows, int64_t* ts, char* dataOutput, char* interBuf, - char* tsOutput, int32_t* numOfOutput, int16_t oType, int16_t oBytes, SUdfInit* buf); -typedef int32_t (*udfInitFunc)(SUdfInit* data); -typedef void (*udfFinalizeFunc)(char* dataOutput, char *interBuf, int32_t* numOfOutput, SUdfInit* buf); -typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf); -typedef void (*udfDestroyFunc)(SUdfInit* buf); - - -#endif // TDENGINE_QUDF_H diff --git a/2.0/src/query/inc/queryLog.h b/2.0/src/query/inc/queryLog.h deleted file mode 100644 index 87a221943a566cae3f873d5859a1a5cc09cf9989..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/queryLog.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QUERY_LOG_H -#define TDENGINE_QUERY_LOG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "tlog.h" - -extern uint32_t qDebugFlag; - -#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qDump(a, l) do { if (qDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)a, l); }} while(0) - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_QUERY_LOG_H diff --git a/2.0/src/query/inc/sql.y b/2.0/src/query/inc/sql.y deleted file mode 100644 index d37aaf5a6d579166978baeb2b4bb7413dcd64edb..0000000000000000000000000000000000000000 --- a/2.0/src/query/inc/sql.y +++ /dev/null @@ -1,931 +0,0 @@ -//lemon parser file to generate sql parse by using finite-state-machine code used to parse sql -//usage: lemon sql.y -%token_prefix TK_ - -%token_type {SStrToken} -%default_type {SStrToken} -%extra_argument {SSqlInfo* pInfo} - -%fallback ID BOOL TINYINT SMALLINT INTEGER BIGINT FLOAT DOUBLE STRING TIMESTAMP BINARY NCHAR. - -%left OR. -%left AND. -%right NOT. -%left EQ NE ISNULL NOTNULL IS LIKE MATCH NMATCH GLOB BETWEEN IN. -%left GT GE LT LE. -%left BITAND BITOR LSHIFT RSHIFT. -%left PLUS MINUS. -%left DIVIDE TIMES. -%left STAR SLASH REM. -%left CONCAT. -%right UMINUS UPLUS BITNOT. - -%include { -#include -#include -#include -#include -#include -#include "qSqlparser.h" -#include "tcmdtype.h" -#include "ttoken.h" -#include "ttokendef.h" -#include "tutil.h" -#include "tvariant.h" -#include "parserInt.h" -} - -%syntax_error { - pInfo->valid = false; - int32_t outputBufLen = tListLen(pInfo->msg); - int32_t len = 0; - - if(TOKEN.z) { - char msg[] = "syntax error near \"%s\""; - int32_t sqlLen = strlen(&TOKEN.z[0]); - - if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > outputBufLen) { - char tmpstr[128] = {0}; - memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1); - len = sprintf(pInfo->msg, msg, tmpstr); - } else { - len = sprintf(pInfo->msg, msg, &TOKEN.z[0]); - } - - } else { - len = sprintf(pInfo->msg, "Incomplete SQL statement"); - } - - assert(len <= outputBufLen); -} - -%parse_accept {} - -program ::= cmd. {} - -//////////////////////////////////THE SHOW STATEMENT/////////////////////////////////////////// -cmd ::= SHOW DATABASES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);} -cmd ::= SHOW TOPICS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);} -cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);} -cmd ::= SHOW MNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);} -cmd ::= SHOW DNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);} -cmd ::= SHOW ACCOUNTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);} -cmd ::= SHOW USERS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_USER, 0, 0);} - -cmd ::= SHOW MODULES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MODULE, 0, 0); } -cmd ::= SHOW QUERIES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_QUERIES, 0, 0); } -cmd ::= SHOW CONNECTIONS.{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONNS, 0, 0);} -cmd ::= SHOW STREAMS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } -cmd ::= SHOW VARIABLES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } -cmd ::= SHOW SCORES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } -cmd ::= SHOW GRANTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); } - -cmd ::= SHOW VNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, 0, 0); } -cmd ::= SHOW VNODES ids(X). { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &X, 0); } - - -%type dbPrefix {SStrToken} -dbPrefix(A) ::=. {A.n = 0; A.type = 0;} -dbPrefix(A) ::= ids(X) DOT. {A = X; } - -%type cpxName {SStrToken} -cpxName(A) ::= . {A.n = 0; } -cpxName(A) ::= DOT ids(Y). {A = Y; A.n += 1; } - -cmd ::= SHOW CREATE TABLE ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_TABLE, 1, &X); -} -cmd ::= SHOW CREATE STABLE ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_STABLE, 1, &X); -} - -cmd ::= SHOW CREATE DATABASE ids(X). { - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_DATABASE, 1, &X); -} - -cmd ::= SHOW dbPrefix(X) TABLES. { - setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &X, 0); -} - -cmd ::= SHOW dbPrefix(X) TABLES LIKE ids(Y). { - setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &X, &Y); -} - -cmd ::= SHOW dbPrefix(X) STABLES. { - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &X, 0); -} - -cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { - SStrToken token; - tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &Y); -} - -cmd ::= SHOW dbPrefix(X) VGROUPS. { - SStrToken token; - tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); -} - -cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). { - SStrToken token; - tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y); -} - -//drop configure for tables -cmd ::= DROP TABLE ifexists(Y) ids(X) cpxName(Z). { - X.n += Z.n; - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, -1); -} - -//drop stable -cmd ::= DROP STABLE ifexists(Y) ids(X) cpxName(Z). { - X.n += Z.n; - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, TSDB_SUPER_TABLE); -} - -cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_DEFAULT, -1); } -cmd ::= DROP TOPIC ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_TOPIC, -1); } -cmd ::= DROP FUNCTION ids(X). { setDropFuncInfo(pInfo, TSDB_SQL_DROP_FUNCTION, &X); } - -cmd ::= DROP DNODE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X); } -cmd ::= DROP USER ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_USER, 1, &X); } -cmd ::= DROP ACCOUNT ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &X); } - -/////////////////////////////////THE USE STATEMENT////////////////////////////////////////// -cmd ::= USE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_USE_DB, 1, &X);} - -/////////////////////////////////THE DESCRIBE STATEMENT///////////////////////////////////// -cmd ::= DESCRIBE ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &X); -} -cmd ::= DESC ids(X) cpxName(Y). { - X.n += Y.n; - setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &X); -} -/////////////////////////////////THE ALTER STATEMENT//////////////////////////////////////// -cmd ::= ALTER USER ids(X) PASS ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &X, &Y, NULL); } -cmd ::= ALTER USER ids(X) PRIVILEGE ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &X, NULL, &Y);} -cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &X, &Y); } -cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } -cmd ::= ALTER LOCAL ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } -cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } -cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} -cmd ::= ALTER TOPIC ids(X) alter_topic_optr(Y). { SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} - -cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} -cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} - -////////////////////////////// COMPACT STATEMENT ////////////////////////////////////////////// - -cmd ::= COMPACT VNODES IN LP exprlist(Y) RP. { setCompactVnodeSql(pInfo, TSDB_SQL_COMPACT_VNODE, Y);} - -// An IDENTIFIER can be a generic identifier, or one of several keywords. -// Any non-standard keyword can also be an identifier. -// And "ids" is an identifer-or-string. -%type ids {SStrToken} -ids(A) ::= ID(X). {A = X; } -ids(A) ::= STRING(X). {A = X; } - -%type ifexists {SStrToken} -ifexists(X) ::= IF EXISTS. { X.n = 1;} -ifexists(X) ::= . { X.n = 0;} - -%type ifnotexists {SStrToken} -ifnotexists(X) ::= IF NOT EXISTS. { X.n = 1;} -ifnotexists(X) ::= . { X.n = 0;} - -/////////////////////////////////THE CREATE STATEMENT/////////////////////////////////////// -//create option for dnode/db/user/account -cmd ::= CREATE DNODE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &X);} -cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). - { setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);} -cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} -cmd ::= CREATE TOPIC ifnotexists(Z) ids(X) topic_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} -cmd ::= CREATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B). { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 1);} -cmd ::= CREATE AGGREGATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B). { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 2);} -cmd ::= CREATE USER ids(X) PASS ids(Y). { setCreateUserSql(pInfo, &X, &Y);} - -bufsize(Y) ::= . { Y.n = 0; } -bufsize(Y) ::= BUFSIZE INTEGER(X). { Y = X; } - -pps(Y) ::= . { Y.n = 0; } -pps(Y) ::= PPS INTEGER(X). { Y = X; } - -tseries(Y) ::= . { Y.n = 0; } -tseries(Y) ::= TSERIES INTEGER(X). { Y = X; } - -dbs(Y) ::= . { Y.n = 0; } -dbs(Y) ::= DBS INTEGER(X). { Y = X; } - -streams(Y) ::= . { Y.n = 0; } -streams(Y) ::= STREAMS INTEGER(X). { Y = X; } - -storage(Y) ::= . { Y.n = 0; } -storage(Y) ::= STORAGE INTEGER(X). { Y = X; } - -qtime(Y) ::= . { Y.n = 0; } -qtime(Y) ::= QTIME INTEGER(X). { Y = X; } - -users(Y) ::= . { Y.n = 0; } -users(Y) ::= USERS INTEGER(X). { Y = X; } - -conns(Y) ::= . { Y.n = 0; } -conns(Y) ::= CONNS INTEGER(X). { Y = X; } - -state(Y) ::= . { Y.n = 0; } -state(Y) ::= STATE ids(X). { Y = X; } - -%type acct_optr {SCreateAcctInfo} -acct_optr(Y) ::= pps(C) tseries(D) storage(P) streams(F) qtime(Q) dbs(E) users(K) conns(L) state(M). { - Y.maxUsers = (K.n>0)?atoi(K.z):-1; - Y.maxDbs = (E.n>0)?atoi(E.z):-1; - Y.maxTimeSeries = (D.n>0)?atoi(D.z):-1; - Y.maxStreams = (F.n>0)?atoi(F.z):-1; - Y.maxPointsPerSecond = (C.n>0)?atoi(C.z):-1; - Y.maxStorage = (P.n>0)?strtoll(P.z, NULL, 10):-1; - Y.maxQueryTime = (Q.n>0)?strtoll(Q.z, NULL, 10):-1; - Y.maxConnections = (L.n>0)?atoi(L.z):-1; - Y.stat = M; -} - -%type intitemlist {SArray*} -%destructor intitemlist {taosArrayDestroy($$);} - -%type intitem {tVariant} -intitemlist(A) ::= intitemlist(X) COMMA intitem(Y). { A = tVariantListAppend(X, &Y, -1); } -intitemlist(A) ::= intitem(X). { A = tVariantListAppend(NULL, &X, -1); } - -intitem(A) ::= INTEGER(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); } - -%type keep {SArray*} -%destructor keep {taosArrayDestroy($$);} -keep(Y) ::= KEEP intitemlist(X). { Y = X; } - -cache(Y) ::= CACHE INTEGER(X). { Y = X; } -replica(Y) ::= REPLICA INTEGER(X). { Y = X; } -quorum(Y) ::= QUORUM INTEGER(X). { Y = X; } -days(Y) ::= DAYS INTEGER(X). { Y = X; } -minrows(Y) ::= MINROWS INTEGER(X). { Y = X; } -maxrows(Y) ::= MAXROWS INTEGER(X). { Y = X; } -blocks(Y) ::= BLOCKS INTEGER(X). { Y = X; } -ctime(Y) ::= CTIME INTEGER(X). { Y = X; } -wal(Y) ::= WAL INTEGER(X). { Y = X; } -fsync(Y) ::= FSYNC INTEGER(X). { Y = X; } -comp(Y) ::= COMP INTEGER(X). { Y = X; } -prec(Y) ::= PRECISION STRING(X). { Y = X; } -update(Y) ::= UPDATE INTEGER(X). { Y = X; } -cachelast(Y) ::= CACHELAST INTEGER(X). { Y = X; } -partitions(Y) ::= PARTITIONS INTEGER(X). { Y = X; } - -%type db_optr {SCreateDbInfo} -db_optr(Y) ::= . {setDefaultCreateDbOption(&Y); Y.dbType = TSDB_DB_TYPE_DEFAULT;} - -db_optr(Y) ::= db_optr(Z) cache(X). { Y = Z; Y.cacheBlockSize = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) replica(X). { Y = Z; Y.replica = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) quorum(X). { Y = Z; Y.quorum = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) days(X). { Y = Z; Y.daysPerFile = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) minrows(X). { Y = Z; Y.minRowsPerBlock = strtod(X.z, NULL); } -db_optr(Y) ::= db_optr(Z) maxrows(X). { Y = Z; Y.maxRowsPerBlock = strtod(X.z, NULL); } -db_optr(Y) ::= db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) ctime(X). { Y = Z; Y.commitTime = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; } -db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } -db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } -db_optr(Y) ::= db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } - -%type topic_optr {SCreateDbInfo} - -//topic_optr(Y) ::= db_optr(Z). { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; } -//topic_optr(Y) ::= topic_optr(Z) partitions(X). { Y = Z; Y.partitions = strtol(X.z, NULL, 10); } - -%type alter_db_optr {SCreateDbInfo} -alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y); Y.dbType = TSDB_DB_TYPE_DEFAULT;} - -alter_db_optr(Y) ::= alter_db_optr(Z) replica(X). { Y = Z; Y.replica = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) quorum(X). { Y = Z; Y.quorum = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) keep(X). { Y = Z; Y.keep = X; } -alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } - -// dynamically update the following two parameters are not allowed. -//alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } -//alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } not support yet - -%type alter_topic_optr {SCreateDbInfo} - -alter_topic_optr(Y) ::= alter_db_optr(Z). { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; } -alter_topic_optr(Y) ::= alter_topic_optr(Z) partitions(X). { Y = Z; Y.partitions = strtol(X.z, NULL, 10); } - -%type typename {SField} -typename(A) ::= ids(X). { - X.type = 0; - tSetColumnType (&A, &X); -} - -//define binary type, e.g., binary(10), nchar(10) -typename(A) ::= ids(X) LP signed(Y) RP. { - if (Y <= 0) { - X.type = 0; - tSetColumnType(&A, &X); - } else { - X.type = -Y; // negative value of name length - tSetColumnType(&A, &X); - } -} - -// define the unsigned number type -typename(A) ::= ids(X) UNSIGNED(Z). { - X.type = 0; - X.n = ((Z.z + Z.n) - X.z); - tSetColumnType (&A, &X); -} - -%type signed {int64_t} -signed(A) ::= INTEGER(X). { A = strtol(X.z, NULL, 10); } -signed(A) ::= PLUS INTEGER(X). { A = strtol(X.z, NULL, 10); } -signed(A) ::= MINUS INTEGER(X). { A = -strtol(X.z, NULL, 10);} - -////////////////////////////////// The CREATE TABLE statement /////////////////////////////// -cmd ::= CREATE TABLE create_table_args. {} -cmd ::= CREATE TABLE create_stable_args. {} -cmd ::= CREATE STABLE create_stable_args. {} -cmd ::= CREATE TABLE create_table_list(Z). { pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = Z;} - -%type create_table_list{SCreateTableSql*} -%destructor create_table_list{destroyCreateTableSql($$);} -create_table_list(A) ::= create_from_stable(Z). { - SCreateTableSql* pCreateTable = calloc(1, sizeof(SCreateTableSql)); - pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); - - taosArrayPush(pCreateTable->childTableInfo, &Z); - pCreateTable->type = TSQL_CREATE_TABLE_FROM_STABLE; - A = pCreateTable; -} - -create_table_list(A) ::= create_table_list(X) create_from_stable(Z). { - taosArrayPush(X->childTableInfo, &Z); - A = X; -} - -%type create_table_args{SCreateTableSql*} -create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. { - A = tSetCreateTableInfo(X, NULL, NULL, TSQL_CREATE_TABLE); - setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); - - V.n += Z.n; - setCreatedTableName(pInfo, &V, &U); -} - -// create super table -%type create_stable_args{SCreateTableSql*} -create_stable_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP TAGS LP columnlist(Y) RP. { - A = tSetCreateTableInfo(X, Y, NULL, TSQL_CREATE_STABLE); - setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); - - V.n += Z.n; - setCreatedTableName(pInfo, &V, &U); -} - -// create table by using super table -// create table table_name using super_table_name tags(tag_values1, tag_values2) -%type create_from_stable{SCreatedTableInfo} -create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) TAGS LP tagitemlist(Y) RP. { - X.n += F.n; - V.n += Z.n; - A = createNewChildTableInfo(&X, NULL, Y, &V, &U); -} - -create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) LP tagNamelist(P) RP TAGS LP tagitemlist(Y) RP. { - X.n += F.n; - V.n += Z.n; - A = createNewChildTableInfo(&X, P, Y, &V, &U); -} - -%type tagNamelist{SArray*} -%destructor tagNamelist {taosArrayDestroy($$);} -tagNamelist(A) ::= tagNamelist(X) COMMA ids(Y). {taosArrayPush(X, &Y); A = X; } -tagNamelist(A) ::= ids(X). {A = taosArrayInit(4, sizeof(SStrToken)); taosArrayPush(A, &X);} - -// create stream -// create table table_name as select count(*) from super_table_name interval(time) -create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) AS select(S). { - A = tSetCreateTableInfo(NULL, NULL, S, TSQL_CREATE_STREAM); - setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); - - V.n += Z.n; - setCreatedTableName(pInfo, &V, &U); -} - -%type column{SField} -%type columnlist{SArray*} -%destructor columnlist {taosArrayDestroy($$);} -columnlist(A) ::= columnlist(X) COMMA column(Y). {taosArrayPush(X, &Y); A = X; } -columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(SField)); taosArrayPush(A, &X);} - -// The information used for a column is the name and type of column: -// tinyint smallint int bigint float double bool timestamp binary(x) nchar(x) -column(A) ::= ids(X) typename(Y). { - tSetColumnInfo(&A, &X, &Y); -} - -%type tagitemlist1 {SArray*} -%destructor tagitemlist1 {taosArrayDestroy($$);} - -%type tagitem1 {SToken} - - -%type tagitemlist {SArray*} -%destructor tagitemlist {taosArrayDestroy($$);} - -%type tagitem {tVariant} -tagitemlist(A) ::= tagitemlist(X) COMMA tagitem(Y). { A = tVariantListAppend(X, &Y, -1); } -tagitemlist(A) ::= tagitem(X). { A = tVariantListAppend(NULL, &X, -1); } - -tagitem(A) ::= INTEGER(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); } -tagitem(A) ::= FLOAT(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); } -tagitem(A) ::= STRING(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); } -tagitem(A) ::= BOOL(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); } -tagitem(A) ::= NULL(X). { X.type = 0; taosVariantCreate(&A, &X); } -tagitem(A) ::= NOW(X). { X.type = TSDB_DATA_TYPE_TIMESTAMP; taosVariantCreate(&A, &X);} - -tagitem(A) ::= MINUS(X) INTEGER(Y).{ - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, &X); -} - -tagitem(A) ::= MINUS(X) FLOAT(Y). { - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, &X); -} - -tagitem(A) ::= PLUS(X) INTEGER(Y). { - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, &X); -} - -tagitem(A) ::= PLUS(X) FLOAT(Y). { - X.n += Y.n; - X.type = Y.type; - toTSDBType(X.type); - taosVariantCreate(&A, &X); -} - -//////////////////////// The SELECT statement ///////////////////////////////// -%type select {SSqlNode*} -%destructor select {destroySqlNode($$);} -select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_option(K) sliding_opt(S) session_option(H) windowstate_option(D) fill_opt(F)groupby_opt(P) having_opt(N) orderby_opt(Z) slimit_opt(G) limit_opt(L). { - A = tSetQuerySqlNode(&T, W, X, Y, P, Z, &K, &H, &D, &S, F, &L, &G, N); -} - -select(A) ::= LP select(B) RP. {A = B;} - -%type union {SArray*} -%destructor union {destroyAllSqlNode($$);} -union(Y) ::= select(X). { Y = setSubclause(NULL, X); } -union(Y) ::= union(Z) UNION ALL select(X). { Y = appendSelectClause(Z, X); } - -cmd ::= union(X). { setSqlInfo(pInfo, X, NULL, TSDB_SQL_SELECT); } - -// Support for the SQL exprssion without from & where subclauses, e.g., -// select database() -// select server_version() -// select client_version() -// select server_state() -select(A) ::= SELECT(T) selcollist(W). { - A = tSetQuerySqlNode(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); -} - -// selcollist is a list of expressions that are to become the return -// values of the SELECT statement. The "*" in statements like -// "SELECT * FROM ..." is encoded as a special expression with an opcode of TK_ALL. -%type selcollist {SArray*} -%destructor selcollist {tSqlExprListDestroy($$);} - -%type sclp {SArray*} -%destructor sclp {tSqlExprListDestroy($$);} -sclp(A) ::= selcollist(X) COMMA. {A = X;} -sclp(A) ::= . {A = 0;} -selcollist(A) ::= sclp(P) distinct(Z) expr(X) as(Y). { - A = tSqlExprListAppend(P, X, Z.n? &Z:0, Y.n?&Y:0); -} - -selcollist(A) ::= sclp(P) STAR. { - tSqlExpr *pNode = tSqlExprCreateIdValue(NULL, TK_ALL); - A = tSqlExprListAppend(P, pNode, 0, 0); -} - -// An option "AS " phrase that can follow one of the expressions that -// define the result set, or one of the tables in the FROM clause. -%type as {SStrToken} -as(X) ::= AS ids(Y). { X = Y; } -as(X) ::= ids(Y). { X = Y; } -as(X) ::= . { X.n = 0; } - -%type distinct {SStrToken} -distinct(X) ::= DISTINCT(Y). { X = Y; } -distinct(X) ::= . { X.n = 0;} - -// A complete FROM clause. -%type from {SRelationInfo*} -%destructor from {destroyRelationInfo($$);} -from(A) ::= FROM tablelist(X). {A = X;} -from(A) ::= FROM sub(X). {A = X;} - -%type sub {SRelationInfo*} -%destructor sub {destroyRelationInfo($$);} -sub(A) ::= LP union(Y) RP. {A = addSubqueryElem(NULL, Y, NULL);} -sub(A) ::= LP union(Y) RP ids(Z). {A = addSubqueryElem(NULL, Y, &Z);} -sub(A) ::= sub(X) COMMA LP union(Y) RP ids(Z).{A = addSubqueryElem(X, Y, &Z);} - -%type tablelist {SRelationInfo*} -%destructor tablelist {destroyRelationInfo($$);} -tablelist(A) ::= ids(X) cpxName(Y). { - X.n += Y.n; - A = setTableNameList(NULL, &X, NULL); -} - -tablelist(A) ::= ids(X) cpxName(Y) ids(Z). { - X.n += Y.n; - A = setTableNameList(NULL, &X, &Z); -} - -tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). { - X.n += Z.n; - A = setTableNameList(Y, &X, NULL); -} - -tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { - X.n += Z.n; - A = setTableNameList(Y, &X, &F); -} - -// The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" -%type tmvar {SStrToken} -tmvar(A) ::= VARIABLE(X). {A = X;} - -%type interval_option {SIntervalVal} -interval_option(N) ::= intervalKey(A) LP tmvar(E) RP. {N.interval = E; N.offset.n = 0; N.token = A;} -interval_option(N) ::= intervalKey(A) LP tmvar(E) COMMA tmvar(X) RP. {N.interval = E; N.offset = X; N.token = A;} -interval_option(N) ::= . {memset(&N, 0, sizeof(N));} - -%type intervalKey {int32_t} -intervalKey(A) ::= INTERVAL. {A = TK_INTERVAL;} -intervalKey(A) ::= EVERY. {A = TK_EVERY; } - -%type session_option {SSessionWindowVal} -session_option(X) ::= . {X.col.n = 0; X.gap.n = 0;} -session_option(X) ::= SESSION LP ids(V) cpxName(Z) COMMA tmvar(Y) RP. { - V.n += Z.n; - X.col = V; - X.gap = Y; -} - -%type windowstate_option {SWindowStateVal} -windowstate_option(X) ::= . { X.col.n = 0; X.col.z = NULL;} -windowstate_option(X) ::= STATE_WINDOW LP ids(V) RP. { X.col = V; } - -%type fill_opt {SArray*} -%destructor fill_opt {taosArrayDestroy($$);} -fill_opt(N) ::= . { N = 0; } -fill_opt(N) ::= FILL LP ID(Y) COMMA tagitemlist(X) RP. { - tVariant A = {0}; - toTSDBType(Y.type); - taosVariantCreate(&A, &Y); - - tVariantListInsert(X, &A, -1, 0); - N = X; -} - -fill_opt(N) ::= FILL LP ID(Y) RP. { - toTSDBType(Y.type); - N = tVariantListAppendToken(NULL, &Y, -1); -} - -%type sliding_opt {SStrToken} -sliding_opt(K) ::= SLIDING LP tmvar(E) RP. {K = E; } -sliding_opt(K) ::= . {K.n = 0; K.z = NULL; K.type = 0; } - -%type orderby_opt {SArray*} -%destructor orderby_opt {taosArrayDestroy($$);} - -%type sortlist {SArray*} -%destructor sortlist {taosArrayDestroy($$);} - -%type sortitem {tVariant} -%destructor sortitem {tVariantDestroy(&$$);} - -orderby_opt(A) ::= . {A = 0;} -orderby_opt(A) ::= ORDER BY sortlist(X). {A = X;} - -sortlist(A) ::= sortlist(X) COMMA item(Y) sortorder(Z). { - A = tVariantListAppend(X, &Y, Z); -} - -sortlist(A) ::= item(Y) sortorder(Z). { - A = tVariantListAppend(NULL, &Y, Z); -} - -%type item {tVariant} -item(A) ::= ids(X) cpxName(Y). { - toTSDBType(X.type); - X.n += Y.n; - - taosVariantCreate(&A, &X); -} - -%type sortorder {int} -sortorder(A) ::= ASC. { A = TSDB_ORDER_ASC; } -sortorder(A) ::= DESC. { A = TSDB_ORDER_DESC;} -sortorder(A) ::= . { A = TSDB_ORDER_ASC; } // Ascending order by default - -//group by clause -%type groupby_opt {SArray*} -%destructor groupby_opt {taosArrayDestroy($$);} -%type grouplist {SArray*} -%destructor grouplist {taosArrayDestroy($$);} - -groupby_opt(A) ::= . { A = 0;} -groupby_opt(A) ::= GROUP BY grouplist(X). { A = X;} - -grouplist(A) ::= grouplist(X) COMMA item(Y). { - A = tVariantListAppend(X, &Y, -1); -} - -grouplist(A) ::= item(X). { - A = tVariantListAppend(NULL, &X, -1); -} - -//having clause, ignore the input condition in having -%type having_opt {tSqlExpr*} -%destructor having_opt {tSqlExprDestroy($$);} -having_opt(A) ::=. {A = 0;} -having_opt(A) ::= HAVING expr(X). {A = X;} - -//limit-offset subclause -%type limit_opt {SLimitVal} -limit_opt(A) ::= . {A.limit = -1; A.offset = 0;} -limit_opt(A) ::= LIMIT signed(X). {A.limit = X; A.offset = 0;} -limit_opt(A) ::= LIMIT signed(X) OFFSET signed(Y). - { A.limit = X; A.offset = Y;} -limit_opt(A) ::= LIMIT signed(X) COMMA signed(Y). - { A.limit = Y; A.offset = X;} - -%type slimit_opt {SLimitVal} -slimit_opt(A) ::= . {A.limit = -1; A.offset = 0;} -slimit_opt(A) ::= SLIMIT signed(X). {A.limit = X; A.offset = 0;} -slimit_opt(A) ::= SLIMIT signed(X) SOFFSET signed(Y). - {A.limit = X; A.offset = Y;} -slimit_opt(A) ::= SLIMIT signed(X) COMMA signed(Y). - {A.limit = Y; A.offset = X;} - -%type where_opt {tSqlExpr*} -%destructor where_opt {tSqlExprDestroy($$);} - -where_opt(A) ::= . {A = 0;} -where_opt(A) ::= WHERE expr(X). {A = X;} - -/////////////////////////// Expression Processing ///////////////////////////// -// -%type expr {tSqlExpr*} -%destructor expr {tSqlExprDestroy($$);} - -expr(A) ::= LP(X) expr(Y) RP(Z). {A = Y; A->exprToken.z = X.z; A->exprToken.n = (Z.z - X.z + 1);} - -expr(A) ::= ID(X). { A = tSqlExprCreateIdValue(&X, TK_ID);} -expr(A) ::= ID(X) DOT ID(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(&X, TK_ID);} -expr(A) ::= ID(X) DOT STAR(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(&X, TK_ALL);} - -expr(A) ::= INTEGER(X). { A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= MINUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= PLUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(&X, TK_INTEGER);} -expr(A) ::= FLOAT(X). { A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= MINUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= PLUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(&X, TK_FLOAT);} -expr(A) ::= STRING(X). { A = tSqlExprCreateIdValue(&X, TK_STRING);} -expr(A) ::= NOW(X). { A = tSqlExprCreateIdValue(&X, TK_NOW); } -expr(A) ::= VARIABLE(X). { A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= PLUS(X) VARIABLE(Y). { X.n += Y.n; X.type = TK_VARIABLE; A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= MINUS(X) VARIABLE(Y). { X.n += Y.n; X.type = TK_VARIABLE; A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} -expr(A) ::= BOOL(X). { A = tSqlExprCreateIdValue(&X, TK_BOOL);} -expr(A) ::= NULL(X). { A = tSqlExprCreateIdValue(&X, TK_NULL);} - -// ordinary functions: min(x), max(x), top(k, 20) -expr(A) ::= ID(X) LP exprlist(Y) RP(E). { (pInfo->funcs, &X); A = tSqlExprCreateFunction(Y, &X, &E, X.type); } - -// for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation -expr(A) ::= ID(X) LP STAR RP(Y). { tStrTokenAppend(pInfo->funcs, &X); A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); } - -// is (not) null expression -expr(A) ::= expr(X) IS NULL. {A = tSqlExprCreate(X, NULL, TK_ISNULL);} -expr(A) ::= expr(X) IS NOT NULL. {A = tSqlExprCreate(X, NULL, TK_NOTNULL);} - -// relational expression -expr(A) ::= expr(X) LT expr(Y). {A = tSqlExprCreate(X, Y, TK_LT);} -expr(A) ::= expr(X) GT expr(Y). {A = tSqlExprCreate(X, Y, TK_GT);} -expr(A) ::= expr(X) LE expr(Y). {A = tSqlExprCreate(X, Y, TK_LE);} -expr(A) ::= expr(X) GE expr(Y). {A = tSqlExprCreate(X, Y, TK_GE);} -expr(A) ::= expr(X) NE expr(Y). {A = tSqlExprCreate(X, Y, TK_NE);} -expr(A) ::= expr(X) EQ expr(Y). {A = tSqlExprCreate(X, Y, TK_EQ);} - -expr(A) ::= expr(X) BETWEEN expr(Y) AND expr(Z). { tSqlExpr* X2 = tSqlExprClone(X); A = tSqlExprCreate(tSqlExprCreate(X, Y, TK_GE), tSqlExprCreate(X2, Z, TK_LE), TK_AND);} - -expr(A) ::= expr(X) AND expr(Y). {A = tSqlExprCreate(X, Y, TK_AND);} -expr(A) ::= expr(X) OR expr(Y). {A = tSqlExprCreate(X, Y, TK_OR); } - -// binary arithmetic expression -expr(A) ::= expr(X) PLUS expr(Y). {A = tSqlExprCreate(X, Y, TK_PLUS); } -expr(A) ::= expr(X) MINUS expr(Y). {A = tSqlExprCreate(X, Y, TK_MINUS); } -expr(A) ::= expr(X) STAR expr(Y). {A = tSqlExprCreate(X, Y, TK_STAR); } -expr(A) ::= expr(X) SLASH expr(Y). {A = tSqlExprCreate(X, Y, TK_DIVIDE);} -expr(A) ::= expr(X) REM expr(Y). {A = tSqlExprCreate(X, Y, TK_REM); } - -// like expression -expr(A) ::= expr(X) LIKE expr(Y). {A = tSqlExprCreate(X, Y, TK_LIKE); } - -// match expression -expr(A) ::= expr(X) MATCH expr(Y). {A = tSqlExprCreate(X, Y, TK_MATCH); } -expr(A) ::= expr(X) NMATCH expr(Y). {A = tSqlExprCreate(X, Y, TK_NMATCH); } - -//in expression -expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSqlExprCreate(X, (tSqlExpr*)Y, TK_IN); } - -%type exprlist {SArray*} -%destructor exprlist {tSqlExprListDestroy($$);} - -%type expritem {tSqlExpr*} -%destructor expritem {tSqlExprDestroy($$);} - -exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSqlExprListAppend(X,Y,0, 0);} -exprlist(A) ::= expritem(X). {A = tSqlExprListAppend(0,X,0, 0);} -expritem(A) ::= expr(X). {A = X;} -expritem(A) ::= . {A = 0;} - -///////////////////////////////////reset query cache////////////////////////////////////// -cmd ::= RESET QUERY CACHE. { setDCLSqlElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} - -///////////////////////////////////sync replica database////////////////////////////////// -cmd ::= SYNCDB ids(X) REPLICA.{ setDCLSqlElems(pInfo, TSDB_SQL_SYNC_DB_REPLICA, 1, &X);} - -///////////////////////////////////ALTER TABLE statement////////////////////////////////// -cmd ::= ALTER TABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). { - X.n += F.n; - - toTSDBType(A.type); - SArray* K = tVariantListAppendToken(NULL, &A, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) MODIFY COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -//////////////////////////////////ALTER TAGS statement///////////////////////////////////// -cmd ::= ALTER TABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { - X.n += Y.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). { - X.n += Z.n; - - toTSDBType(Y.type); - SArray* A = tVariantListAppendToken(NULL, &Y, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tVariantListAppendToken(NULL, &Y, -1); - - toTSDBType(Z.type); - A = tVariantListAppendToken(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tVariantListAppendToken(NULL, &Y, -1); - A = tVariantListAppend(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER TABLE ids(X) cpxName(F) MODIFY TAG columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -///////////////////////////////////ALTER STABLE statement////////////////////////////////// -cmd ::= ALTER STABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) DROP COLUMN ids(A). { - X.n += F.n; - - toTSDBType(A.type); - SArray* K = tVariantListAppendToken(NULL, &A, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) MODIFY COLUMN columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -//////////////////////////////////ALTER TAGS statement///////////////////////////////////// -cmd ::= ALTER STABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { - X.n += Y.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -cmd ::= ALTER STABLE ids(X) cpxName(Z) DROP TAG ids(Y). { - X.n += Z.n; - - toTSDBType(Y.type); - SArray* A = tVariantListAppendToken(NULL, &Y, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tVariantListAppendToken(NULL, &Y, -1); - - toTSDBType(Z.type); - A = tVariantListAppendToken(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { - X.n += F.n; - - toTSDBType(Y.type); - SArray* A = tVariantListAppendToken(NULL, &Y, -1); - A = tVariantListAppend(A, &Z, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -cmd ::= ALTER STABLE ids(X) cpxName(F) MODIFY TAG columnlist(A). { - X.n += F.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} - -////////////////////////////////////////kill statement/////////////////////////////////////// -cmd ::= KILL CONNECTION INTEGER(Y). {setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &Y);} -cmd ::= KILL STREAM INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &X);} -cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &X);} - -%fallback ID ABORT AFTER ASC ATTACH BEFORE BEGIN CASCADE CLUSTER CONFLICT COPY DATABASE DEFERRED - DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD - LIKE MATCH NMATCH KEY OF OFFSET RAISE REPLACE RESTRICT ROW STATEMENT TRIGGER VIEW ALL - NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT TBNAME JOIN STABLE NULL INSERT INTO VALUES. diff --git a/2.0/src/query/src/qAggMain.c b/2.0/src/query/src/qAggMain.c deleted file mode 100644 index 7ab38555062547ec604f3811f220867633f2ea3c..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qAggMain.c +++ /dev/null @@ -1,4691 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "taosdef.h" -#include "tmsg.h" -#include "texpr.h" -#include "ttype.h" -#include "tsdb.h" -#include "tglobal.h" - -#include "qAggMain.h" -#include "qFill.h" -#include "qHistogram.h" -#include "qPercentile.h" -#include "qTsbuf.h" -#include "queryLog.h" -#include "qUdf.h" - -#define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput)) -#define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes) - -#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) -#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) - -#define GET_TRUE_DATA_TYPE() \ - int32_t type = 0; \ - if (pCtx->currentStage == MERGE_STAGE) { \ - type = pCtx->outputType; \ - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); \ - } else { \ - type = pCtx->inputType; \ - } - -#define SET_VAL(ctx, numOfElem, res) \ - do { \ - if ((numOfElem) <= 0) { \ - break; \ - } \ - GET_RES_INFO(ctx)->numOfRes = (res); \ - } while (0) - -#define INC_INIT_VAL(ctx, res) (GET_RES_INFO(ctx)->numOfRes += (res)); - -#define DO_UPDATE_TAG_COLUMNS(ctx, ts) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { \ - __ctx->tag.i64 = (ts); \ - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ - } \ - aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ - } \ - } while (0) - -#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ - } \ - } while (0); - -void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} - -void doFinalizer(SQLFunctionCtx *pCtx) { RESET_RESULT_INFO(GET_RES_INFO(pCtx)); } - -typedef struct tValuePair { - tVariant v; - int64_t timestamp; - char * pTags; // the corresponding tags of each record in the final result -} tValuePair; - -typedef struct SSpreadInfo { - double min; - double max; - int8_t hasResult; -} SSpreadInfo; - -typedef struct SSumInfo { - union { - int64_t isum; - uint64_t usum; - double dsum; - }; - int8_t hasResult; -} SSumInfo; - -// the attribute of hasResult is not needed since the num attribute would server as this purpose -typedef struct SAvgInfo { - double sum; - int64_t num; -} SAvgInfo; - -typedef struct SStddevInfo { - double avg; - int64_t num; - double res; - int8_t stage; -} SStddevInfo; - -typedef struct SStddevdstInfo { - int64_t num; - double res; -} SStddevdstInfo; - -typedef struct SFirstLastInfo { - int8_t hasResult; - TSKEY ts; -} SFirstLastInfo; - -typedef struct SFirstLastInfo SLastrowInfo; -typedef struct SPercentileInfo { - tMemBucket *pMemBucket; - int32_t stage; - double minval; - double maxval; - int64_t numOfElems; -} SPercentileInfo; - -typedef struct STopBotInfo { - int32_t num; - tValuePair **res; -} STopBotInfo; - -// leastsquares do not apply to super table -typedef struct SLeastsquaresInfo { - double mat[2][3]; - double startVal; - int64_t num; -} SLeastsquaresInfo; - -typedef struct SAPercentileInfo { - SHistogramInfo *pHisto; -} SAPercentileInfo; - -typedef struct STSCompInfo { - STSBuf *pTSBuf; -} STSCompInfo; - -typedef struct SRateInfo { - double correctionValue; - double firstValue; - TSKEY firstKey; - double lastValue; - TSKEY lastKey; - int8_t hasResult; // flag to denote has value - bool isIRate; // true for IRate functions, false for Rate functions -} SRateInfo; - -typedef struct SDerivInfo { - double prevValue; // previous value - TSKEY prevTs; // previous timestamp - bool ignoreNegative;// ignore the negative value - int64_t tsWindow; // time window for derivative - bool valueSet; // the value has been set already -} SDerivInfo; - -int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, - int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable, SUdfInfo* pUdfInfo) { - if (!isValidDataType(dataType)) { - qError("Illegal data type %d or data type length %d", dataType, dataBytes); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG_DUMMY || - functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAGPRJ || - functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) { - *type = (int16_t)dataType; - *bytes = (int16_t)dataBytes; - - if (functionId == TSDB_FUNC_INTERP) { - *interBytes = sizeof(SInterpInfoDetail); - } else { - *interBytes = 0; - } - - return TSDB_CODE_SUCCESS; - } - - // (uid, tid) + VGID + TAGSIZE + VARSTR_HEADER_SIZE - if (functionId == TSDB_FUNC_TID_TAG) { // todo use struct - *type = TSDB_DATA_TYPE_BINARY; - *bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE); - *interBytes = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == TSDB_FUNC_BLKINFO) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = 16384; - *interBytes = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == TSDB_FUNC_COUNT) { - *type = TSDB_DATA_TYPE_BIGINT; - *bytes = sizeof(int64_t); - *interBytes = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == TSDB_FUNC_ARITHM) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); - *interBytes = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == TSDB_FUNC_TS_COMP) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = 1; // this results is compressed ts data, only one byte - *interBytes = POINTER_BYTES; - return TSDB_CODE_SUCCESS; - } - - if (functionId == TSDB_FUNC_DERIVATIVE) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); // this results is compressed ts data, only one byte - *interBytes = sizeof(SDerivInfo); - return TSDB_CODE_SUCCESS; - } - - if (isSuperTable) { - if (functionId < 0) { - if (pUdfInfo->bufSize > 0) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = pUdfInfo->bufSize; - *interBytes = *bytes; - } else { - *type = pUdfInfo->resType; - *bytes = pUdfInfo->resBytes; - *interBytes = *bytes; - } - - return TSDB_CODE_SUCCESS; - } - - if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = (int16_t)(dataBytes + DATA_SET_FLAG_SIZE); - *interBytes = *bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_SUM) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SSumInfo); - *interBytes = *bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_AVG) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SAvgInfo); - *interBytes = *bytes; - return TSDB_CODE_SUCCESS; - - } else if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_IRATE) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(SRateInfo); - *interBytes = sizeof(SRateInfo); - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = (int16_t)(sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param); - *interBytes = *bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_SPREAD) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SSpreadInfo); - *interBytes = *bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_APERCT) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1) + sizeof(SHistogramInfo) + sizeof(SAPercentileInfo); - *interBytes = *bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_LAST_ROW) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = (int16_t)(sizeof(SLastrowInfo) + dataBytes); - *interBytes = *bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_TWA) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(STwaInfo); - *interBytes = *bytes; - return TSDB_CODE_SUCCESS; - } - } - - if (functionId == TSDB_FUNC_SUM) { - if (IS_SIGNED_NUMERIC_TYPE(dataType)) { - *type = TSDB_DATA_TYPE_BIGINT; - } else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) { - *type = TSDB_DATA_TYPE_UBIGINT; - } else { - *type = TSDB_DATA_TYPE_DOUBLE; - } - - *bytes = sizeof(int64_t); - *interBytes = sizeof(SSumInfo); - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_APERCT) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); - *interBytes = - sizeof(SAPercentileInfo) + sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1); - return TSDB_CODE_SUCCESS; - } else if (functionId == TSDB_FUNC_TWA) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); - *interBytes = sizeof(STwaInfo); - return TSDB_CODE_SUCCESS; - } - - if (functionId < 0) { - *type = pUdfInfo->resType; - *bytes = pUdfInfo->resBytes; - - if (pUdfInfo->bufSize > 0) { - *interBytes = pUdfInfo->bufSize; - } else { - *interBytes = *bytes; - } - - return TSDB_CODE_SUCCESS; - } - - if (functionId == TSDB_FUNC_AVG) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); - *interBytes = sizeof(SAvgInfo); - } else if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_IRATE) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); - *interBytes = sizeof(SRateInfo); - } else if (functionId == TSDB_FUNC_STDDEV) { - *type = TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); - *interBytes = sizeof(SStddevInfo); - } else if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { - *type = (int16_t)dataType; - *bytes = (int16_t)dataBytes; - *interBytes = dataBytes + DATA_SET_FLAG_SIZE; - } else if (functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_LAST) { - *type = (int16_t)dataType; - *bytes = (int16_t)dataBytes; - *interBytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); - } else if (functionId == TSDB_FUNC_SPREAD) { - *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; - *bytes = sizeof(double); - *interBytes = sizeof(SSpreadInfo); - } else if (functionId == TSDB_FUNC_PERCT) { - *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; - *bytes = (int16_t)sizeof(double); - *interBytes = (int16_t)sizeof(SPercentileInfo); - } else if (functionId == TSDB_FUNC_LEASTSQR) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = TMAX(TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string - *interBytes = *bytes; - } else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); - *interBytes = *bytes; - } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - *type = (int16_t)dataType; - *bytes = (int16_t)dataBytes; - - size_t size = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param; - - // the output column may be larger than sizeof(STopBotInfo) - *interBytes = (int32_t)size; - } else if (functionId == TSDB_FUNC_LAST_ROW) { - *type = (int16_t)dataType; - *bytes = (int16_t)dataBytes; - *interBytes = dataBytes; - } else if (functionId == TSDB_FUNC_STDDEV_DST) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SStddevdstInfo); - *interBytes = (*bytes); - - } else { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} - -// TODO use hash table -int32_t isValidFunction(const char* name, int32_t len) { - for(int32_t i = 0; i <= TSDB_FUNC_BLKINFO; ++i) { - int32_t nameLen = (int32_t) strlen(aAggs[i].name); - if (len != nameLen) { - continue; - } - - if (strncasecmp(aAggs[i].name, name, len) == 0) { - return i; - } - } - - return -1; -} - -static bool function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { - if (pResultInfo->initialized) { - return false; - } - - memset(pCtx->pOutput, 0, (size_t)pCtx->outputBytes); - initResultRowEntry(pResultInfo, pCtx->interBufBytes); - return true; -} - -/** - * in handling the stable query, function_finalizer is called after the secondary - * merge being completed, during the first merge procedure, which is executed at the - * vnode side, the finalize will never be called. - * - * @param pCtx - */ -static void function_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - } - - doFinalizer(pCtx); -} - -/* - * count function does need the finalize, if data is missing, the default value, which is 0, is used - * count function does not use the pCtx->interResBuf to keep the intermediate buffer - */ -static void count_function(SQLFunctionCtx *pCtx) { - int32_t numOfElem = 0; - - /* - * 1. column data missing (schema modified) causes pCtx->hasNull == true. pCtx->preAggVals.isSet == true; - * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->preAggVals.isSet == true; - * 3. for primary key column, pCtx->hasNull always be false, pCtx->preAggVals.isSet == false; - */ - if (pCtx->preAggVals.isSet) { - numOfElem = pCtx->size - pCtx->preAggVals.statis.numOfNull; - } else { - if (pCtx->hasNull) { - for (int32_t i = 0; i < pCtx->size; ++i) { - char *val = GET_INPUT_DATA(pCtx, i); - if (isNull(val, pCtx->inputType)) { - continue; - } - - numOfElem += 1; - } - } else { - //when counting on the primary time stamp column and no statistics data is presented, use the size value directly. - numOfElem = pCtx->size; - } - } - - if (numOfElem > 0) { - GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; - } - - *((int64_t *)pCtx->pOutput) += numOfElem; - SET_VAL(pCtx, numOfElem, 1); -} - -static void count_func_merge(SQLFunctionCtx *pCtx) { - int64_t *pData = (int64_t *)GET_INPUT_DATA_LIST(pCtx); - for (int32_t i = 0; i < pCtx->size; ++i) { - *((int64_t *)pCtx->pOutput) += pData[i]; - } - - SET_VAL(pCtx, pCtx->size, 1); -} - -/** - * 1. If the column value for filter exists, we need to load the SFields, which serves - * as the pre-filter to decide if the actual data block is required or not. - * 2. If it queries on the non-primary timestamp column, SFields is also required to get the not-null value. - * - * @param colId - * @param filterCols - * @return - */ -int32_t countRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return BLK_DATA_NO_NEEDED; - } else { - return BLK_DATA_STATIS_NEEDED; - } -} - -int32_t noDataRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_NO_NEEDED; -} -#define LIST_ADD_N_DOUBLE_FLOAT(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - SET_DOUBLE_VAL(&(x) , GET_DOUBLE_VAL(&(x)) + GET_FLOAT_VAL(&(d)[i])); \ - (numOfElem)++; \ - } \ - } while(0) -#define LIST_ADD_N_DOUBLE(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - SET_DOUBLE_VAL(&(x) , (x) + (d)[i]); \ - (numOfElem)++; \ - } \ - } while(0) - -#define LIST_ADD_N(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - (x) += (d)[i]; \ - (numOfElem)++; \ - } \ - } while(0) - -#define UPDATE_DATA(ctx, left, right, num, sign, k) \ - do { \ - if (((left) < (right)) ^ (sign)) { \ - (left) = (right); \ - DO_UPDATE_TAG_COLUMNS(ctx, k); \ - (num) += 1; \ - } \ - } while (0) - -#define DUPATE_DATA_WITHOUT_TS(ctx, left, right, num, sign) \ - do { \ - if (((left) < (right)) ^ (sign)) { \ - (left) = (right); \ - DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx); \ - (num) += 1; \ - } \ - } while (0) - -#define LOOPCHECK_N(val, list, ctx, tsdbType, sign, num) \ - for (int32_t i = 0; i < ((ctx)->size); ++i) { \ - if ((ctx)->hasNull && isNull((char *)&(list)[i], tsdbType)) { \ - continue; \ - } \ - TSKEY key = (ctx)->ptsList != NULL? GET_TS_DATA(ctx, i):0; \ - UPDATE_DATA(ctx, val, (list)[i], num, sign, key); \ - } - -#define TYPED_LOOPCHECK_N(type, data, list, ctx, tsdbType, sign, notNullElems) \ - do { \ - type *_data = (type *)data; \ - type *_list = (type *)list; \ - LOOPCHECK_N(*_data, _list, ctx, tsdbType, sign, notNullElems); \ - } while (0) - -static void do_sum(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - // Only the pre-computing information loaded and actual data does not loaded - if (pCtx->preAggVals.isSet) { - notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; - assert(pCtx->size >= pCtx->preAggVals.statis.numOfNull); - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - int64_t *retVal = (int64_t *)pCtx->pOutput; - *retVal += pCtx->preAggVals.statis.sum; - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - uint64_t *retVal = (uint64_t *)pCtx->pOutput; - *retVal += (uint64_t)pCtx->preAggVals.statis.sum; - } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - double *retVal = (double*) pCtx->pOutput; - SET_DOUBLE_VAL(retVal, *retVal + GET_DOUBLE_VAL((const char*)&(pCtx->preAggVals.statis.sum))); - } - } else { // computing based on the true data block - void *pData = GET_INPUT_DATA_LIST(pCtx); - notNullElems = 0; - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - int64_t *retVal = (int64_t *)pCtx->pOutput; - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_ADD_N(*retVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_ADD_N(*retVal, pCtx, pData, int16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_ADD_N(*retVal, pCtx, pData, int32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - LIST_ADD_N(*retVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType); - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - uint64_t *retVal = (uint64_t *)pCtx->pOutput; - - if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_ADD_N(*retVal, pCtx, pData, uint8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_ADD_N(*retVal, pCtx, pData, uint16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_ADD_N(*retVal, pCtx, pData, uint32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_ADD_N(*retVal, pCtx, pData, uint64_t, notNullElems, pCtx->inputType); - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *retVal = (double *)pCtx->pOutput; - LIST_ADD_N_DOUBLE(*retVal, pCtx, pData, double, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = (double *)pCtx->pOutput; - LIST_ADD_N_DOUBLE_FLOAT(*retVal, pCtx, pData, float, notNullElems, pCtx->inputType); - } - } - - // data in the check operation are all null, not output - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; - } -} - -static void sum_function(SQLFunctionCtx *pCtx) { - do_sum(pCtx); - - // keep the result data in output buffer, not in the intermediate buffer - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { - // set the flag for super table query - SSumInfo *pSum = (SSumInfo *)pCtx->pOutput; - pSum->hasResult = DATA_SET_FLAG; - } -} - -static void sum_func_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - GET_TRUE_DATA_TYPE(); - assert(pCtx->stableQuery); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char * input = GET_INPUT_DATA(pCtx, i); - SSumInfo *pInput = (SSumInfo *)input; - if (pInput->hasResult != DATA_SET_FLAG) { - continue; - } - - notNullElems++; - - if (IS_SIGNED_NUMERIC_TYPE(type)) { - *(int64_t *)pCtx->pOutput += pInput->isum; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - *(uint64_t *) pCtx->pOutput += pInput->usum; - } else { - SET_DOUBLE_VAL((double *)pCtx->pOutput, *(double *)pCtx->pOutput + pInput->dsum); - } - } - - SET_VAL(pCtx, notNullElems, 1); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - if (notNullElems > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static int32_t statisRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_STATIS_NEEDED; -} - -static int32_t dataBlockRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_ALL_NEEDED; -} - -// todo: if column in current data block are null, opt for this case -static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order == TSDB_ORDER_DESC) { - return BLK_DATA_NO_NEEDED; - } - - // no result for first query, data block is required - if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) { - return BLK_DATA_ALL_NEEDED; - } else { - return BLK_DATA_NO_NEEDED; - } -} - -static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order != pCtx->param[0].i64) { - return BLK_DATA_NO_NEEDED; - } - - if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) { - return BLK_DATA_ALL_NEEDED; - } else { - return BLK_DATA_NO_NEEDED; - } -} - -static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order == TSDB_ORDER_DESC) { - return BLK_DATA_NO_NEEDED; - } - - // not initialized yet, it is the first block, load it. - if (pCtx->pOutput == NULL) { - return BLK_DATA_ALL_NEEDED; - } - - // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is - // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid - SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes); - if (pInfo->hasResult != DATA_SET_FLAG) { - return BLK_DATA_ALL_NEEDED; - } else { // data in current block is not earlier than current result - return (pInfo->ts <= w->skey) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; - } -} - -static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order != pCtx->param[0].i64) { - return BLK_DATA_NO_NEEDED; - } - - // not initialized yet, it is the first block, load it. - if (pCtx->pOutput == NULL) { - return BLK_DATA_ALL_NEEDED; - } - - // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is - // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid - SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes); - if (pInfo->hasResult != DATA_SET_FLAG) { - return BLK_DATA_ALL_NEEDED; - } else { - return (pInfo->ts > w->ekey) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; - } -} - -////////////////////////////////////////////////////////////////////////////////////////////// -/* - * The intermediate result of average is kept in the interResultBuf. - * For super table query, once the avg_function/avg_function_f is finished, copy the intermediate - * result into output buffer. - */ -static void avg_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - // NOTE: keep the intermediate result into the interResultBuf - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - double *pVal = &pAvgInfo->sum; - - if (pCtx->preAggVals.isSet) { // Pre-aggregation - notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; - assert(notNullElems >= 0); - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - *pVal += pCtx->preAggVals.statis.sum; - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - *pVal += (uint64_t) pCtx->preAggVals.statis.sum; - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - *pVal += GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.sum)); - } - } else { - void *pData = GET_INPUT_DATA_LIST(pCtx); - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_ADD_N(*pVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_ADD_N(*pVal, pCtx, pData, int16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_ADD_N(*pVal, pCtx, pData, int32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - LIST_ADD_N(*pVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_ADD_N_DOUBLE(*pVal, pCtx, pData, double, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_ADD_N_DOUBLE_FLOAT(*pVal, pCtx, pData, float, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint64_t, notNullElems, pCtx->inputType); - } - } - - if (!pCtx->hasNull) { - assert(notNullElems == pCtx->size); - } - - SET_VAL(pCtx, notNullElems, 1); - pAvgInfo->num += notNullElems; - - if (notNullElems > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); - } -} - -static void avg_func_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - double *sum = (double*) pCtx->pOutput; - char *input = GET_INPUT_DATA_LIST(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SAvgInfo *pInput = (SAvgInfo *)input; - if (pInput->num == 0) { // current input is null - continue; - } - - SET_DOUBLE_VAL(sum, *sum + pInput->sum); - - // keep the number of data into the temp buffer - *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo) += pInput->num; - } -} - -/* - * the average value is calculated in finalize routine, since current routine does not know the exact number of points - */ -static void avg_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - if (pCtx->currentStage == MERGE_STAGE) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - - if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput,(*(double *)pCtx->pOutput) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo)); - } else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY - assert(IS_NUMERIC_TYPE(pCtx->inputType)); - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - - if (pAvgInfo->num == 0) { // all data are NULL or empty table - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput, pAvgInfo->sum / pAvgInfo->num); - } - - // cannot set the numOfIteratedElems again since it is set during previous iteration - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////////////////// - -static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { - // data in current data block are qualified to the query - if (pCtx->preAggVals.isSet) { - *notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; - assert(*notNullElems >= 0); - - if (*notNullElems == 0) { - return; - } - - void* tval = NULL; - int16_t index = 0; - - if (isMin) { - tval = &pCtx->preAggVals.statis.min; - index = pCtx->preAggVals.statis.minIndex; - } else { - tval = &pCtx->preAggVals.statis.max; - index = pCtx->preAggVals.statis.maxIndex; - } - - TSKEY key = TSKEY_INITIAL_VAL; - if (pCtx->ptsList != NULL) { - /** - * NOTE: work around the bug caused by invalid pre-calculated function. - * Here the selectivity + ts will not return correct value. - * - * The following codes of 3 lines will be removed later. - */ -// if (index < 0 || index >= pCtx->size + pCtx->startOffset) { -// index = 0; -// } - - // the index is the original position, not the relative position - key = pCtx->ptsList[index]; - } - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - int64_t val = GET_INT64_VAL(tval); - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - int8_t *data = (int8_t *)pOutput; - - UPDATE_DATA(pCtx, *data, (int8_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - int16_t *data = (int16_t *)pOutput; - - UPDATE_DATA(pCtx, *data, (int16_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *data = (int32_t *)pOutput; -#if defined(_DEBUG_VIEW) - qDebug("max value updated according to pre-cal:%d", *data); -#endif - - if ((*data < val) ^ isMin) { - *data = (int32_t)val; - for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) { - SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; - if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { - __ctx->tag.i64 = key; - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; - } - - aAggs[TSDB_FUNC_TAG].xFunction(__ctx); - } - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - int64_t *data = (int64_t *)pOutput; - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - uint64_t val = GET_UINT64_VAL(tval); - if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - uint8_t *data = (uint8_t *)pOutput; - - UPDATE_DATA(pCtx, *data, (uint8_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - uint16_t *data = (uint16_t *)pOutput; - UPDATE_DATA(pCtx, *data, (uint16_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - uint32_t *data = (uint32_t *)pOutput; - UPDATE_DATA(pCtx, *data, (uint32_t)val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - uint64_t *data = (uint64_t *)pOutput; - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *data = (double *)pOutput; - double val = GET_DOUBLE_VAL(tval); - - UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - float *data = (float *)pOutput; - double val = GET_DOUBLE_VAL(tval); - - UPDATE_DATA(pCtx, *data, (float)val, notNullElems, isMin, key); - } - - return; - } - - void *p = GET_INPUT_DATA_LIST(pCtx); - TSKEY *tsList = GET_TS_LIST(pCtx); - - *notNullElems = 0; - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - TYPED_LOOPCHECK_N(int8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - TYPED_LOOPCHECK_N(int16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *pData = p; - int32_t *retVal = (int32_t*) pOutput; - - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*)&pData[i], pCtx->inputType)) { - continue; - } - - if ((*retVal < pData[i]) ^ isMin) { - *retVal = pData[i]; - TSKEY k = tsList[i]; - - DO_UPDATE_TAG_COLUMNS(pCtx, k); - } - - *notNullElems += 1; - } -#if defined(_DEBUG_VIEW) - qDebug("max value updated:%d", *retVal); -#endif - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - TYPED_LOOPCHECK_N(int64_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - TYPED_LOOPCHECK_N(uint8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - TYPED_LOOPCHECK_N(uint16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - TYPED_LOOPCHECK_N(uint32_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - TYPED_LOOPCHECK_N(uint64_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - TYPED_LOOPCHECK_N(double, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - TYPED_LOOPCHECK_N(float, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } -} - -static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; // not initialized since it has been initialized - } - - GET_TRUE_DATA_TYPE(); - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)pCtx->pOutput) = INT8_MAX; - break; - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *) pCtx->pOutput = UINT8_MAX; - break; - case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)pCtx->pOutput) = INT16_MAX; - break; - case TSDB_DATA_TYPE_USMALLINT: - *((uint16_t *)pCtx->pOutput) = UINT16_MAX; - break; - case TSDB_DATA_TYPE_INT: - *((int32_t *)pCtx->pOutput) = INT32_MAX; - break; - case TSDB_DATA_TYPE_UINT: - *((uint32_t *)pCtx->pOutput) = UINT32_MAX; - break; - case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)pCtx->pOutput) = INT64_MAX; - break; - case TSDB_DATA_TYPE_UBIGINT: - *((uint64_t *)pCtx->pOutput) = UINT64_MAX; - break; - case TSDB_DATA_TYPE_FLOAT: - *((float *)pCtx->pOutput) = FLT_MAX; - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_VAL(((double *)pCtx->pOutput), DBL_MAX); - break; - default: - qError("illegal data type:%d in min/max query", pCtx->inputType); - } - - return true; -} - -static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; // not initialized since it has been initialized - } - - GET_TRUE_DATA_TYPE(); - - switch (type) { - case TSDB_DATA_TYPE_INT: - *((int32_t *)pCtx->pOutput) = INT32_MIN; - break; - case TSDB_DATA_TYPE_UINT: - *((uint32_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_FLOAT: - *((float *)pCtx->pOutput) = -FLT_MAX; - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_VAL(((double *)pCtx->pOutput), -DBL_MAX); - break; - case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)pCtx->pOutput) = INT64_MIN; - break; - case TSDB_DATA_TYPE_UBIGINT: - *((uint64_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)pCtx->pOutput) = INT16_MIN; - break; - case TSDB_DATA_TYPE_USMALLINT: - *((uint16_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)pCtx->pOutput) = INT8_MIN; - break; - case TSDB_DATA_TYPE_UTINYINT: - *((uint8_t *)pCtx->pOutput) = 0; - break; - default: - qError("illegal data type:%d in min/max query", pCtx->inputType); - } - - return true; -} - -/* - * the output result of min/max function is the final output buffer, not the intermediate result buffer - */ -static void min_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - minMax_function(pCtx, pCtx->pOutput, 1, ¬NullElems); - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - - // set the flag for super table query - if (pCtx->stableQuery) { - *(pCtx->pOutput + pCtx->inputBytes) = DATA_SET_FLAG; - } - } -} - -static void max_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - minMax_function(pCtx, pCtx->pOutput, 0, ¬NullElems); - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - - // set the flag for super table query - if (pCtx->stableQuery) { - *(pCtx->pOutput + pCtx->inputBytes) = DATA_SET_FLAG; - } - } -} - -static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { - int32_t notNullElems = 0; - - GET_TRUE_DATA_TYPE(); - assert(pCtx->stableQuery); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *input = GET_INPUT_DATA(pCtx, i); - if (input[bytes] != DATA_SET_FLAG) { - continue; - } - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t v = GET_INT8_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int8_t *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t v = GET_INT16_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int16_t *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t v = GET_INT32_VAL(input); - if ((*(int32_t *)output < v) ^ isMin) { - *(int32_t *)output = v; - - for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { - SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j]; - aAggs[TSDB_FUNC_TAG].xFunction(__ctx); - } - - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float v = GET_FLOAT_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(float *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double v = GET_DOUBLE_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(double *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t v = GET_INT64_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int64_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t v = GET_UINT8_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint8_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t v = GET_UINT16_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint16_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UINT: { - uint32_t v = GET_UINT32_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint32_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t v = GET_UINT64_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint64_t *)output, v, notNullElems, isMin); - break; - } - - default: - break; - } - } - - return notNullElems; -} - -static void min_func_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->pOutput, 1); - - SET_VAL(pCtx, notNullElems, 1); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (notNullElems > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void max_func_merge(SQLFunctionCtx *pCtx) { - int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->pOutput, 0); - - SET_VAL(pCtx, numOfElem, 1); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (numOfElem > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -#define LOOP_STDDEV_IMPL(type, r, d, ctx, delta, _type, num) \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&((type *)d)[i], (_type))) { \ - continue; \ - } \ - (num) += 1; \ - (r) += TPOW2(((type *)d)[i] - (delta)); \ - } - -static void stddev_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->currentStage == REPEAT_SCAN && pStd->stage == 0) { - pStd->stage++; - avg_finalizer(pCtx); - - pResInfo->initialized = true; // set it initialized to avoid re-initialization - - // save average value into tmpBuf, for second stage scan - SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); - - pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput); - assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); - } - - if (pStd->stage == 0) { - // the first stage is to calculate average value - avg_function(pCtx); - } else if (pStd->num > 0) { - // the second stage to calculate standard deviation - // if pStd->num == 0, there are no numbers in the first round check. No need to do the second round - double *retVal = &pStd->res; - double avg = pStd->avg; - - void *pData = GET_INPUT_DATA_LIST(pCtx); - int32_t num = 0; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { - continue; - } - num += 1; - *retVal += TPOW2(((int32_t *)pData)[i] - avg); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - LOOP_STDDEV_IMPL(uint8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UINT: { - LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - default: - qError("stddev function not support data type:%d", pCtx->inputType); - } - - SET_VAL(pCtx, 1, 1); - } -} - -static void stddev_finalizer(SQLFunctionCtx *pCtx) { - SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - - if (pStd->num <= 0) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - } else { - double *retValue = (double *)pCtx->pOutput; - SET_DOUBLE_VAL(retValue, sqrt(pStd->res / pStd->num)); - SET_VAL(pCtx, 1, 1); - } - - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////////// -int32_t tsCompare(const void* p1, const void* p2) { - TSKEY k = *(TSKEY*)p1; - SResPair* pair = (SResPair*)p2; - - if (k == pair->key) { - return 0; - } else { - return k < pair->key? -1:1; - } -} - -static void stddev_dst_function(SQLFunctionCtx *pCtx) { - SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - - // the second stage to calculate standard deviation - double *retVal = &pStd->res; - - // all data are null, no need to proceed - SArray* resList = (SArray*) pCtx->param[0].pz; - if (resList == NULL) { - return; - } - - // find the correct group average results according to the tag value - int32_t len = (int32_t) taosArrayGetSize(resList); - assert(len > 0); - - double avg = 0; - if (len == 1) { - SResPair* p = taosArrayGet(resList, 0); - avg = p->avg; - } else { // todo opt performance by using iterator since the timestamp lsit is matched with the output result - SResPair* p = bsearch(&pCtx->startTs, resList->pData, len, sizeof(SResPair), tsCompare); - if (p == NULL) { - return; - } - - avg = p->avg; - } - - void *pData = GET_INPUT_DATA_LIST(pCtx); - int32_t num = 0; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { - continue; - } - num += 1; - *retVal += TPOW2(((int32_t *)pData)[i] - avg); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UINT: { - LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - default: - qError("stddev function not support data type:%d", pCtx->inputType); - } - - pStd->num += num; - SET_VAL(pCtx, num, 1); - - // copy to the final output buffer for super table - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)), sizeof(SAvgInfo)); -} - -static void stddev_dst_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevdstInfo* pRes = GET_ROWCELL_INTERBUF(pResInfo); - - char *input = GET_INPUT_DATA_LIST(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SStddevdstInfo *pInput = (SStddevdstInfo *)input; - if (pInput->num == 0) { // current input is null - continue; - } - - pRes->num += pInput->num; - pRes->res += pInput->res; - } -} - -static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) { - SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - - if (pStd->num <= 0) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - } else { - double *retValue = (double *)pCtx->pOutput; - SET_DOUBLE_VAL(retValue, sqrt(pStd->res / pStd->num)); - SET_VAL(pCtx, 1, 1); - } - - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////////// -static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - // used to keep the timestamp for comparison - pCtx->param[1].nType = 0; - pCtx->param[1].i64 = 0; - - return true; -} - -// todo opt for null block -static void first_function(SQLFunctionCtx *pCtx) { - if (pCtx->order == TSDB_ORDER_DESC) { - return; - } - - int32_t notNullElems = 0; - - // handle the null value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - memcpy(pCtx->pOutput, data, pCtx->inputBytes); - if (pCtx->ptsList != NULL) { - TSKEY k = GET_TS_DATA(pCtx, i); - DO_UPDATE_TAG_COLUMNS(pCtx, k); - } - - SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->complete = true; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = GET_TS_LIST(pCtx); - - SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); - - if (pInfo->hasResult != DATA_SET_FLAG || timestamp[index] < pInfo->ts) { - memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->ts = timestamp[index]; - - DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); - } -} - -/* - * format of intermediate result: "timestamp,value" need to compare the timestamp in the first part (before the comma) - * to decide if the value is earlier than current intermediate result - */ -static void first_dist_function(SQLFunctionCtx *pCtx) { - /* - * do not to check data in the following cases: - * 1. data block that are not loaded - * 2. scan data files in desc order - */ - if (pCtx->order == TSDB_ORDER_DESC/* || pCtx->preAggVals.dataBlockLoaded == false*/) { - return; - } - - int32_t notNullElems = 0; - - // find the first not null value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - first_data_assign_impl(pCtx, data, i); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void first_dist_func_merge(SQLFunctionCtx *pCtx) { - assert(pCtx->stableQuery); - - char * pData = GET_INPUT_DATA_LIST(pCtx); - SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - // The param[1] is used to keep the initial value of max ts value - if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64 > pInput->ts) { - memcpy(pCtx->pOutput, pData, pCtx->outputBytes); - pCtx->param[1].i64 = pInput->ts; - pCtx->param[1].nType = pCtx->outputType; - - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } - - SET_VAL(pCtx, 1, 1); - GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -////////////////////////////////////////////////////////////////////////////////////////// -/* - * last function: - * 1. since the last block may be all null value, so, we simply access the last block is not valid - * each block need to be checked. - * 2. If numOfNull == pBlock->numOfBlocks, the whole block is empty. Otherwise, there is at - * least one data in this block that is not null.(TODO opt for this case) - */ -static void last_function(SQLFunctionCtx *pCtx) { - if (pCtx->order != pCtx->param[0].i64) { - return; - } - - SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - if (pCtx->order == TSDB_ORDER_DESC) { - - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { - continue; - } - - memcpy(pCtx->pOutput, data, pCtx->inputBytes); - - TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; - DO_UPDATE_TAG_COLUMNS(pCtx, ts); - - pResInfo->hasResult = DATA_SET_FLAG; - pResInfo->complete = true; // set query completed on this column - notNullElems++; - break; - } - } else { // ascending order - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { - continue; - } - - TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; - - char* buf = GET_ROWCELL_INTERBUF(pResInfo); - if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { - pResInfo->hasResult = DATA_SET_FLAG; - memcpy(pCtx->pOutput, data, pCtx->inputBytes); - - *(TSKEY*)buf = ts; - DO_UPDATE_TAG_COLUMNS(pCtx, ts); - } - - notNullElems++; - break; - } - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = GET_TS_LIST(pCtx); - - SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); - - if (pInfo->hasResult != DATA_SET_FLAG || pInfo->ts < timestamp[index]) { -#if defined(_DEBUG_VIEW) - qDebug("assign index:%d, ts:%" PRId64 ", val:%d, ", index, timestamp[index], *(int32_t *)pData); -#endif - - memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->ts = timestamp[index]; - - DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); - } -} - -static void last_dist_function(SQLFunctionCtx *pCtx) { - /* - * 1. for scan data is not the required order - * 2. for data blocks that are not loaded, no need to check data - */ - if (pCtx->order != pCtx->param[0].i64) { - return; - } - - int32_t notNullElems = 0; - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - if (!pCtx->requireNull) { - continue; - } - } - - last_data_assign_impl(pCtx, data, i); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -/* - * in the secondary merge(local reduce), the output is limited by the - * final output size, so the main difference between last_dist_func_merge and second_merge - * is: the output data format in computing - */ -static void last_dist_func_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_DATA_LIST(pCtx); - - SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - /* - * param[1] used to keep the corresponding timestamp to decide if current result is - * the true last result - */ - if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64 < pInput->ts) { - memcpy(pCtx->pOutput, pData, pCtx->outputBytes); - pCtx->param[1].i64 = pInput->ts; - pCtx->param[1].nType = pCtx->outputType; - - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } - - SET_VAL(pCtx, 1, 1); - GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -////////////////////////////////////////////////////////////////////////////////// -/* - * NOTE: last_row does not use the interResultBuf to keep the result - */ -static void last_row_function(SQLFunctionCtx *pCtx) { - assert(pCtx->size >= 1); - char *pData = GET_INPUT_DATA_LIST(pCtx); - - // assign the last element in current data block - assignVal(pCtx->pOutput, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - - // set the result to final result buffer in case of super table query - if (pCtx->stableQuery) { - SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->pOutput + pCtx->inputBytes); - pInfo1->ts = GET_TS_DATA(pCtx, pCtx->size - 1); - pInfo1->hasResult = DATA_SET_FLAG; - - DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); - } else { - TSKEY ts = GET_TS_DATA(pCtx, pCtx->size - 1); - DO_UPDATE_TAG_COLUMNS(pCtx, ts); - } - - SET_VAL(pCtx, pCtx->size, 1); -} - -static void last_row_finalizer(SQLFunctionCtx *pCtx) { - // do nothing at the first stage - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////// -static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int64_t tsKey, char *pTags, - SExtTagsInfo *pTagInfo, int16_t stage) { - dst->v.nType = type; - dst->v.i64 = *(int64_t *)val; - dst->timestamp = tsKey; - - int32_t size = 0; - if (stage == MERGE_STAGE) { - memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); - } else { // the tags are dumped from the ctx tag fields - for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { - SQLFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; - if (ctx->functionId == TSDB_FUNC_TS_DUMMY) { - ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; - ctx->tag.i64 = tsKey; - } - - tVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true); - size += pTagInfo->pTagCtxList[i]->outputBytes; - } - } -} - -#define VALUEPAIRASSIGN(dst, src, __l) \ - do { \ - (dst)->timestamp = (src)->timestamp; \ - (dst)->v = (src)->v; \ - memcpy((dst)->pTags, (src)->pTags, (size_t)(__l)); \ - } while (0) - -static int32_t topBotComparFn(const void *p1, const void *p2, const void *param) -{ - uint16_t type = *(uint16_t *) param; - tValuePair *val1 = *(tValuePair **) p1; - tValuePair *val2 = *(tValuePair **) p2; - - if (IS_SIGNED_NUMERIC_TYPE(type)) { - if (val1->v.i64 == val2->v.i64) { - return 0; - } - - return (val1->v.i64 > val2->v.i64) ? 1 : -1; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - if (val1->v.u64 == val2->v.u64) { - return 0; - } - - return (val1->v.u64 > val2->v.u64) ? 1 : -1; - } - - if (val1->v.dKey == val2->v.dKey) { - return 0; - } - - return (val1->v.dKey > val2->v.dKey) ? 1 : -1; -} - -static void topBotSwapFn(void *dst, void *src, const void *param) -{ - char tag[32768]; - tValuePair temp; - uint16_t tagLen = *(uint16_t *) param; - tValuePair *vdst = *(tValuePair **) dst; - tValuePair *vsrc = *(tValuePair **) src; - - memset(tag, 0, sizeof(tag)); - temp.pTags = tag; - - VALUEPAIRASSIGN(&temp, vdst, tagLen); - VALUEPAIRASSIGN(vdst, vsrc, tagLen); - VALUEPAIRASSIGN(vsrc, &temp, tagLen); -} - -static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, - SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { - tVariant val = {0}; - tVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); - - tValuePair **pList = pInfo->res; - assert(pList != NULL); - - if (pInfo->num < maxLen) { - valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - - taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); - - pInfo->num++; - } else { - if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 > pList[0]->v.i64) || - (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 > pList[0]->v.u64) || - (IS_FLOAT_TYPE(type) && val.dKey > pList[0]->v.dKey)) { - valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); - } - } -} - -static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, - SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { - tVariant val = {0}; - tVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); - - tValuePair **pList = pInfo->res; - assert(pList != NULL); - - if (pInfo->num < maxLen) { - valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - - taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); - - pInfo->num++; - } else { - if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 < pList[0]->v.i64) || - (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 < pList[0]->v.u64) || - (IS_FLOAT_TYPE(type) && val.dKey < pList[0]->v.dKey)) { - valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); - } - } -} - -static int32_t resAscComparFn(const void *pLeft, const void *pRight) { - tValuePair *pLeftElem = *(tValuePair **)pLeft; - tValuePair *pRightElem = *(tValuePair **)pRight; - - if (pLeftElem->timestamp == pRightElem->timestamp) { - return 0; - } else { - return pLeftElem->timestamp > pRightElem->timestamp ? 1 : -1; - } -} - -static int32_t resDescComparFn(const void *pLeft, const void *pRight) { return -resAscComparFn(pLeft, pRight); } - -static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { - tValuePair *pLeftElem = *(tValuePair **)pLeft; - tValuePair *pRightElem = *(tValuePair **)pRight; - - if (IS_FLOAT_TYPE(pLeftElem->v.nType)) { - if (pLeftElem->v.dKey == pRightElem->v.dKey) { - return 0; - } else { - return pLeftElem->v.dKey > pRightElem->v.dKey ? 1 : -1; - } - } else if (IS_SIGNED_NUMERIC_TYPE(pLeftElem->v.nType)){ - if (pLeftElem->v.i64 == pRightElem->v.i64) { - return 0; - } else { - return pLeftElem->v.i64 > pRightElem->v.i64 ? 1 : -1; - } - } else { - if (pLeftElem->v.u64 == pRightElem->v.u64) { - return 0; - } else { - return pLeftElem->v.u64 > pRightElem->v.u64 ? 1 : -1; - } - } -} - -static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); } - -static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); - - tValuePair **tvp = pRes->res; - - int32_t step = QUERY_ASC_FORWARD_STEP; - int32_t len = (int32_t)(GET_RES_INFO(pCtx)->numOfRes); - - switch (type) { - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - int32_t *output = (int32_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int32_t)tvp[i]->v.i64; - } - break; - } - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - int64_t *output = (int64_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i]->v.i64; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *output = (double *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - SET_DOUBLE_VAL(output, tvp[i]->v.dKey); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *output = (float *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (float)tvp[i]->v.dKey; - } - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *output = (int16_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int16_t)tvp[i]->v.i64; - } - break; - } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { - int8_t *output = (int8_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int8_t)tvp[i]->v.i64; - } - break; - } - default: { - qError("top/bottom function not support data type:%d", pCtx->inputType); - return; - } - } - - // set the output timestamp of each record. - TSKEY *output = pCtx->ptsOutputBuf; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i]->timestamp; - } - - // set the corresponding tag data for each record - // todo check malloc failure - char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); - for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { - pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput; - } - - for (int32_t i = 0; i < len; ++i, output += step) { - int16_t offset = 0; - for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { - memcpy(pData[j], tvp[i]->pTags + offset, (size_t)pCtx->tagInfo.pTagCtxList[j]->outputBytes); - offset += pCtx->tagInfo.pTagCtxList[j]->outputBytes; - pData[j] += pCtx->tagInfo.pTagCtxList[j]->outputBytes; - } - } - - tfree(pData); -} - -/* - * Parameters values: - * 1. param[0]: maximum allowable results - * 2. param[1]: order by type (time or value) - * 3. param[2]: asc/desc order - * - * top/bottom use the intermediate result buffer to keep the intermediate result - */ -static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - // only the first_stage_merge is directly written data into final output buffer - if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { - return (STopBotInfo*) pCtx->pOutput; - } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer - return GET_ROWCELL_INTERBUF(pResInfo); - } -} - - -/* - * keep the intermediate results during scan data blocks in the format of: - * +-----------------------------------+-------------one value pair-----------+------------next value pair-----------+ - * |-------------pointer area----------|----ts---+-----+-----n tags-----------|----ts---+-----+-----n tags-----------| - * +..[Value Pointer1][Value Pointer2].|timestamp|value|tags1|tags2|....|tagsn|timestamp|value|tags1|tags2|....|tagsn+ - */ -static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { - char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); - pTopBotInfo->res = (tValuePair**) tmp; - tmp += POINTER_BYTES * pCtx->param[0].i64; - - size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; - - for (int32_t i = 0; i < pCtx->param[0].i64; ++i) { - pTopBotInfo->res[i] = (tValuePair*) tmp; - pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); - tmp += size; - } -} - -bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo == NULL) { - return true; - } - - STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx); - - // required number of results are not reached, continue load data block - if (pTopBotInfo->num < pCtx->param[0].i64) { - return true; - } - - if ((void *)pTopBotInfo->res[0] != (void *)((char *)pTopBotInfo + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { - buildTopBotStruct(pTopBotInfo, pCtx); - } - - tValuePair **pRes = (tValuePair**) pTopBotInfo->res; - - if (pCtx->functionId == TSDB_FUNC_TOP) { - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - return GET_INT8_VAL(maxval) > pRes[0]->v.i64; - case TSDB_DATA_TYPE_SMALLINT: - return GET_INT16_VAL(maxval) > pRes[0]->v.i64; - case TSDB_DATA_TYPE_INT: - return GET_INT32_VAL(maxval) > pRes[0]->v.i64; - case TSDB_DATA_TYPE_BIGINT: - return GET_INT64_VAL(maxval) > pRes[0]->v.i64; - case TSDB_DATA_TYPE_FLOAT: - return GET_FLOAT_VAL(maxval) > pRes[0]->v.dKey; - case TSDB_DATA_TYPE_DOUBLE: - return GET_DOUBLE_VAL(maxval) > pRes[0]->v.dKey; - default: - return true; - } - } else { - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - return GET_INT8_VAL(minval) < pRes[0]->v.i64; - case TSDB_DATA_TYPE_SMALLINT: - return GET_INT16_VAL(minval) < pRes[0]->v.i64; - case TSDB_DATA_TYPE_INT: - return GET_INT32_VAL(minval) < pRes[0]->v.i64; - case TSDB_DATA_TYPE_BIGINT: - return GET_INT64_VAL(minval) < pRes[0]->v.i64; - case TSDB_DATA_TYPE_FLOAT: - return GET_FLOAT_VAL(minval) < pRes[0]->v.dKey; - case TSDB_DATA_TYPE_DOUBLE: - return GET_DOUBLE_VAL(minval) < pRes[0]->v.dKey; - default: - return true; - } - } -} - -static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - STopBotInfo *pInfo = getTopBotOutputInfo(pCtx); - buildTopBotStruct(pInfo, pCtx); - return true; -} - -static void top_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - assert(pRes->num >= 0); - - if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { - buildTopBotStruct(pRes, pCtx); - } - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems++; - - // NOTE: Set the default timestamp if it is missing [todo refactor] - TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; - do_top_function_add(pRes, (int32_t)pCtx->param[0].i64, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - // treat the result as only one result - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void top_func_merge(SQLFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); - - // construct the input data struct from binary data - buildTopBotStruct(pInput, pCtx); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - // the intermediate result is binary, we only use the output data type - for (int32_t i = 0; i < pInput->num; ++i) { - int16_t type = (pCtx->outputType == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->outputType; - do_top_function_add(pOutput, (int32_t)pCtx->param[0].i64, &pInput->res[i]->v.i64, pInput->res[i]->timestamp, - type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); - } - - SET_VAL(pCtx, pInput->num, pOutput->num); - - if (pOutput->num > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void bottom_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - - if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { - buildTopBotStruct(pRes, pCtx); - } - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems++; - // NOTE: Set the default timestamp if it is missing [todo refactor] - TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; - do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - // treat the result as only one result - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void bottom_func_merge(SQLFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); - - // construct the input data struct from binary data - buildTopBotStruct(pInput, pCtx); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - // the intermediate result is binary, we only use the output data type - for (int32_t i = 0; i < pInput->num; ++i) { - int16_t type = (pCtx->outputType == TSDB_DATA_TYPE_FLOAT) ? TSDB_DATA_TYPE_DOUBLE : pCtx->outputType; - do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i64, &pInput->res[i]->v.i64, pInput->res[i]->timestamp, type, - &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); - } - - SET_VAL(pCtx, pInput->num, pOutput->num); - - if (pOutput->num > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - // data in temporary list is less than the required number of results, not enough qualified number of results - STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); - if (pRes->num == 0) { // no result - assert(pResInfo->hasResult != DATA_SET_FLAG); - // TODO: - } - - GET_RES_INFO(pCtx)->numOfRes = pRes->num; - tValuePair **tvp = pRes->res; - - // user specify the order of output by sort the result according to timestamp - if (pCtx->param[1].i64 == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - __compar_fn_t comparator = (pCtx->param[2].i64 == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; - qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); - } else /*if (pCtx->param[1].i64 > PRIMARYKEY_TIMESTAMP_COL_INDEX)*/ { - __compar_fn_t comparator = (pCtx->param[2].i64 == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; - qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); - } - - GET_TRUE_DATA_TYPE(); - copyTopBotRes(pCtx, type); - - doFinalizer(pCtx); -} - -/////////////////////////////////////////////////////////////////////////////////////////////// -static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - // in the first round, get the min-max value of all involved data - SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo); - SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX); - SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX); - pInfo->numOfElems = 0; - - return true; -} - -static void percentile_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { - pInfo->stage += 1; - - // all data are null, set it completed - if (pInfo->numOfElems == 0) { - pResInfo->complete = true; - - return; - } else { - pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); - } - } - - // the first stage, only acquire the min/max value - if (pInfo->stage == 0) { - if (pCtx->preAggVals.isSet) { - double tmin = 0.0, tmax = 0.0; - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.min); - tmax = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.max); - } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - tmin = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.min); - tmax = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.max); - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.min); - tmax = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.max); - } else { - assert(true); - } - - if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) { - SET_DOUBLE_VAL(&pInfo->minval, tmin); - } - - if (GET_DOUBLE_VAL(&pInfo->maxval) < tmax) { - SET_DOUBLE_VAL(&pInfo->maxval, tmax); - } - - pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull); - } else { - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, data); - - if (v < GET_DOUBLE_VAL(&pInfo->minval)) { - SET_DOUBLE_VAL(&pInfo->minval, v); - } - - if (v > GET_DOUBLE_VAL(&pInfo->maxval)) { - SET_DOUBLE_VAL(&pInfo->maxval, v); - } - - pInfo->numOfElems += 1; - } - } - - return; - } - - // the second stage, calculate the true percentile value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems += 1; - tMemBucketPut(pInfo->pMemBucket, data, 1); - } - - SET_VAL(pCtx, notNullElems, 1); - pResInfo->hasResult = DATA_SET_FLAG; -} - -static void percentile_finalizer(SQLFunctionCtx *pCtx) { - double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64 : pCtx->param[0].dKey; - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); - - tMemBucket * pMemBucket = ppInfo->pMemBucket; - if (pMemBucket == NULL || pMemBucket->total == 0) { // check for null - assert(ppInfo->numOfElems == 0); - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - } else { - SET_DOUBLE_VAL((double *)pCtx->pOutput, getPercentile(pMemBucket, v)); - } - - tMemBucketDestroy(pMemBucket); - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////// -static void buildHistogramInfo(SAPercentileInfo* pInfo) { - pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo)); - pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo)); -} - -static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo* pInfo = NULL; - - if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { - pInfo = (SAPercentileInfo*) pCtx->pOutput; - } else { - pInfo = GET_ROWCELL_INTERBUF(pResInfo); - } - - buildHistogramInfo(pInfo); - return pInfo; -} - -static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - - char *tmp = (char *)pInfo + sizeof(SAPercentileInfo); - pInfo->pHisto = tHistogramCreateFrom(tmp, MAX_HISTOGRAM_BIN); - return true; -} - -static void apercentile_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - - assert(pInfo->pHisto->elems != NULL); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems += 1; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, data); - tHistogramAdd(&pInfo->pHisto, v); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void apercentile_func_merge(SQLFunctionCtx *pCtx) { - SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_DATA_LIST(pCtx); - - pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); - pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); - - if (pInput->pHisto->numOfElems <= 0) { - return; - } - - SAPercentileInfo *pOutput = getAPerctInfo(pCtx); - SHistogramInfo *pHisto = pOutput->pHisto; - - if (pHisto->numOfElems <= 0) { - memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - } else { - //TODO(dengyihao): avoid memcpy - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); - memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - tHistogramDestroy(&pRes); - } - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - SET_VAL(pCtx, 1, 1); -} - -static void apercentile_finalizer(SQLFunctionCtx *pCtx) { - double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i64 : pCtx->param[0].dKey; - - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->currentStage == MERGE_STAGE) { - if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null - assert(pOutput->pHisto->numOfElems > 0); - - double ratio[] = {v}; - double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); - - memcpy(pCtx->pOutput, res, sizeof(double)); - free(res); - } else { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - } else { - if (pOutput->pHisto->numOfElems > 0) { - double ratio[] = {v}; - - double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); - memcpy(pCtx->pOutput, res, sizeof(double)); - free(res); - } else { // no need to free - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - } - - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////// -static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // 2*3 matrix - pInfo->startVal = pCtx->param[0].dKey; - return true; -} - -#define LEASTSQR_CAL(p, x, y, index, step) \ - do { \ - (p)[0][0] += (double)(x) * (x); \ - (p)[0][1] += (double)(x); \ - (p)[0][2] += (double)(x) * (y)[index]; \ - (p)[1][2] += (y)[index]; \ - (x) += step; \ - } while (0) - -#define LEASTSQR_CAL_LOOP(ctx, param, x, y, tsdbType, n, step) \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&(y)[i], tsdbType)) { \ - continue; \ - } \ - (n)++; \ - LEASTSQR_CAL(param, x, y, i, step); \ - } - -static void leastsquares_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - double(*param)[3] = pInfo->mat; - double x = pInfo->startVal; - - void *pData = GET_INPUT_DATA_LIST(pCtx); - - int32_t numOfElem = 0; - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *p = pData; - // LEASTSQR_CAL_LOOP(pCtx, param, pParamData, p); - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) p, pCtx->inputType)) { - continue; - } - - param[0][0] += x * x; - param[0][1] += x; - param[0][2] += x * p[i]; - param[1][2] += p[i]; - - x += pCtx->param[1].dKey; - numOfElem++; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - }; - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - int8_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_UINT: { - uint32_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); - break; - } - } - - pInfo->startVal = x; - pInfo->num += numOfElem; - - if (pInfo->num > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } - - SET_VAL(pCtx, numOfElem, 1); -} - -static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { - // no data in query - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - if (pInfo->num == 0) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - - double(*param)[3] = pInfo->mat; - - param[1][1] = (double)pInfo->num; - param[1][0] = param[0][1]; - - param[0][0] -= param[1][0] * (param[0][1] / param[1][1]); - param[0][2] -= param[1][2] * (param[0][1] / param[1][1]); - param[0][1] = 0; - param[1][2] -= param[0][2] * (param[1][0] / param[0][0]); - param[1][0] = 0; - param[0][2] /= param[0][0]; - - param[1][2] /= param[1][1]; - - int32_t maxOutputSize = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE - VARSTR_HEADER_SIZE; - size_t n = snprintf(varDataVal(pCtx->pOutput), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}", - param[0][2], param[1][2]); - - varDataSetLen(pCtx->pOutput, n); - doFinalizer(pCtx); -} - -static void date_col_output_function(SQLFunctionCtx *pCtx) { - SET_VAL(pCtx, pCtx->size, 1); - *(int64_t *)(pCtx->pOutput) = pCtx->startTs; -} - -static void col_project_function(SQLFunctionCtx *pCtx) { - // the number of output rows should not affect the final number of rows, so set it to be 0 - if (pCtx->numOfParams == 2) { - return; - } - - // only one row is required. - if (pCtx->param[0].i64 == 1) { - SET_VAL(pCtx, pCtx->size, 1); - } else { - INC_INIT_VAL(pCtx, pCtx->size); - } - - char *pData = GET_INPUT_DATA_LIST(pCtx); - if (pCtx->order == TSDB_ORDER_ASC) { - int32_t numOfRows = (pCtx->param[0].i64 == 1)? 1:pCtx->size; - memcpy(pCtx->pOutput, pData, (size_t) numOfRows * pCtx->inputBytes); - } else { - for(int32_t i = 0; i < pCtx->size; ++i) { - memcpy(pCtx->pOutput + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, - pCtx->inputBytes); - } - } -} - -/** - * only used for tag projection query in select clause - * @param pCtx - * @return - */ -static void tag_project_function(SQLFunctionCtx *pCtx) { - INC_INIT_VAL(pCtx, pCtx->size); - - assert(pCtx->inputBytes == pCtx->outputBytes); - - tVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true); - char* data = pCtx->pOutput; - pCtx->pOutput += pCtx->outputBytes; - - // directly copy from the first one - for (int32_t i = 1; i < pCtx->size; ++i) { - memmove(pCtx->pOutput, data, pCtx->outputBytes); - pCtx->pOutput += pCtx->outputBytes; - } -} - -/** - * used in group by clause. when applying group by tags, the tags value is - * assign by using tag function. - * NOTE: there is only ONE output for ONE query range - * @param pCtx - * @return - */ -static void copy_function(SQLFunctionCtx *pCtx); - -static void tag_function(SQLFunctionCtx *pCtx) { - SET_VAL(pCtx, 1, 1); - if (pCtx->currentStage == MERGE_STAGE) { - copy_function(pCtx); - } else { - tVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true); - } -} - -static void copy_function(SQLFunctionCtx *pCtx) { - SET_VAL(pCtx, pCtx->size, 1); - - char *pData = GET_INPUT_DATA_LIST(pCtx); - assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType); -} - -enum { - INITIAL_VALUE_NOT_ASSIGNED = 0, -}; - -static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - // diff function require the value is set to -1 - pCtx->param[1].nType = INITIAL_VALUE_NOT_ASSIGNED; - return false; -} - -static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - // diff function require the value is set to -1 - SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResultInfo); - - pDerivInfo->ignoreNegative = pCtx->param[1].i64; - pDerivInfo->prevTs = -1; - pDerivInfo->tsWindow = pCtx->param[0].i64; - pDerivInfo->valueSet = false; - return false; -} - -static void deriv_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo); - - void *data = GET_INPUT_DATA_LIST(pCtx); - - int32_t notNullElems = 0; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - - TSKEY *pTimestamp = pCtx->ptsOutputBuf; - TSKEY *tsList = GET_TS_LIST(pCtx); - - double *pOutput = (double *)pCtx->pOutput; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *pData = (int32_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - SET_DOUBLE_VAL(pOutput, ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs)); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - - break; - }; - - case TSDB_DATA_TYPE_BIGINT: { - int64_t *pData = (int64_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = (double) pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *pData = (double *)data; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_FLOAT: { - float *pData = (float *)data; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *pData = (int16_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: { - int8_t *pData = (int8_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - default: - qError("error input type"); - } - - GET_RES_INFO(pCtx)->numOfRes += notNullElems; -} - -#define DIFF_IMPL(ctx, d, type) \ - do { \ - if ((ctx)->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { \ - (ctx)->param[1].nType = (ctx)->inputType; \ - *(type *)&(ctx)->param[1].i64 = *(type *)(d); \ - } else { \ - *(type *)(ctx)->pOutput = *(type *)(d) - (*(type *)(&(ctx)->param[1].i64)); \ - *(type *)(&(ctx)->param[1].i64) = *(type *)(d); \ - *(int64_t *)(ctx)->ptsOutputBuf = GET_TS_DATA(ctx, index); \ - } \ - } while (0); - -// TODO difference in date column -static void diff_function(SQLFunctionCtx *pCtx) { - void *data = GET_INPUT_DATA_LIST(pCtx); - bool isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED); - - int32_t notNullElems = 0; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - - TSKEY* pTimestamp = pCtx->ptsOutputBuf; - TSKEY* tsList = GET_TS_LIST(pCtx); - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *pData = (int32_t *)data; - int32_t *pOutput = (int32_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].i64 = pData[i]; - pCtx->param[1].nType = pCtx->inputType; - notNullElems++; - } - break; - }; - case TSDB_DATA_TYPE_BIGINT: { - int64_t *pData = (int64_t *)data; - int64_t *pOutput = (int64_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = pData[i] - pCtx->param[1].i64; // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].i64 = pData[i]; - pCtx->param[1].nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *pData = (double *)data; - double *pOutput = (double *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - SET_DOUBLE_VAL(pOutput, pData[i] - pCtx->param[1].dKey); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].dKey = pData[i]; - pCtx->param[1].nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *pData = (float *)data; - float *pOutput = (float *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (float)(pData[i] - pCtx->param[1].dKey); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].dKey = pData[i]; - pCtx->param[1].nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *pData = (int16_t *)data; - int16_t *pOutput = (int16_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].i64 = pData[i]; - pCtx->param[1].nType = pCtx->inputType; - notNullElems++; - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: { - int8_t *pData = (int8_t *)data; - int8_t *pOutput = (int8_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].i64 = pData[i]; - pCtx->param[1].nType = pCtx->inputType; - notNullElems++; - } - break; - } - default: - qError("error input type"); - } - - // initial value is not set yet - if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED || notNullElems <= 0) { - /* - * 1. current block and blocks before are full of null - * 2. current block may be null value - */ - assert(pCtx->hasNull); - } else { - int32_t forwardStep = (isFirstBlock) ? notNullElems - 1 : notNullElems; - - GET_RES_INFO(pCtx)->numOfRes += forwardStep; - } -} - -char *getArithColumnData(void *param, const char* name, int32_t colId) { - SArithmeticSupport *pSupport = (SArithmeticSupport *)param; - - int32_t index = -1; - for (int32_t i = 0; i < pSupport->numOfCols; ++i) { - if (colId == pSupport->colList[i].colId) { - index = i; - break; - } - } - - assert(index >= 0); - return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; -} - -static void arithmetic_function(SQLFunctionCtx *pCtx) { - GET_RES_INFO(pCtx)->numOfRes += pCtx->size; - SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - - arithmeticTreeTraverse(sas->pExprInfo->pExpr, pCtx->size, pCtx->pOutput, sas, pCtx->order, getArithColumnData); -} - -#define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ - { \ - type *inputData = (type *)data; \ - for (int32_t i = 0; i < elemCnt; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&inputData[i], tsdbType)) { \ - continue; \ - } \ - if (inputData[i] < minOutput) { \ - minOutput = (double)inputData[i]; \ - } \ - if (inputData[i] > maxOutput) { \ - maxOutput = (double)inputData[i]; \ - } \ - numOfNotNullElem++; \ - } \ - } - -///////////////////////////////////////////////////////////////////////////////// -static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // this is the server-side setup function in client-side, the secondary merge do not need this procedure - if (pCtx->currentStage == MERGE_STAGE) { - pCtx->param[0].dKey = DBL_MAX; - pCtx->param[3].dKey = -DBL_MAX; - } else { - pInfo->min = DBL_MAX; - pInfo->max = -DBL_MAX; - } - - return true; -} - -static void spread_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - int32_t numOfElems = 0; - - // todo : opt with pre-calculated result - // column missing cause the hasNull to be true - if (pCtx->preAggVals.isSet) { - numOfElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; - - // all data are null in current data block, ignore current data block - if (numOfElems == 0) { - goto _spread_over; - } - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType) || IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType) || - (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)) { - if (pInfo->min > pCtx->preAggVals.statis.min) { - pInfo->min = (double)pCtx->preAggVals.statis.min; - } - - if (pInfo->max < pCtx->preAggVals.statis.max) { - pInfo->max = (double)pCtx->preAggVals.statis.max; - } - } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - if (pInfo->min > GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.min))) { - pInfo->min = GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.min)); - } - - if (pInfo->max < GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.max))) { - pInfo->max = GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.max)); - } - } - - goto _spread_over; - } - - void *pData = GET_INPUT_DATA_LIST(pCtx); - numOfElems = 0; - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int8_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int16_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int32_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT || pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int64_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, double, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, float, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint8_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint16_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint32_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint64_t, pCtx->inputType, numOfElems); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == numOfElems); - } - - _spread_over: - SET_VAL(pCtx, numOfElems, 1); - - if (numOfElems > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - pInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); - } -} - -/* - * here we set the result value back to the intermediate buffer, to apply the finalize the function - * the final result is generated in spread_function_finalizer - */ -void spread_func_merge(SQLFunctionCtx *pCtx) { - SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_DATA_LIST(pCtx); - if (pData->hasResult != DATA_SET_FLAG) { - return; - } - - if (pCtx->param[0].dKey > pData->min) { - pCtx->param[0].dKey = pData->min; - } - - if (pCtx->param[3].dKey < pData->max) { - pCtx->param[3].dKey = pData->max; - } - - GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -void spread_function_finalizer(SQLFunctionCtx *pCtx) { - /* - * here we do not check the input data types, because in case of metric query, - * the type of intermediate data is binary - */ - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - if (pCtx->currentStage == MERGE_STAGE) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - - if (pResInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput, pCtx->param[3].dKey - pCtx->param[0].dKey); - } else { - assert(IS_NUMERIC_TYPE(pCtx->inputType) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); - - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput, pInfo->max - pInfo->min); - } - - GET_RES_INFO(pCtx)->numOfRes = 1; // todo add test case - doFinalizer(pCtx); -} - - -/** - * param[1]: start time - * param[2]: end time - * @param pCtx - */ -static bool twa_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->p.key = INT64_MIN; - pInfo->win = TSWINDOW_INITIALIZER; - return true; -} - -static double twa_get_area(SPoint1 s, SPoint1 e) { - if ((s.val >= 0 && e.val >= 0)|| (s.val <=0 && e.val <= 0)) { - return (s.val + e.val) * (e.key - s.key) / 2; - } - - double x = (s.key * e.val - e.key * s.val)/(e.val - s.val); - double val = (s.val * (x - s.key) + e.val * (e.key - x)) / 2; - return val; -} - -static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { - int32_t notNullElems = 0; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *tsList = GET_TS_LIST(pCtx); - - int32_t i = index; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - SPoint1* last = &pInfo->p; - - if (pCtx->start.key != INT64_MIN) { - assert((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || - (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); - - assert(last->key == INT64_MIN); - - last->key = tsList[i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); - - pInfo->dOutput += twa_get_area(pCtx->start, *last); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pCtx->start.key; - notNullElems++; - i += step; - } else if (pInfo->p.key == INT64_MIN) { - last->key = tsList[i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = last->key; - notNullElems++; - i += step; - } - - // calculate the value of - switch(pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double)val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double)val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t *val = (uint8_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t *val = (uint16_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UINT: { - uint32_t *val = (uint32_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t *val = (uint64_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double) val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - default: assert(0); - } - - // the last interpolated time window value - if (pCtx->end.key != INT64_MIN) { - pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end); - pInfo->p = pCtx->end; - } - - pInfo->win.ekey = pInfo->p.key; - return notNullElems; -} - -static void twa_function(SQLFunctionCtx *pCtx) { - void *data = GET_INPUT_DATA_LIST(pCtx); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // skip null value - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1); - while (pCtx->hasNull && i < pCtx->size && i >= 0 && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { - i += step; - } - - int32_t notNullElems = 0; - if (i >= 0 && i < pCtx->size) { - notNullElems = twa_function_impl(pCtx, i, pCtx->size); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } - - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, pInfo, sizeof(STwaInfo)); - } -} - -/* - * To copy the input to interResBuf to avoid the input buffer space be over writen - * by next input data. The TWA function only applies to each table, so no merge procedure - * is required, we simply copy to the resut ot interResBuffer. - */ -void twa_function_copy(SQLFunctionCtx *pCtx) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); - pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult; -} - -void twa_function_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); - if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - return; - } - - assert(pInfo->win.ekey == pInfo->p.key && pInfo->hasResult == pResInfo->hasResult); - if (pInfo->win.ekey == pInfo->win.skey) { - SET_DOUBLE_VAL((double *)pCtx->pOutput, pInfo->p.val); - } else { - SET_DOUBLE_VAL((double *)pCtx->pOutput , pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey)); - } - - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -/** - * - * @param pCtx - */ - -static void interp_function_impl(SQLFunctionCtx *pCtx) { - int32_t type = (int32_t) pCtx->param[2].i64; - if (type == TSDB_FILL_NONE) { - return; - } - - bool ascQuery = (pCtx->order == TSDB_ORDER_ASC); - - if (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { - *(TSKEY *)pCtx->pOutput = pCtx->startTs; - } else if (type == TSDB_FILL_NULL) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - } else if (type == TSDB_FILL_SET_VALUE) { - tVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true); - } else { - if (pCtx->start.key != INT64_MIN && ((ascQuery && pCtx->start.key <= pCtx->startTs && pCtx->end.key >= pCtx->startTs) || ((!ascQuery) && pCtx->start.key >= pCtx->startTs && pCtx->end.key <= pCtx->startTs))) { - if (type == TSDB_FILL_PREV) { - if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->start.val); - } else { - assignVal(pCtx->pOutput, pCtx->start.ptr, pCtx->outputBytes, pCtx->inputType); - } - } else if (type == TSDB_FILL_NEXT) { - if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->end.val); - } else { - assignVal(pCtx->pOutput, pCtx->end.ptr, pCtx->outputBytes, pCtx->inputType); - } - } else if (type == TSDB_FILL_LINEAR) { - SPoint point1 = {.key = pCtx->start.key, .val = &pCtx->start.val}; - SPoint point2 = {.key = pCtx->end.key, .val = &pCtx->end.val}; - SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; - - int32_t srcType = pCtx->inputType; - if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? - if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } else { - taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); - } - } else { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } - } - } else { - // no data generated yet - if (pCtx->size < 1) { - return; - } - - // check the timestamp in input buffer - TSKEY skey = GET_TS_DATA(pCtx, 0); - - if (type == TSDB_FILL_PREV) { - if ((ascQuery && skey > pCtx->startTs) || ((!ascQuery) && skey < pCtx->startTs)) { - return; - } - - if (pCtx->size > 1) { - TSKEY ekey = GET_TS_DATA(pCtx, 1); - if ((ascQuery && ekey > skey && ekey <= pCtx->startTs) || - ((!ascQuery) && ekey < skey && ekey >= pCtx->startTs)){ - skey = ekey; - } - } - assignVal(pCtx->pOutput, pCtx->pInput, pCtx->outputBytes, pCtx->inputType); - } else if (type == TSDB_FILL_NEXT) { - TSKEY ekey = skey; - char* val = NULL; - - if ((ascQuery && ekey < pCtx->startTs) || ((!ascQuery) && ekey > pCtx->startTs)) { - if (pCtx->size > 1) { - ekey = GET_TS_DATA(pCtx, 1); - if ((ascQuery && ekey < pCtx->startTs) || ((!ascQuery) && ekey > pCtx->startTs)) { - return; - } - - val = ((char*)pCtx->pInput) + pCtx->inputBytes; - } else { - return; - } - } else { - val = (char*)pCtx->pInput; - } - - assignVal(pCtx->pOutput, val, pCtx->outputBytes, pCtx->inputType); - } else if (type == TSDB_FILL_LINEAR) { - if (pCtx->size <= 1) { - return; - } - - TSKEY ekey = GET_TS_DATA(pCtx, 1); - - // no data generated yet - if ((ascQuery && !(skey <= pCtx->startTs && ekey >= pCtx->startTs)) - || ((!ascQuery) && !(skey >= pCtx->startTs && ekey <= pCtx->startTs))) { - return; - } - - char *start = GET_INPUT_DATA(pCtx, 0); - char *end = GET_INPUT_DATA(pCtx, 1); - - SPoint point1 = {.key = skey, .val = start}; - SPoint point2 = {.key = ekey, .val = end}; - SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; - - int32_t srcType = pCtx->inputType; - if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? - if (isNull(start, srcType) || isNull(end, srcType)) { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } else { - taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, srcType); - } - } else { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } - } - } - } - - SET_VAL(pCtx, 1, 1); -} - -static void interp_function(SQLFunctionCtx *pCtx) { - // at this point, the value is existed, return directly - if (pCtx->size > 0) { - bool ascQuery = (pCtx->order == TSDB_ORDER_ASC); - TSKEY key; - char *pData; - int32_t typedData = 0; - - if (ascQuery) { - key = GET_TS_DATA(pCtx, 0); - pData = GET_INPUT_DATA(pCtx, 0); - } else { - key = pCtx->start.key; - if (key == INT64_MIN) { - key = GET_TS_DATA(pCtx, 0); - pData = GET_INPUT_DATA(pCtx, 0); - } else { - if (!(IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL)) { - pData = pCtx->start.ptr; - } else { - typedData = 1; - pData = (char *)&pCtx->start.val; - } - } - } - - //if (key == pCtx->startTs && (ascQuery || !(IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL))) { - if (key == pCtx->startTs) { - if (typedData) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, *(double *)pData); - } else { - assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType); - } - - SET_VAL(pCtx, 1, 1); - } else { - interp_function_impl(pCtx); - } - } else { //no qualified data rows and interpolation is required - interp_function_impl(pCtx); - } -} - -static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; // not initialized since it has been initialized - } - - STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->pTSBuf = tsBufCreate(false, pCtx->order); - pInfo->pTSBuf->tsOrder = pCtx->order; - return true; -} - -static void ts_comp_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; - - const char *input = GET_INPUT_DATA_LIST(pCtx); - - // primary ts must be existed, so no need to check its existance - if (pCtx->order == TSDB_ORDER_ASC) { - tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); - } else { - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *d = GET_INPUT_DATA(pCtx, i); - tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); - } - } - - SET_VAL(pCtx, pCtx->size, 1); - pResInfo->hasResult = DATA_SET_FLAG; -} - -static void ts_comp_finalize(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - STSBuf * pTSbuf = pInfo->pTSBuf; - - tsBufFlush(pTSbuf); - qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal); - - // TODO refactor transfer ownership of current file - *(FILE **)pCtx->pOutput = pTSbuf->f; - - pResInfo->complete = true; - - // get the file size - struct stat fStat; - if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) { - pResInfo->numOfRes = fStat.st_size; - } - - pTSbuf->remainOpen = true; - tsBufDestroy(pTSbuf); - - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////////////////// -// rate functions -static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) { - if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->firstKey) || - (pRateInfo->firstKey >= pRateInfo->lastKey)) { - return 0.0; - } - - double diff = 0; - if (pRateInfo->isIRate) { - // If the previous value of the last is greater than the last value, only keep the last point instead of the delta - // value between two values. - diff = pRateInfo->lastValue; - if (diff >= pRateInfo->firstValue) { - diff -= pRateInfo->firstValue; - } - } else { - diff = pRateInfo->correctionValue + pRateInfo->lastValue - pRateInfo->firstValue; - if (diff <= 0) { - return 0; - } - } - - int64_t duration = pRateInfo->lastKey - pRateInfo->firstKey; - if (duration == 0) { - return 0; - } - - return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0; -} - -static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SRateInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->correctionValue = 0; - pInfo->firstKey = INT64_MIN; - pInfo->lastKey = INT64_MIN; - pInfo->firstValue = (double) INT64_MIN; - pInfo->lastValue = (double) INT64_MIN; - - pInfo->hasResult = 0; - pInfo->isIRate = (pCtx->functionId == TSDB_FUNC_IRATE); - return true; -} - -static void rate_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = GET_TS_LIST(pCtx); - - qDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *pData = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - qDebug("%p rate_function() index of null data:%d", pCtx, i); - continue; - } - - notNullElems++; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, pData); - - if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { - pRateInfo->firstValue = v; - pRateInfo->firstKey = primaryKey[i]; - } - - if (INT64_MIN == pRateInfo->lastValue) { - pRateInfo->lastValue = v; - } else if (v < pRateInfo->lastValue) { - pRateInfo->correctionValue += pRateInfo->lastValue; - } - - pRateInfo->lastValue = v; - pRateInfo->lastKey = primaryKey[i]; - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pRateInfo->hasResult = DATA_SET_FLAG; - pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); - } -} - -static void rate_func_copy(SQLFunctionCtx *pCtx) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); - pResInfo->hasResult = ((SRateInfo*)pCtx->pInput)->hasResult; -} - -static void rate_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - - if (pRateInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - return; - } - - SET_DOUBLE_VAL((double*) pCtx->pOutput, do_calc_rate(pRateInfo, (double) TSDB_TICK_PER_SECOND(pCtx->param[0].i64))); - - // cannot set the numOfIteratedElems again since it is set during previous iteration - pResInfo->numOfRes = 1; - pResInfo->hasResult = DATA_SET_FLAG; - - doFinalizer(pCtx); -} - -static void irate_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = GET_TS_LIST(pCtx); - - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *pData = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - continue; - } - - notNullElems++; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, pData); - - if ((INT64_MIN == pRateInfo->lastKey) || primaryKey[i] > pRateInfo->lastKey) { - pRateInfo->lastValue = v; - pRateInfo->lastKey = primaryKey[i]; - continue; - } - - if ((INT64_MIN == pRateInfo->firstKey) || primaryKey[i] > pRateInfo->firstKey) { - pRateInfo->firstValue = v; - pRateInfo->firstKey = primaryKey[i]; - break; - } - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pRateInfo->hasResult = DATA_SET_FLAG; - pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); - } -} - -void blockInfo_func(SQLFunctionCtx* pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); - - int32_t len = *(int32_t*) pCtx->pInput; - blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); - pDist->rowSize = (uint16_t)pCtx->param[0].i64; - - memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len); - - pResInfo->numOfRes = 1; - pResInfo->hasResult = DATA_SET_FLAG; -} - -static void mergeTableBlockDist(SResultRowCellInfo* pResInfo, const STableBlockDist* pSrc) { - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); - assert(pDist != NULL && pSrc != NULL); - - pDist->numOfTables += pSrc->numOfTables; - pDist->numOfRowsInMemTable += pSrc->numOfRowsInMemTable; - pDist->numOfSmallBlocks += pSrc->numOfSmallBlocks; - pDist->numOfFiles += pSrc->numOfFiles; - pDist->totalSize += pSrc->totalSize; - pDist->totalRows += pSrc->totalRows; - - if (pResInfo->hasResult == DATA_SET_FLAG) { - pDist->maxRows = TMAX(pDist->maxRows, pSrc->maxRows); - pDist->minRows = TMIN(pDist->minRows, pSrc->minRows); - } else { - pDist->maxRows = pSrc->maxRows; - pDist->minRows = pSrc->minRows; - - int32_t maxSteps = TSDB_MAX_MAX_ROW_FBLOCK/TSDB_BLOCK_DIST_STEP_ROWS; - if (TSDB_MAX_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { - ++maxSteps; - } - pDist->dataBlockInfos = taosArrayInit(maxSteps, sizeof(SFileBlockInfo)); - taosArraySetSize(pDist->dataBlockInfos, maxSteps); - } - - size_t steps = taosArrayGetSize(pSrc->dataBlockInfos); - for (int32_t i = 0; i < steps; ++i) { - int32_t srcNumBlocks = ((SFileBlockInfo*)taosArrayGet(pSrc->dataBlockInfos, i))->numBlocksOfStep; - SFileBlockInfo* blockInfo = (SFileBlockInfo*)taosArrayGet(pDist->dataBlockInfos, i); - blockInfo->numBlocksOfStep += srcNumBlocks; - } -} - -void block_func_merge(SQLFunctionCtx* pCtx) { - STableBlockDist info = {0}; - int32_t len = *(int32_t*) pCtx->pInput; - blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - mergeTableBlockDist(pResInfo, &info); - taosArrayDestroy(info.dataBlockInfos); - - pResInfo->numOfRes = 1; - pResInfo->hasResult = DATA_SET_FLAG; -} - -void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents, - double* percents, int32_t* percentiles) { - if (totalBlocks == 0) { - for (int32_t i = 0; i < numOfPercents; ++i) { - percentiles[i] = 0; - } - return; - } - - SArray *blocksInfos = pTableBlockDist->dataBlockInfos; - size_t numSteps = taosArrayGetSize(blocksInfos); - size_t cumulativeBlocks = 0; - - int percentIndex = 0; - for (int32_t indexStep = 0; indexStep < numSteps; ++indexStep) { - int32_t numStepBlocks = ((SFileBlockInfo *)taosArrayGet(blocksInfos, indexStep))->numBlocksOfStep; - if (numStepBlocks == 0) continue; - cumulativeBlocks += numStepBlocks; - - while (percentIndex < numOfPercents) { - double blockRank = totalBlocks * percents[percentIndex]; - if (blockRank <= cumulativeBlocks) { - percentiles[percentIndex] = indexStep; - ++percentIndex; - } else { - break; - } - } - } - - for (int32_t i = 0; i < numOfPercents; ++i) { - percentiles[i] = (percentiles[i]+1) * TSDB_BLOCK_DIST_STEP_ROWS - TSDB_BLOCK_DIST_STEP_ROWS/2; - } -} - -void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { - if (pTableBlockDist == NULL) { - return; - } - - SArray* blockInfos = pTableBlockDist->dataBlockInfos; - uint64_t totalRows = pTableBlockDist->totalRows; - size_t numSteps = taosArrayGetSize(blockInfos); - int64_t totalBlocks = 0; - int64_t min = -1, max = -1, avg = 0; - - for (int32_t i = 0; i < numSteps; i++) { - SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); - int64_t blocks = blockInfo->numBlocksOfStep; - totalBlocks += blocks; - } - - avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0; - min = totalBlocks > 0 ? pTableBlockDist->minRows : 0; - max = totalBlocks > 0 ? pTableBlockDist->maxRows : 0; - - double stdDev = 0; - if (totalBlocks > 0) { - double variance = 0; - for (int32_t i = 0; i < numSteps; i++) { - SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); - int64_t blocks = blockInfo->numBlocksOfStep; - int32_t rows = (i * TSDB_BLOCK_DIST_STEP_ROWS + TSDB_BLOCK_DIST_STEP_ROWS / 2); - variance += blocks * (rows - avg) * (rows - avg); - } - variance = variance / totalBlocks; - stdDev = sqrt(variance); - } - - double percents[] = {0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.99}; - int32_t percentiles[] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; - assert(sizeof(percents)/sizeof(double) == sizeof(percentiles)/sizeof(int32_t)); - getPercentiles(pTableBlockDist, totalBlocks, sizeof(percents)/sizeof(double), percents, percentiles); - - uint64_t totalLen = pTableBlockDist->totalSize; - int32_t rowSize = pTableBlockDist->rowSize; - int32_t smallBlocks = pTableBlockDist->numOfSmallBlocks; - double compRatio = (totalRows>0) ? ((double)(totalLen)/(rowSize*totalRows)) : 1; - int sz = sprintf(result + VARSTR_HEADER_SIZE, - "summary: \n\t " - "5th=[%d], 10th=[%d], 20th=[%d], 30th=[%d], 40th=[%d], 50th=[%d]\n\t " - "60th=[%d], 70th=[%d], 80th=[%d], 90th=[%d], 95th=[%d], 99th=[%d]\n\t " - "Min=[%"PRId64"(Rows)] Max=[%"PRId64"(Rows)] Avg=[%"PRId64"(Rows)] Stddev=[%.2f] \n\t " - "Rows=[%"PRIu64"], Blocks=[%"PRId64"], SmallBlocks=[%d], Size=[%.3f(Kb)] Comp=[%.2f]\n\t " - "RowsInMem=[%d] \n\t", - percentiles[0], percentiles[1], percentiles[2], percentiles[3], percentiles[4], percentiles[5], - percentiles[6], percentiles[7], percentiles[8], percentiles[9], percentiles[10], percentiles[11], - min, max, avg, stdDev, - totalRows, totalBlocks, smallBlocks, totalLen/1024.0, compRatio, - pTableBlockDist->numOfRowsInMemTable); - varDataSetLen(result, sz); - UNUSED(sz); -} - -void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); - - pDist->rowSize = (uint16_t)pCtx->param[0].i64; - generateBlockDistResult(pDist, pCtx->pOutput); - - if (pDist->dataBlockInfos != NULL) { - taosArrayDestroy(pDist->dataBlockInfos); - pDist->dataBlockInfos = NULL; - } - - // cannot set the numOfIteratedElems again since it is set during previous iteration - pResInfo->numOfRes = 1; - pResInfo->hasResult = DATA_SET_FLAG; - - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////////////////// -/* - * function compatible list. - * tag and ts are not involved in the compatibility check - * - * 1. functions that are not simultaneously present with any other functions. e.g., diff/ts_z/top/bottom - * 2. functions that are only allowed to be present only with same functions. e.g., last_row, interp - * 3. functions that are allowed to be present with other functions. - * e.g., count/sum/avg/min/max/stddev/percentile/apercentile/first/last... - * - */ -int32_t functionCompatList[] = { - // count, sum, avg, min, max, stddev, percentile, apercentile, first, last - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - // last_row,top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_comp - 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, - // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, stddev_dst, interp rate irate - 1, 1, 1, 1, -1, 1, 1, 1, 5, 1, 1, - // tid_tag, derivative, blk_info - 6, 8, 7, -}; - -SAggFunctionInfo aAggs[] = {{ - // 0, count function does not invoke the finalize function - "count", - TSDB_FUNC_COUNT, - TSDB_FUNC_COUNT, - TSDB_BASE_FUNC_SO, - function_setup, - count_function, - doFinalizer, - count_func_merge, - countRequired, - }, - { - // 1 - "sum", - TSDB_FUNC_SUM, - TSDB_FUNC_SUM, - TSDB_BASE_FUNC_SO, - function_setup, - sum_function, - function_finalizer, - sum_func_merge, - statisRequired, - }, - { - // 2 - "avg", - TSDB_FUNC_AVG, - TSDB_FUNC_AVG, - TSDB_BASE_FUNC_SO, - function_setup, - avg_function, - avg_finalizer, - avg_func_merge, - statisRequired, - }, - { - // 3 - "min", - TSDB_FUNC_MIN, - TSDB_FUNC_MIN, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - min_func_setup, - min_function, - function_finalizer, - min_func_merge, - statisRequired, - }, - { - // 4 - "max", - TSDB_FUNC_MAX, - TSDB_FUNC_MAX, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - max_func_setup, - max_function, - function_finalizer, - max_func_merge, - statisRequired, - }, - { - // 5 - "stddev", - TSDB_FUNC_STDDEV, - TSDB_FUNC_STDDEV_DST, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, - function_setup, - stddev_function, - stddev_finalizer, - noop1, - dataBlockRequired, - }, - { - // 6 - "percentile", - TSDB_FUNC_PERCT, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, - percentile_function_setup, - percentile_function, - percentile_finalizer, - noop1, - dataBlockRequired, - }, - { - // 7 - "apercentile", - TSDB_FUNC_APERCT, - TSDB_FUNC_APERCT, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_STABLE, - apercentile_function_setup, - apercentile_function, - apercentile_finalizer, - apercentile_func_merge, - dataBlockRequired, - }, - { - // 8 - "first", - TSDB_FUNC_FIRST, - TSDB_FUNC_FIRST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - function_setup, - first_function, - function_finalizer, - noop1, - firstFuncRequired, - }, - { - // 9 - "last", - TSDB_FUNC_LAST, - TSDB_FUNC_LAST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - function_setup, - last_function, - function_finalizer, - noop1, - lastFuncRequired, - }, - { - // 10 - "last_row", - TSDB_FUNC_LAST_ROW, - TSDB_FUNC_LAST_ROW, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS | - TSDB_FUNCSTATE_SELECTIVITY, - first_last_function_setup, - last_row_function, - last_row_finalizer, - last_dist_func_merge, - dataBlockRequired, - }, - { - // 11 - "top", - TSDB_FUNC_TOP, - TSDB_FUNC_TOP, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | - TSDB_FUNCSTATE_SELECTIVITY, - top_bottom_function_setup, - top_function, - top_bottom_func_finalizer, - top_func_merge, - dataBlockRequired, - }, - { - // 12 - "bottom", - TSDB_FUNC_BOTTOM, - TSDB_FUNC_BOTTOM, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | - TSDB_FUNCSTATE_SELECTIVITY, - top_bottom_function_setup, - bottom_function, - top_bottom_func_finalizer, - bottom_func_merge, - dataBlockRequired, - }, - { - // 13 - "spread", - TSDB_FUNC_SPREAD, - TSDB_FUNC_SPREAD, - TSDB_BASE_FUNC_SO, - spread_function_setup, - spread_function, - spread_function_finalizer, - spread_func_merge, - countRequired, - }, - { - // 14 - "twa", - TSDB_FUNC_TWA, - TSDB_FUNC_TWA, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - twa_function_setup, - twa_function, - twa_function_finalizer, - twa_function_copy, - dataBlockRequired, - }, - { - // 15 - "leastsquares", - TSDB_FUNC_LEASTSQR, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, - leastsquares_function_setup, - leastsquares_function, - leastsquares_finalizer, - noop1, - dataBlockRequired, - }, - { - // 16 - "ts", - TSDB_FUNC_TS, - TSDB_FUNC_TS, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - function_setup, - date_col_output_function, - doFinalizer, - copy_function, - noDataRequired, - }, - { - // 17 - "ts", - TSDB_FUNC_TS_DUMMY, - TSDB_FUNC_TS_DUMMY, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - function_setup, - noop1, - doFinalizer, - copy_function, - dataBlockRequired, - }, - { - // 18 - "tag_dummy", - TSDB_FUNC_TAG_DUMMY, - TSDB_FUNC_TAG_DUMMY, - TSDB_BASE_FUNC_SO, - function_setup, - tag_function, - doFinalizer, - copy_function, - noDataRequired, - }, - { - // 19 - "ts", - TSDB_FUNC_TS_COMP, - TSDB_FUNC_TS_COMP, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, - ts_comp_function_setup, - ts_comp_function, - ts_comp_finalize, - copy_function, - dataBlockRequired, - }, - { - // 20 - "tag", - TSDB_FUNC_TAG, - TSDB_FUNC_TAG, - TSDB_BASE_FUNC_SO, - function_setup, - tag_function, - doFinalizer, - copy_function, - noDataRequired, - }, - { - // 21, column project sql function - "colprj", - TSDB_FUNC_PRJ, - TSDB_FUNC_PRJ, - TSDB_BASE_FUNC_MO | TSDB_FUNCSTATE_NEED_TS, - function_setup, - col_project_function, - doFinalizer, - copy_function, - dataBlockRequired, - }, - { - // 22, multi-output, tag function has only one result - "tagprj", - TSDB_FUNC_TAGPRJ, - TSDB_FUNC_TAGPRJ, - TSDB_BASE_FUNC_MO, - function_setup, - tag_project_function, - doFinalizer, - copy_function, - noDataRequired, - }, - { - // 23 - "arithmetic", - TSDB_FUNC_ARITHM, - TSDB_FUNC_ARITHM, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS, - function_setup, - arithmetic_function, - doFinalizer, - copy_function, - dataBlockRequired, - }, - { - // 24 - "diff", - TSDB_FUNC_DIFF, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, - diff_function_setup, - diff_function, - doFinalizer, - noop1, - dataBlockRequired, - }, - // distributed version used in two-stage aggregation processes - { - // 25 - "first_dist", - TSDB_FUNC_FIRST_DST, - TSDB_FUNC_FIRST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, - first_last_function_setup, - first_dist_function, - function_finalizer, - first_dist_func_merge, - firstDistFuncRequired, - }, - { - // 26 - "last_dist", - TSDB_FUNC_LAST_DST, - TSDB_FUNC_LAST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, - first_last_function_setup, - last_dist_function, - function_finalizer, - last_dist_func_merge, - lastDistFuncRequired, - }, - { - // 27 - "stddev", // return table id and the corresponding tags for join match and subscribe - TSDB_FUNC_STDDEV_DST, - TSDB_FUNC_AVG, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STABLE, - function_setup, - stddev_dst_function, - stddev_dst_finalizer, - stddev_dst_merge, - dataBlockRequired, - }, - { - // 28 - "interp", - TSDB_FUNC_INTERP, - TSDB_FUNC_INTERP, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS , - function_setup, - interp_function, - doFinalizer, - copy_function, - dataBlockRequired, - }, - { - // 29 - "rate", - TSDB_FUNC_RATE, - TSDB_FUNC_RATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - rate_function, - rate_finalizer, - rate_func_copy, - dataBlockRequired, - }, - { - // 30 - "irate", - TSDB_FUNC_IRATE, - TSDB_FUNC_IRATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - irate_function, - rate_finalizer, - rate_func_copy, - dataBlockRequired, - }, - { - // 31 - "tbid", // return table id and the corresponding tags for join match and subscribe - TSDB_FUNC_TID_TAG, - TSDB_FUNC_TID_TAG, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE, - function_setup, - noop1, - noop1, - noop1, - dataBlockRequired, - }, - { //32 - "derivative", // return table id and the corresponding tags for join match and subscribe - TSDB_FUNC_DERIVATIVE, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, - deriv_function_setup, - deriv_function, - doFinalizer, - noop1, - dataBlockRequired, - }, - { - // 33 - "_block_dist", // return table id and the corresponding tags for join match and subscribe - TSDB_FUNC_BLKINFO, - TSDB_FUNC_BLKINFO, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STABLE, - function_setup, - blockInfo_func, - blockinfo_func_finalizer, - block_func_merge, - dataBlockRequired, - }}; diff --git a/2.0/src/query/src/qExecutor.c b/2.0/src/query/src/qExecutor.c deleted file mode 100644 index 0c37a3d330bad5ea20cd78717e71a54ba1a7be81..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qExecutor.c +++ /dev/null @@ -1,8990 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "os.h" -#include "qFill.h" -#include "tmsg.h" -#include "tglobal.h" - -#include "exception.h" -#include "hash.h" -#include "qExecutor.h" -#include "qResultbuf.h" -#include "qUtil.h" -#include "queryLog.h" -#include "tcompare.h" -#include "tcompression.h" -#include "texpr.h" -#include "tlosertree.h" -#include "tscLog.h" -#include "tscript.h" -#include "ttype.h" - -#define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) -#define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) -#define IS_REPEAT_SCAN(runtime) ((runtime)->scanFlag == REPEAT_SCAN) -#define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) -#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) - -#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) -#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) - -#define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0} - -#define MULTI_KEY_DELIM "-" - -#define TIME_WINDOW_COPY(_dst, _src) do {\ - (_dst).skey = (_src).skey;\ - (_dst).ekey = (_src).ekey;\ -} while (0) - -enum { - TS_JOIN_TS_EQUAL = 0, - TS_JOIN_TS_NOT_EQUALS = 1, - TS_JOIN_TAG_NOT_EQUALS = 2, -}; - -typedef enum SResultTsInterpType { - RESULT_ROW_START_INTERP = 1, - RESULT_ROW_END_INTERP = 2, -} SResultTsInterpType; - -#if 0 -static UNUSED_FUNC void *u_malloc (size_t __size) { - uint32_t v = rand(); - - if (v % 1000 <= 0) { - return NULL; - } else { - return malloc(__size); - } -} - -static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { - uint32_t v = rand(); - if (v % 1000 <= 0) { - return NULL; - } else { - return calloc(num, __size); - } -} - -static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { - uint32_t v = rand(); - if (v % 5 <= 1) { - return NULL; - } else { - return realloc(p, __size); - } -} - -#define calloc u_calloc -#define malloc u_malloc -#define realloc u_realloc -#endif - -#define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) -#define GET_NUM_OF_TABLEGROUP(q) taosArrayGetSize((q)->tableqinfoGroupInfo.pGroupList) -#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) - -#define TSKEY_MAX_ADD(a,b) \ -do { \ - if (a < 0) { a = a + b; break;} \ - if (sizeof(a) == sizeof(int32_t)) { \ - if((b) > 0 && ((b) >= INT32_MAX - (a))){\ - a = INT32_MAX; \ - } else { \ - a = a + b; \ - } \ - } else { \ - if((b) > 0 && ((b) >= INT64_MAX - (a))){\ - a = INT64_MAX; \ - } else { \ - a = a + b; \ - } \ - } \ -} while(0) - -#define TSKEY_MIN_SUB(a,b) \ -do { \ - if (a >= 0) { a = a + b; break;} \ - if (sizeof(a) == sizeof(int32_t)){ \ - if((b) < 0 && ((b) <= INT32_MIN - (a))){\ - a = INT32_MIN; \ - } else { \ - a = a + b; \ - } \ - } else { \ - if((b) < 0 && ((b) <= INT64_MIN-(a))) {\ - a = INT64_MIN; \ - } else { \ - a = a + b; \ - } \ - } \ -} while (0) - -uint64_t queryHandleId = 0; - -int32_t getMaximumIdleDurationSec() { - return tsShellActivityTimer * 2; -} -int64_t genQueryId(void) { - int64_t uid = 0; - int64_t did = tsDnodeId; - - uid = did << 54; - - int64_t pid = ((int64_t)taosGetPId()) & 0x3FF; - - uid |= pid << 44; - - int64_t ts = taosGetTimestampMs() & 0x1FFFFFFFF; - - uid |= ts << 11; - - int64_t sid = atomic_add_fetch_64(&queryHandleId, 1) & 0x7FF; - - uid |= sid; - - qDebug("gen qid:0x%"PRIx64, uid); - - return uid; -} - -static void getNextTimeWindow(SQueryAttr* pQueryAttr, STimeWindow* tw) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - if (pQueryAttr->interval.intervalUnit != 'n' && pQueryAttr->interval.intervalUnit != 'y') { - tw->skey += pQueryAttr->interval.sliding * factor; - tw->ekey = tw->skey + pQueryAttr->interval.interval - 1; - return; - } - - int64_t key = tw->skey, interval = pQueryAttr->interval.interval; - //convert key to second - key = convertTimePrecision(key, pQueryAttr->precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pQueryAttr->interval.intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - localtime_r(&t, &tm); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); - - tw->ekey -= 1; -} - -static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); -static void setResultOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SQLFunctionCtx* pCtx, - int32_t numOfCols, int32_t* rowCellInfoOffset); - -void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); -static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx); - -static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex); - -static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); -static bool hasMainOutput(SQueryAttr *pQueryAttr); - -static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols); - -static int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, tVariant* pTag, STableQueryInfo *pTableQueryInfo); -static void releaseQueryBuf(size_t numOfTables); -static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); -static STsdbQueryCond createTsdbQueryCond(SQueryAttr* pQueryAttr, STimeWindow* win); -static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo); - -static void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream); - -static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr); - -static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); -static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); -static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput); -static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput); -static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); -static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); -static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); -static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); -static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); -static void destroyOperatorInfo(SOperatorInfo* pOperator); - -static void doSetOperatorCompleted(SOperatorInfo* pOperator) { - pOperator->status = OP_EXEC_DONE; - if (pOperator->pRuntimeEnv != NULL) { - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); - } -} - -static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); - -static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); -static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); - -static void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size); -static void getAlignQueryTimeWindow(SQueryAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); -static void setResultBufSize(SQueryAttr* pQueryAttr, SRspResultInfo* pResultInfo); -static void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); -static void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); -static void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); -static void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, - SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId); - -SArray* getOrderCheckColumns(SQueryAttr* pQuery); - - -typedef struct SRowCompSupporter { - SQueryRuntimeEnv *pRuntimeEnv; - int16_t dataOffset; - __compar_fn_t comFunc; -} SRowCompSupporter; - -static int compareRowData(const void *a, const void *b, const void *userData) { - const SResultRow *pRow1 = (const SResultRow *)a; - const SResultRow *pRow2 = (const SResultRow *)b; - - SRowCompSupporter *supporter = (SRowCompSupporter *)userData; - SQueryRuntimeEnv* pRuntimeEnv = supporter->pRuntimeEnv; - - tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pRow1->pageId); - tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pRow2->pageId); - - int16_t offset = supporter->dataOffset; - char *in1 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset); - char *in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset); - - return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0; -} - -static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, SSDataBlock* pDataBlock) { - SArray *columnOrderList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); - size_t size = taosArrayGetSize(columnOrderList); - taosArrayDestroy(columnOrderList); - - if (size <= 0) { - return; - } - - int32_t orderId = pRuntimeEnv->pQueryAttr->order.orderColId; - if (orderId <= 0) { - return; - } - - bool found = false; - int16_t dataOffset = 0; - - for (int32_t j = 0; j < pDataBlock->info.numOfCols; ++j) { - SColumnInfoData* pColInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock->pDataBlock, j); - if (orderId == j) { - found = true; - break; - } - - dataOffset += pColInfoData->info.bytes; - } - - if (found == false) { - return; - } - - int16_t type = pRuntimeEnv->pQueryAttr->pExpr1[orderId].base.resType; - - SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)}; - taosArraySortPWithExt(pGroupResInfo->pRows, compareRowData, &support); -} - -//setup the output buffer for each operator -SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows) { - const static int32_t minSize = 8; - - SSDataBlock *res = calloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = numOfOutput; - - res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); - for (int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData idata = {{0}}; - idata.info.type = pExpr[i].base.resType; - idata.info.bytes = pExpr[i].base.resBytes; - idata.info.colId = pExpr[i].base.resColId; - - int32_t size = TMAX(idata.info.bytes * numOfRows, minSize); - idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform - taosArrayPush(res->pDataBlock, &idata); - } - - return res; -} - -void* blockDataDestroy(SSDataBlock* pBlock) { - if (pBlock == NULL) { - return NULL; - } - - int32_t numOfOutput = pBlock->info.numOfCols; - for(int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - tfree(pColInfoData->pData); - } - - taosArrayDestroy(pBlock->pDataBlock); - tfree(pBlock->pBlockStatis); - tfree(pBlock); - return NULL; -} - -int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - bool hasMainFunction = hasMainOutput(pQueryAttr); - - int32_t maxOutput = 0; - for (int32_t j = 0; j < numOfOutput; ++j) { - int32_t id = pCtx[j].functionId; - - /* - * ts, tag, tagprj function can not decide the output number of current query - * the number of output result is decided by main output - */ - if (hasMainFunction && (id == TSDB_FUNC_TS || id == TSDB_FUNC_TAG || id == TSDB_FUNC_TAGPRJ)) { - continue; - } - - SResultRowCellInfo *pResInfo = GET_RES_INFO(&pCtx[j]); - if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { - maxOutput = pResInfo->numOfRes; - } - } - - assert(maxOutput >= 0); - return maxOutput; -} - -static void clearNumOfRes(SQLFunctionCtx* pCtx, int32_t numOfOutput) { - for (int32_t j = 0; j < numOfOutput; ++j) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(&pCtx[j]); - pResInfo->numOfRes = 0; - } -} - -static bool isSelectivityWithTagsQuery(SQLFunctionCtx *pCtx, int32_t numOfOutput) { - bool hasTags = false; - int32_t numOfSelectivity = 0; - - for (int32_t i = 0; i < numOfOutput; ++i) { - int32_t functId = pCtx[i].functionId; - if (functId == TSDB_FUNC_TAG_DUMMY || functId == TSDB_FUNC_TS_DUMMY) { - hasTags = true; - continue; - } - - if ((aAggs[functId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - numOfSelectivity++; - } - } - - return (numOfSelectivity > 0 && hasTags); -} - -static bool isProjQuery(SQueryAttr *pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functId = pQueryAttr->pExpr1[i].base.functionId; - if (functId != TSDB_FUNC_PRJ && functId != TSDB_FUNC_TAGPRJ) { - return false; - } - } - - return true; -} - -static bool hasNull(SColIndex* pColIndex, SDataStatis *pStatis) { - if (TSDB_COL_IS_TAG(pColIndex->flag) || TSDB_COL_IS_UD_COL(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return false; - } - - if (pStatis != NULL && pStatis->numOfNull == 0) { - return false; - } - - return true; -} - -static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, SQueryRuntimeEnv* pRuntimeEnv) { - // more than the capacity, reallocate the resources - if (pResultRowInfo->size < pResultRowInfo->capacity) { - return; - } - - int64_t newCapacity = 0; - if (pResultRowInfo->capacity > 10000) { - newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25); - } else { - newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); - } - - char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); - if (t == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - pResultRowInfo->pResult = (SResultRow **)t; - - int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; - memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc); - - pResultRowInfo->capacity = (int32_t)newCapacity; -} - -static bool chkResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, - int16_t bytes, bool masterscan, uint64_t uid) { - bool existed = false; - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); - - SResultRow **p1 = - (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - - // in case of repeat scan/reverse scan, no new time window added. - if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQueryAttr)) { - if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. - return p1 != NULL; - } - - if (p1 != NULL) { - if (pResultRowInfo->size == 0) { - existed = false; - assert(pResultRowInfo->curPos == -1); - } else if (pResultRowInfo->size == 1) { - existed = (pResultRowInfo->pResult[0] == (*p1)); - } else { // check if current pResultRowInfo contains the existed pResultRow - SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid, pResultRowInfo); - int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); - if (index != NULL) { - existed = true; - } else { - existed = false; - } - } - } - - return existed; - } - - return p1 != NULL; -} - - -static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, int64_t tid, - char* pData, int16_t bytes, bool masterscan, uint64_t tableGroupId) { - bool existed = false; - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tableGroupId); - - SResultRow **p1 = - (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - - // in case of repeat scan/reverse scan, no new time window added. - if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQueryAttr)) { - if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. - return (p1 != NULL)? *p1:NULL; - } - - if (p1 != NULL) { - if (pResultRowInfo->size == 0) { - existed = false; - assert(pResultRowInfo->curPos == -1); - } else if (pResultRowInfo->size == 1) { - existed = (pResultRowInfo->pResult[0] == (*p1)); - pResultRowInfo->curPos = 0; - } else { // check if current pResultRowInfo contains the existed pResultRow - SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); - int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); - if (index != NULL) { - pResultRowInfo->curPos = (int32_t) *index; - existed = true; - } else { - existed = false; - } - } - } - } else { - // In case of group by column query, the required SResultRow object must be existed in the pResultRowInfo object. - if (p1 != NULL) { - return *p1; - } - } - - if (!existed) { - prepareResultListBuffer(pResultRowInfo, pRuntimeEnv); - - SResultRow *pResult = NULL; - if (p1 == NULL) { - pResult = getNewResultRow(pRuntimeEnv->pool); - int32_t ret = initResultRow(pResult); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - // add a new result set for a new group - taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); - SResultRowCell cell = {.groupId = tableGroupId, .pRow = pResult}; - taosArrayPush(pRuntimeEnv->pResultRowArrayList, &cell); - } else { - pResult = *p1; - } - - pResultRowInfo->curPos = pResultRowInfo->size; - pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; - - int64_t index = pResultRowInfo->curPos; - SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); - taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); - } - - // too many time window in query - if (pResultRowInfo->size > MAX_INTERVAL_TIME_WINDOW) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); - } - - return pResultRowInfo->pResult[pResultRowInfo->curPos]; -} - -static void getInitialStartTimeWindow(SQueryAttr* pQueryAttr, TSKEY ts, STimeWindow* w) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - getAlignQueryTimeWindow(pQueryAttr, ts, ts, pQueryAttr->window.ekey, w); - } else { - // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - getAlignQueryTimeWindow(pQueryAttr, ts, pQueryAttr->window.ekey, ts, w); - - int64_t key = w->skey; - while(key < ts) { // moving towards end - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - key = taosTimeAdd(key, pQueryAttr->interval.sliding, pQueryAttr->interval.slidingUnit, pQueryAttr->precision); - } else { - key += pQueryAttr->interval.sliding; - } - - if (key >= ts) { - break; - } - - w->skey = key; - } - } -} - -// get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQueryAttr *pQueryAttr) { - STimeWindow w = {0}; - - if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value - getInitialStartTimeWindow(pQueryAttr, ts, &w); - - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; - } else { - w.ekey = w.skey + pQueryAttr->interval.interval - 1; - } - } else { - w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; - } - - if (w.skey > ts || w.ekey < ts) { - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - w.skey = taosTimeTruncate(ts, &pQueryAttr->interval, pQueryAttr->precision); - w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; - } else { - int64_t st = w.skey; - - if (st > ts) { - st -= ((st - ts + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; - } - - int64_t et = st + pQueryAttr->interval.interval - 1; - if (et < ts) { - st += ((ts - et + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; - } - - w.skey = st; - w.ekey = w.skey + pQueryAttr->interval.interval - 1; - } - } - - /* - * query border check, skey should not be bounded by the query time range, since the value skey will - * be used as the time window index value. So we only change ekey of time window accordingly. - */ - if (w.ekey > pQueryAttr->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) { - w.ekey = pQueryAttr->window.ekey; - } - - return w; -} - -// get the correct time window according to the handled timestamp -static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQueryAttr *pQueryAttr) { - STimeWindow w = {0}; - - if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value - getInitialStartTimeWindow(pQueryAttr, ts, &w); - - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; - } else { - w.ekey = w.skey + pQueryAttr->interval.interval - 1; - } - } else { - w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; - } - - /* - * query border check, skey should not be bounded by the query time range, since the value skey will - * be used as the time window index value. So we only change ekey of time window accordingly. - */ - if (w.ekey > pQueryAttr->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) { - w.ekey = pQueryAttr->window.ekey; - } - - return w; -} - - - -// a new buffer page for each table. Needs to opt this design -static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, uint32_t size) { - if (pWindowRes->pageId != -1) { - return 0; - } - - tFilePage *pData = NULL; - - // in the first scan, new space needed for results - int32_t pageId = -1; - SIDList list = getDataBufPagesIdList(pResultBuf, tid); - - if (taosArrayGetSize(list) == 0) { - pData = getNewDataBuf(pResultBuf, tid, &pageId); - } else { - SPageInfo* pi = getLastPageInfo(list); - pData = getResBufPage(pResultBuf, pi->pageId); - pageId = pi->pageId; - - if (pData->num + size > pResultBuf->pageSize) { - // release current page first, and prepare the next one - releaseResBufPageInfo(pResultBuf, pi); - pData = getNewDataBuf(pResultBuf, tid, &pageId); - if (pData != NULL) { - assert(pData->num == 0); // number of elements must be 0 for new allocated buffer - } - } - } - - if (pData == NULL) { - return -1; - } - - // set the number of rows in current disk page - if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer - pWindowRes->pageId = pageId; - pWindowRes->offset = (int32_t)pData->num; - - pData->num += size; - assert(pWindowRes->pageId >= 0); - } - - return 0; -} - -static bool chkWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, - bool masterscan, SResultRow **pResult, int64_t groupId, SQLFunctionCtx* pCtx, - int32_t numOfOutput, int32_t* rowCellInfoOffset) { - assert(win->skey <= win->ekey); - - return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); -} - -static int32_t setResultOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, STimeWindow *win, - bool masterscan, SResultRow **pResult, int64_t tableGroupId, SQLFunctionCtx* pCtx, - int32_t numOfOutput, int32_t* rowCellInfoOffset) { - assert(win->skey <= win->ekey); - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - - SResultRow *pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId); - if (pResultRow == NULL) { - *pResult = NULL; - return TSDB_CODE_SUCCESS; - } - - // not assign result buffer yet, add new result buffer - if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) tableGroupId, pRuntimeEnv->pQueryAttr->intermediateResultRowSize); - if (ret != TSDB_CODE_SUCCESS) { - return -1; - } - } - - // set time window for current result - pResultRow->win = (*win); - *pResult = pResultRow; - setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); - - return TSDB_CODE_SUCCESS; -} - -static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { - assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); - if (type == RESULT_ROW_START_INTERP) { - pResult->startInterp = true; - } else { - pResult->endInterp = true; - } -} - -static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { - assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); - if (type == RESULT_ROW_START_INTERP) { - return pResult->startInterp == true; - } else { - return pResult->endInterp == true; - } -} - -static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, - int16_t order, int64_t *pData) { - int32_t forwardStep = 0; - - if (order == TSDB_ORDER_ASC) { - int32_t end = searchFn((char*) &pData[pos], numOfRows - pos, ekey, order); - if (end >= 0) { - forwardStep = end; - - if (pData[end + pos] == ekey) { - forwardStep += 1; - } - } - } else { - int32_t end = searchFn((char *)pData, pos + 1, ekey, order); - if (end >= 0) { - forwardStep = pos - end; - - if (pData[end] == ekey) { - forwardStep += 1; - } - } - } - - assert(forwardStep >= 0); - return forwardStep; -} - -static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) { - int64_t skey = TSKEY_INITIAL_VAL; - int32_t i = 0; - for (i = pResultRowInfo->size - 1; i >= 0; --i) { - SResultRow *pResult = pResultRowInfo->pResult[i]; - if (pResult->closed) { - break; - } - - // new closed result rows - if (timeWindowInterpo) { - if (pResult->endInterp && ((pResult->win.skey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery))) { - if (i > 0) { // the first time window, the startInterp is false. - assert(pResult->startInterp); - } - - closeResultRow(pResultRowInfo, i); - } else { - skey = pResult->win.skey; - } - } else { - if ((pResult->win.ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) { - closeResultRow(pResultRowInfo, i); - } else { - skey = pResult->win.skey; - } - } - } - - // all result rows are closed, set the last one to be the skey - if (skey == TSKEY_INITIAL_VAL) { - if (pResultRowInfo->size == 0) { -// assert(pResultRowInfo->current == NULL); - assert(pResultRowInfo->curPos == -1); - pResultRowInfo->curPos = -1; - } else { - pResultRowInfo->curPos = pResultRowInfo->size - 1; - } - } else { - - for (i = pResultRowInfo->size - 1; i >= 0; --i) { - SResultRow *pResult = pResultRowInfo->pResult[i]; - if (pResult->closed) { - break; - } - } - - if (i == pResultRowInfo->size - 1) { - pResultRowInfo->curPos = i; - } else { - pResultRowInfo->curPos = i + 1; // current not closed result object - } - } - - //pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->curIndex]->win.skey; -} - -static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, SQueryAttr* pQueryAttr, TSKEY lastKey) { - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - if ((lastKey > pQueryAttr->window.ekey && ascQuery) || (lastKey < pQueryAttr->window.ekey && (!ascQuery))) { - closeAllResultRows(pResultRowInfo); - pResultRowInfo->curPos = pResultRowInfo->size - 1; - } else { - int32_t step = ascQuery ? 1 : -1; - doUpdateResultRowIndex(pResultRowInfo, lastKey - step, ascQuery, pQueryAttr->timeWindowInterpo); - } -} - -static int32_t getNumOfRowsInTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, - int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { - assert(startPos >= 0 && startPos < pDataBlockInfo->rows); - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - STableQueryInfo* item = pRuntimeEnv->current; - - int32_t num = -1; - int32_t order = pQueryAttr->order.order; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) { - num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); - if (updateLastKey) { // update the last key - item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; - } - } else { - num = pDataBlockInfo->rows - startPos; - if (updateLastKey) { - item->lastKey = pDataBlockInfo->window.ekey + step; - } - } - } else { // desc - if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) { - num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); - if (updateLastKey) { // update the last key - item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; - } - } else { - num = startPos + 1; - if (updateLastKey) { - item->lastKey = pDataBlockInfo->window.skey + step; - } - } - } - - assert(num >= 0); - return num; -} - -void doInvokeUdf(SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type) { - int32_t output = 0; - - if (pUdfInfo == NULL || pUdfInfo->funcs[type] == NULL) { - qError("empty udf function, type:%d", type); - return; - } - - qDebug("invoke udf function:%s,%p", pUdfInfo->name, pUdfInfo->funcs[type]); - - switch (type) { - case TSDB_UDF_FUNC_NORMAL: - if (pUdfInfo->isScript) { - (*(scriptNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])(pUdfInfo->pScriptCtx, - (char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, pCtx->startTs, pCtx->pOutput, - (char *)pCtx->ptsOutputBuf, &output, pCtx->outputType, pCtx->outputBytes); - } else { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); - - (*(udfNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])((char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, - pCtx->pOutput, interBuf, (char *)pCtx->ptsOutputBuf, &output, pCtx->outputType, pCtx->outputBytes, &pUdfInfo->init); - } - - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - pCtx->resultInfo->numOfRes = output; - } else { - pCtx->resultInfo->numOfRes += output; - } - - if (pCtx->resultInfo->numOfRes > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } - - break; - - case TSDB_UDF_FUNC_MERGE: - if (pUdfInfo->isScript) { - (*(scriptMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pUdfInfo->pScriptCtx, pCtx->pInput, pCtx->size, pCtx->pOutput, &output); - } else { - (*(udfMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pCtx->pInput, pCtx->size, pCtx->pOutput, &output, &pUdfInfo->init); - } - - // set the output value exist - pCtx->resultInfo->numOfRes = output; - if (output > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } - - break; - - case TSDB_UDF_FUNC_FINALIZE: { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); - if (pUdfInfo->isScript) { - (*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, &output); - } else { - (*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, &output, &pUdfInfo->init); - } - // set the output value exist - pCtx->resultInfo->numOfRes = output; - if (output > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } - - break; - } - } -} - -static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, - int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - bool hasAggregates = pCtx[0].preAggVals.isSet; - - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].size = forwardStep; - pCtx[k].startTs = pWin->skey; - - // keep it temporarialy - char* start = pCtx[k].pInput; - - int32_t pos = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? offset : offset - (forwardStep - 1); - if (pCtx[k].pInput != NULL) { - pCtx[k].pInput = (char *)pCtx[k].pInput + pos * pCtx[k].inputBytes; - } - - if (tsCol != NULL) { - pCtx[k].ptsList = &tsCol[pos]; - } - - // not a whole block involved in query processing, statistics data can not be used - // NOTE: the original value of isSet have been changed here - if (pCtx[k].preAggVals.isSet && forwardStep < numOfTotal) { - pCtx[k].preAggVals.isSet = false; - } - - int32_t functionId = pCtx[k].functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { - if (functionId < 0) { // load the script and exec, pRuntimeEnv->pUdfInfo - SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; - doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL); - } else { - aAggs[functionId].xFunction(&pCtx[k]); - } - } - - // restore it - pCtx[k].preAggVals.isSet = hasAggregates; - pCtx[k].pInput = start; - } -} - - -static int32_t getNextQualifiedWindow(SQueryAttr* pQueryAttr, STimeWindow *pNext, SDataBlockInfo *pDataBlockInfo, - TSKEY *primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) { - getNextTimeWindow(pQueryAttr, pNext); - - // next time window is not in current block - if ((pNext->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) || - (pNext->ekey < pDataBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { - return -1; - } - - TSKEY startKey = -1; - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - startKey = pNext->skey; - if (startKey < pQueryAttr->window.skey) { - startKey = pQueryAttr->window.skey; - } - } else { - startKey = pNext->ekey; - if (startKey > pQueryAttr->window.skey) { - startKey = pQueryAttr->window.skey; - } - } - - int32_t startPos = 0; - - // tumbling time window query, a special case of sliding time window query - if (pQueryAttr->interval.sliding == pQueryAttr->interval.interval && prevPosition != -1) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - startPos = prevPosition + factor; - } else { - if (startKey <= pDataBlockInfo->window.skey && QUERY_IS_ASC_QUERY(pQueryAttr)) { - startPos = 0; - } else if (startKey >= pDataBlockInfo->window.ekey && !QUERY_IS_ASC_QUERY(pQueryAttr)) { - startPos = pDataBlockInfo->rows - 1; - } else { - startPos = searchFn((char *)primaryKeys, pDataBlockInfo->rows, startKey, pQueryAttr->order.order); - } - } - - /* interp query with fill should not skip time window */ - if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) { - return startPos; - } - - /* - * This time window does not cover any data, try next time window, - * this case may happen when the time window is too small - */ - if (primaryKeys == NULL) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - assert(pDataBlockInfo->window.skey <= pNext->ekey); - } else { - assert(pDataBlockInfo->window.ekey >= pNext->skey); - } - } else { - if (QUERY_IS_ASC_QUERY(pQueryAttr) && primaryKeys[startPos] > pNext->ekey) { - TSKEY next = primaryKeys[startPos]; - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, &pQueryAttr->interval, pQueryAttr->precision); - pNext->ekey = taosTimeAdd(pNext->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; - } else { - pNext->ekey += ((next - pNext->ekey + pQueryAttr->interval.sliding - 1)/pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; - pNext->skey = pNext->ekey - pQueryAttr->interval.interval + 1; - } - } else if ((!QUERY_IS_ASC_QUERY(pQueryAttr)) && primaryKeys[startPos] < pNext->skey) { - TSKEY next = primaryKeys[startPos]; - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, &pQueryAttr->interval, pQueryAttr->precision); - pNext->ekey = taosTimeAdd(pNext->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; - } else { - pNext->skey -= ((pNext->skey - next + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; - pNext->ekey = pNext->skey + pQueryAttr->interval.interval - 1; - } - } - } - - return startPos; -} - -static FORCE_INLINE TSKEY reviseWindowEkey(SQueryAttr *pQueryAttr, STimeWindow *pWindow) { - TSKEY ekey = -1; - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - ekey = pWindow->ekey; - if (ekey > pQueryAttr->window.ekey) { - ekey = pQueryAttr->window.ekey; - } - } else { - ekey = pWindow->skey; - if (ekey < pQueryAttr->window.ekey) { - ekey = pQueryAttr->window.ekey; - } - } - - return ekey; -} - -static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { - if (type == RESULT_ROW_START_INTERP) { - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].start.key = INT64_MIN; - } - } else { - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].end.key = INT64_MIN; - } - } -} - -static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, - int32_t rowIndex) { - if (pDataBlock == NULL) { - return; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - for (int32_t k = 0; k < pQueryAttr->numOfCols; ++k) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); - memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); - } -} - -static TSKEY getStartTsKey(SQueryAttr* pQueryAttr, STimeWindow* win, const TSKEY* tsCols, int32_t rows) { - TSKEY ts = TSKEY_INITIAL_VAL; - - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - if (tsCols == NULL) { - ts = ascQuery? win->skey : win->ekey; - } else { - int32_t offset = ascQuery? 0:rows-1; - ts = tsCols[offset]; - } - - return ts; -} - -static void setArithParams(SArithmeticSupport* sas, SExprInfo *pExprInfo, SSDataBlock* pSDataBlock) { - sas->numOfCols = (int32_t) pSDataBlock->info.numOfCols; - sas->pExprInfo = pExprInfo; - if (sas->colList != NULL) { - return; - } - sas->colList = calloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo)); - for(int32_t i = 0; i < sas->numOfCols; ++i) { - SColumnInfoData* pColData = taosArrayGet(pSDataBlock->pDataBlock, i); - sas->colList[i] = pColData->info; - } - - sas->data = calloc(sas->numOfCols, POINTER_BYTES); - - // set the input column data - for (int32_t f = 0; f < pSDataBlock->info.numOfCols; ++f) { - SColumnInfoData *pColumnInfoData = taosArrayGet(pSDataBlock->pDataBlock, f); - sas->data[f] = pColumnInfoData->pData; - } -} - -static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); -static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { - for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { - pCtx[i].order = order; - pCtx[i].size = pBlock->info.rows; - pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; - - setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); - } -} - -void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { - if (pCtx[0].functionId == TSDB_FUNC_ARITHM) { - SArithmeticSupport* pSupport = (SArithmeticSupport*) pCtx[0].param[1].pz; - if (pSupport->colList == NULL) { - doSetInputDataBlock(pOperator, pCtx, pBlock, order); - } else { - doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); - } - } else { - if (pBlock->pDataBlock != NULL) { - doSetInputDataBlock(pOperator, pCtx, pBlock, order); - } else { - doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); - } - } -} - -static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { - for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { - pCtx[i].order = order; - pCtx[i].size = pBlock->info.rows; - pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; - - setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); - - if (pCtx[i].functionId == TSDB_FUNC_ARITHM) { - setArithParams((SArithmeticSupport*)pCtx[i].param[1].pz, &pOperator->pExpr[i], pBlock); - } else { - SColIndex* pCol = &pOperator->pExpr[i].base.colInfo; - if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || (pCtx[i].functionId == TSDB_FUNC_BLKINFO) || - (TSDB_COL_IS_TAG(pCol->flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { - SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; - SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); - - // in case of the block distribution query, the inputBytes is not a constant value. - pCtx[i].pInput = p->pData; - assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type); - - if (pCtx[i].functionId < 0) { - SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); - pCtx[i].ptsList = (int64_t*) tsInfo->pData; - - continue; - } - - uint32_t status = aAggs[pCtx[i].functionId].status; - if ((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) { - SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); - // In case of the top/bottom query again the nest query result, which has no timestamp column - // don't set the ptsList attribute. - if (tsInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - pCtx[i].ptsList = (int64_t*) tsInfo->pData; - } else { - pCtx[i].ptsList = NULL; - } - } - } else if (TSDB_COL_IS_UD_COL(pCol->flag) && (pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { - SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; - SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); - - pCtx[i].pInput = p->pData; - assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type); - for(int32_t j = 0; j < pBlock->info.rows; ++j) { - char* dst = p->pData + j * p->info.bytes; - tVariantDump(&pOperator->pExpr[i].base.param[1], dst, p->info.type, true); - } - } - } - } -} - -static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { - pCtx[k].startTs = startTs;// this can be set during create the struct - - int32_t functionId = pCtx[k].functionId; - if (functionId < 0) { - SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; - doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL); - } else { - aAggs[functionId].xFunction(&pCtx[k]); - } - } - } -} - -static void projectApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].startTs = pQueryAttr->window.skey; - - // Always set the asc order for merge stage process - if (pCtx[k].currentStage == MERGE_STAGE) { - pCtx[k].order = TSDB_ORDER_ASC; - } - - pCtx[k].startTs = pQueryAttr->window.skey; - - if (pCtx[k].functionId < 0) { - // load the script and exec - SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; - doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL); - } else { - aAggs[pCtx[k].functionId].xFunction(&pCtx[k]); - } - } -} - -void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, - int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SExprInfo* pExpr = pOperator->pExpr; - - SQLFunctionCtx* pCtx = pInfo->pCtx; - - for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { - int32_t functionId = pCtx[k].functionId; - if (functionId != TSDB_FUNC_TWA && functionId != TSDB_FUNC_INTERP) { - pCtx[k].start.key = INT64_MIN; - continue; - } - - SColIndex * pColIndex = &pExpr[k].base.colInfo; - int16_t index = pColIndex->colIndex; - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, index); - - assert(pColInfo->info.colId == pColIndex->colId && curTs != windowKey); - double v1 = 0, v2 = 0, v = 0; - - if (prevRowIndex == -1) { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]); - } else { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); - } - - GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes); - - if (functionId == TSDB_FUNC_INTERP) { - if (type == RESULT_ROW_START_INTERP) { - pCtx[k].start.key = prevTs; - pCtx[k].start.val = v1; - - pCtx[k].end.key = curTs; - pCtx[k].end.val = v2; - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - if (prevRowIndex == -1) { - pCtx[k].start.ptr = (char *)pRuntimeEnv->prevRow[index]; - } else { - pCtx[k].start.ptr = (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes; - } - - pCtx[k].end.ptr = (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes; - } - } - } else if (functionId == TSDB_FUNC_TWA) { - SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; - SPoint point2 = (SPoint){.key = curTs, .val = &v2}; - SPoint point = (SPoint){.key = windowKey, .val = &v }; - - taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); - - if (type == RESULT_ROW_START_INTERP) { - pCtx[k].start.key = point.key; - pCtx[k].start.val = v; - } else { - pCtx[k].end.key = point.key; - pCtx[k].end.val = v; - } - } - } -} - -static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, int32_t pos, - int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) { - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - - TSKEY curTs = tsCols[pos]; - TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0]; - - // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. - // start exactly from this point, no need to do interpolation - TSKEY key = ascQuery? win->skey:win->ekey; - if (key == curTs) { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - return true; - } - - if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - return true; - } - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))? lastTs:tsCols[pos - step]; - - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, - key, RESULT_ROW_START_INTERP); - return true; -} - -static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, - int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { - SQueryRuntimeEnv *pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t numOfOutput = pOperatorInfo->numOfOutput; - - TSKEY actualEndKey = tsCols[endRowIndex]; - - TSKEY key = QUERY_IS_ASC_QUERY(pQueryAttr)? win->ekey:win->skey; - - // not ended in current data block, do not invoke interpolation - if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQueryAttr)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { - setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); - return false; - } - - // there is actual end point of current time window, no interpolation need - if (key == actualEndKey) { - setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); - return true; - } - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - int32_t nextRowIndex = endRowIndex + step; - assert(nextRowIndex >= 0); - - TSKEY nextKey = tsCols[nextRowIndex]; - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey, - nextRowIndex, key, RESULT_ROW_END_INTERP); - return true; -} - -static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SQLFunctionCtx* pCtx, - SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - if (!pQueryAttr->timeWindowInterpo) { - return; - } - - assert(pBlock != NULL); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - - if (pBlock->pDataBlock == NULL){ - tscError("pBlock->pDataBlock == NULL"); - return; - } - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0); - - TSKEY *tsCols = (TSKEY *)(pColInfo->pData); - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { // it is not interpolated, now start to generated the interpolated value - int32_t startRowIndex = startPos; - bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, pBlock->pDataBlock, - tsCols, win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } - } else { - setNotInterpoWindowKey(pCtx, pQueryAttr->numOfOutput, RESULT_ROW_START_INTERP); - } - - // point interpolation does not require the end key time window interpolation. - if (pQueryAttr->pointInterpQuery) { - return; - } - - // interpolation query does not generate the time window end interpolation - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); - if (!done) { - int32_t endRowIndex = startPos + (forwardStep - 1) * step; - - TSKEY endKey = QUERY_IS_ASC_QUERY(pQueryAttr)? pBlock->info.window.ekey:pBlock->info.window.skey; - bool interp = setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } else { - setNotInterpoWindowKey(pCtx, pQueryAttr->numOfOutput, RESULT_ROW_END_INTERP); - } -} - -static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { - STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; - - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - int32_t numOfOutput = pOperatorInfo->numOfOutput; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - - int32_t prevIndex = pResultRowInfo->curPos; - - TSKEY* tsCols = NULL; - if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); - tsCols = (int64_t*) pColDataInfo->pData; - assert(tsCols[0] == pSDataBlock->info.window.skey && - tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); - } - - int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); - - STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - - SResultRow* pResult = NULL; - int32_t ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, - numOfOutput, pInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - int32_t forwardStep = 0; - TSKEY ekey = reviseWindowEkey(pQueryAttr, &win); - forwardStep = - getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); - - // prev time window not interpolation yet. - int32_t curIndex = pResultRowInfo->curPos; - if (prevIndex != -1 && prevIndex < curIndex && pQueryAttr->timeWindowInterpo) { - for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. - SResultRow* pRes = getResultRow(pResultRowInfo, j); - if (pRes->closed) { - assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); - continue; - } - - STimeWindow w = pRes->win; - ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &w, masterScan, &pResult, - tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - - doTimeWindowInterpolation(pOperatorInfo, pInfo, pSDataBlock->pDataBlock, *(TSKEY*)pRuntimeEnv->prevRow[0], -1, - tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); - - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pInfo->pCtx, pQueryAttr->numOfOutput, RESULT_ROW_START_INTERP); - - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &w, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput); - } - - // restore current time window - ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, - numOfOutput, pInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } - - // window start key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &win, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); - - STimeWindow nextWin = win; - while (1) { - int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(pQueryAttr, &nextWin, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); - if (startPos < 0) { - break; - } - - // null data, failed to allocate more memory buffer - int32_t code = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &nextWin, masterScan, &pResult, tableGroupId, - pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - ekey = reviseWindowEkey(pQueryAttr, &nextWin); - forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); - - // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &nextWin, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &nextWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); - } - - if (pQueryAttr->timeWindowInterpo) { - int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; - saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); - } - - updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); -} - - -static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { - STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; - - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - int32_t numOfOutput = pOperatorInfo->numOfOutput; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - - TSKEY* tsCols = NULL; - if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); - tsCols = (int64_t*) pColDataInfo->pData; - assert(tsCols[0] == pSDataBlock->info.window.skey && - tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); - } - - int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); - - STimeWindow win = getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - - SResultRow* pResult = NULL; - int32_t forwardStep = 0; - int32_t ret = 0; - STimeWindow preWin = win; - - while (1) { - // null data, failed to allocate more memory buffer - ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &win, masterScan, &pResult, - tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - TSKEY ekey = reviseWindowEkey(pQueryAttr, &win); - forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); - - // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); - preWin = win; - - int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(pQueryAttr, &win, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); - if (startPos < 0) { - if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) { - int32_t code = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.tid, &win, masterScan, &pResult, tableGroupId, - pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - startPos = pSDataBlock->info.rows - 1; - - // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); - } - - break; - } - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - - if (pQueryAttr->timeWindowInterpo) { - int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; - saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); - } - - updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); -} - - - -static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - STableQueryInfo* item = pRuntimeEnv->current; - - SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int16_t bytes = pColInfoData->info.bytes; - int16_t type = pColInfoData->info.type; - - if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { - qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_QID(pRuntimeEnv)); - return; - } - - SColumnInfoData* pFirstColData = taosArrayGet(pSDataBlock->pDataBlock, 0); - int64_t* tsList = (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (int64_t*) pFirstColData->pData:NULL; - - STimeWindow w = TSWINDOW_INITIALIZER; - - int32_t num = 0; - for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { - char* val = ((char*)pColInfoData->pData) + bytes * j; - if (isNull(val, type)) { - continue; - } - - // Compare with the previous row of this column, and do not set the output buffer again if they are identical. - if (pInfo->prevData == NULL) { - pInfo->prevData = malloc(bytes); - memcpy(pInfo->prevData, val, bytes); - num++; - continue; - } - - if (IS_VAR_DATA_TYPE(type)) { - int32_t len = varDataLen(val); - if(len == varDataLen(pInfo->prevData) && memcmp(varDataVal(pInfo->prevData), varDataVal(val), len) == 0) { - num++; - continue; - } - } else { - if (memcmp(pInfo->prevData, val, bytes) == 0) { - num++; - continue; - } - } - - if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, bytes); - } - - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, item->groupIndex); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - - doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, j - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); - - num = 1; - memcpy(pInfo->prevData, val, bytes); - } - - if (num > 0) { - char* val = ((char*)pColInfoData->pData) + bytes * (pSDataBlock->info.rows - num); - memcpy(pInfo->prevData, val, bytes); - - if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, bytes); - } - - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, item->groupIndex); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - - doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); - } - - tfree(pInfo->prevData); -} - -static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - STableQueryInfo* item = pRuntimeEnv->current; - - // primary timestamp column - SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, 0); - - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - SOptrBasicInfo* pBInfo = &pInfo->binfo; - - int64_t gap = pOperator->pRuntimeEnv->pQueryAttr->sw.gap; - pInfo->numOfRows = 0; - if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) { - pInfo->reptScan = true; - pInfo->prevTs = INT64_MIN; - } - - TSKEY* tsList = (TSKEY*)pColInfoData->pData; - for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { - if (pInfo->prevTs == INT64_MIN) { - pInfo->curWindow.skey = tsList[j]; - pInfo->curWindow.ekey = tsList[j]; - pInfo->prevTs = tsList[j]; - pInfo->numOfRows = 1; - pInfo->start = j; - } else if (tsList[j] - pInfo->prevTs <= gap && (tsList[j] - pInfo->prevTs) >= 0) { - pInfo->curWindow.ekey = tsList[j]; - pInfo->prevTs = tsList[j]; - pInfo->numOfRows += 1; - if (j == 0 && pInfo->start != 0) { - pInfo->numOfRows = 1; - pInfo->start = 0; - } - } else { // start a new session window - SResultRow* pResult = NULL; - - pInfo->curWindow.ekey = pInfo->curWindow.skey; - int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.tid, &pInfo->curWindow, masterScan, - &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, - pBInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); - - pInfo->curWindow.skey = tsList[j]; - pInfo->curWindow.ekey = tsList[j]; - pInfo->prevTs = tsList[j]; - pInfo->numOfRows = 1; - pInfo->start = j; - } - } - - SResultRow* pResult = NULL; - - pInfo->curWindow.ekey = pInfo->curWindow.skey; - int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.tid, &pInfo->curWindow, masterScan, - &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, - pBInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); -} - -static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { - if (IS_VAR_DATA_TYPE(type)) { - if (pResultRow->key == NULL) { - pResultRow->key = malloc(varDataTLen(pData)); - varDataCopy(pResultRow->key, pData); - } else { - assert(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0); - } - } else { - int64_t v = -1; - GET_TYPED_DATA(v, int64_t, type, pData); - - pResultRow->win.skey = v; - pResultRow->win.ekey = v; - } -} - -static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - - int32_t *rowCellInfoOffset = binfo->rowCellInfoOffset; - SResultRowInfo *pResultRowInfo = &binfo->resultRowInfo; - SQLFunctionCtx *pCtx = binfo->pCtx; - - // not assign result buffer yet, add new result buffer, TODO remove it - char* d = pData; - int16_t len = bytes; - if (IS_VAR_DATA_TYPE(type)) { - d = varDataVal(pData); - len = varDataLen(pData); - } - - int64_t tid = 0; - SResultRow *pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, d, len, true, groupIndex); - assert (pResultRow != NULL); - - setResultRowKey(pResultRow, pData, type); - if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->pQueryAttr->resultRowSize); - if (ret != 0) { - return -1; - } - } - - setResultOutputBuf(pRuntimeEnv, pResultRow, pCtx, numOfCols, rowCellInfoOffset); - initCtxOutputBuffer(pCtx, numOfCols); - return TSDB_CODE_SUCCESS; -} - -static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock) { - for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { - SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); - if (TSDB_COL_IS_TAG(pColIndex->flag)) { - continue; - } - - int32_t colId = pColIndex->colId; - - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - if (pColInfo->info.colId == colId) { - return i; - } - } - } - - assert(0); - return -1; -} - -static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - // in case of timestamp column, always generated results. - int32_t functionId = pCtx->functionId; - if (functionId == TSDB_FUNC_TS) { - return true; - } - - if (pResInfo->complete || functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { - return false; - } - - if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST) { - return QUERY_IS_ASC_QUERY(pQueryAttr); - } - - // denote the order type - if ((functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_LAST)) { - return pCtx->param[0].i64 == pQueryAttr->order.order; - } - - // in the reverse table scan, only the following functions need to be executed - if (IS_REVERSE_SCAN(pRuntimeEnv) || - (pRuntimeEnv->scanFlag == REPEAT_SCAN && functionId != TSDB_FUNC_STDDEV && functionId != TSDB_FUNC_PERCT)) { - return false; - } - - return true; -} - -void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex) { - SDataStatis *pStatis = NULL; - - if (pSDataBlock->pBlockStatis != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { - pStatis = &pSDataBlock->pBlockStatis[pColIndex->colIndex]; - - pCtx->preAggVals.statis = *pStatis; - pCtx->preAggVals.isSet = true; - assert(pCtx->preAggVals.statis.numOfNull <= pSDataBlock->info.rows); - } else { - pCtx->preAggVals.isSet = false; - } - - pCtx->hasNull = hasNull(pColIndex, pStatis); - - // set the statistics data for primary time stamp column - if (pCtx->functionId == TSDB_FUNC_SPREAD && pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - pCtx->preAggVals.isSet = true; - pCtx->preAggVals.statis.min = pSDataBlock->info.window.skey; - pCtx->preAggVals.statis.max = pSDataBlock->info.window.ekey; - } -} - -// set the output buffer for the selectivity + tag query -static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { - if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) { - return TSDB_CODE_SUCCESS; - } - - int32_t num = 0; - int16_t tagLen = 0; - - SQLFunctionCtx* p = NULL; - SQLFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); - if (pTagCtx == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < numOfOutput; ++i) { - int32_t functionId = pCtx[i].functionId; - - if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { - tagLen += pCtx[i].outputBytes; - pTagCtx[num++] = &pCtx[i]; - } else if ((aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - p = &pCtx[i]; - } else if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG) { - // tag function may be the group by tag column - // ts may be the required primary timestamp column - continue; - } else { - // the column may be the normal column, group by normal_column, the functionId is TSDB_FUNC_PRJ - } - } - if (p != NULL) { - p->tagInfo.pTagCtxList = pTagCtx; - p->tagInfo.numOfTagCols = num; - p->tagInfo.tagsLen = tagLen; - } else { - tfree(pTagCtx); - } - - return TSDB_CODE_SUCCESS; -} - -static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, - int32_t** rowCellInfoOffset) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); - if (pFuncCtx == NULL) { - return NULL; - } - - *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t)); - if (*rowCellInfoOffset == 0) { - tfree(pFuncCtx); - return NULL; - } - - for (int32_t i = 0; i < numOfOutput; ++i) { - SSqlExpr *pSqlExpr = &pExpr[i].base; - SQLFunctionCtx* pCtx = &pFuncCtx[i]; - - SColIndex *pIndex = &pSqlExpr->colInfo; - - if (TSDB_COL_REQ_NULL(pIndex->flag)) { - pCtx->requireNull = true; - pIndex->flag &= ~(TSDB_COL_NULL); - } else { - pCtx->requireNull = false; - } - - pCtx->inputBytes = pSqlExpr->colBytes; - pCtx->inputType = pSqlExpr->colType; - - pCtx->ptsOutputBuf = NULL; - - pCtx->outputBytes = pSqlExpr->resBytes; - pCtx->outputType = pSqlExpr->resType; - - pCtx->order = pQueryAttr->order.order; - pCtx->functionId = pSqlExpr->functionId; - pCtx->stableQuery = pQueryAttr->stableQuery; - pCtx->interBufBytes = pSqlExpr->interBytes; - pCtx->start.key = INT64_MIN; - pCtx->end.key = INT64_MIN; - - pCtx->numOfParams = pSqlExpr->numOfParams; - for (int32_t j = 0; j < pCtx->numOfParams; ++j) { - int16_t type = pSqlExpr->param[j].nType; - int16_t bytes = pSqlExpr->param[j].nLen; - if (pSqlExpr->functionId == TSDB_FUNC_STDDEV_DST) { - continue; - } - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx->param[j], pSqlExpr->param[j].pz, bytes, type); - } else { - tVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlExpr->param[j].i64, bytes, type); - } - } - - // set the order information for top/bottom query - int32_t functionId = pCtx->functionId; - - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - int32_t f = pExpr[0].base.functionId; - assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY); - - pCtx->param[2].i64 = pQueryAttr->order.order; - pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[3].i64 = functionId; - pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; - - pCtx->param[1].i64 = pQueryAttr->order.orderColId; - } else if (functionId == TSDB_FUNC_INTERP) { - pCtx->param[2].i64 = (int8_t)pQueryAttr->fillType; - if (pQueryAttr->fillVal != NULL) { - if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) { - pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; - } else { // todo refactor, tVariantCreateFromBinary should handle the NULL value - if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType); - } - } - } - } else if (functionId == TSDB_FUNC_TS_COMP) { - pCtx->param[0].i64 = pQueryAttr->vgId; //TODO this should be the parameter from client - pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; - } else if (functionId == TSDB_FUNC_TWA) { - pCtx->param[1].i64 = pQueryAttr->window.skey; - pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[2].i64 = pQueryAttr->window.ekey; - pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - } else if (functionId == TSDB_FUNC_ARITHM) { - pCtx->param[1].pz = (char*) &pRuntimeEnv->sasArray[i]; - } - } - - for(int32_t i = 1; i < numOfOutput; ++i) { - (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowCellInfo) + pExpr[i - 1].base.interBytes); - } - - setCtxTagColumnInfo(pFuncCtx, numOfOutput); - - return pFuncCtx; -} - -static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { - if (pCtx == NULL) { - return NULL; - } - - for (int32_t i = 0; i < numOfOutput; ++i) { - for (int32_t j = 0; j < pCtx[i].numOfParams; ++j) { - tVariantDestroy(&pCtx[i].param[j]); - } - - tVariantDestroy(&pCtx[i].tag); - tfree(pCtx[i].tagInfo.pTagCtxList); - } - - tfree(pCtx); - return NULL; -} - -static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) { - qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_QID(pRuntimeEnv)); - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - pRuntimeEnv->prevGroupId = INT32_MIN; - pRuntimeEnv->pQueryAttr = pQueryAttr; - - pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - pRuntimeEnv->pResultRowListSet = taosHashInit(numOfTables * 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - pRuntimeEnv->keyBuf = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); - pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); - pRuntimeEnv->pResultRowArrayList = taosArrayInit(numOfTables, sizeof(SResultRowCell)); - - pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); - pRuntimeEnv->tagVal = malloc(pQueryAttr->tagLen); - - // NOTE: pTableCheckInfo need to update the query time range and the lastKey info - pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - - pRuntimeEnv->sasArray = calloc(pQueryAttr->numOfOutput, sizeof(SArithmeticSupport)); - - if (pRuntimeEnv->sasArray == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL || - pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) { - goto _clean; - } - - if (pQueryAttr->numOfCols) { - char* start = POINTER_BYTES * pQueryAttr->numOfCols + (char*) pRuntimeEnv->prevRow; - pRuntimeEnv->prevRow[0] = start; - for(int32_t i = 1; i < pQueryAttr->numOfCols; ++i) { - pRuntimeEnv->prevRow[i] = pRuntimeEnv->prevRow[i - 1] + pQueryAttr->tableCols[i-1].bytes; - } - - if (pQueryAttr->tableCols[0].type == TSDB_DATA_TYPE_TIMESTAMP) { - *(int64_t*) pRuntimeEnv->prevRow[0] = INT64_MIN; - } - } - - qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_QID(pRuntimeEnv)); - - // group by normal column, sliding window query, interval query are handled by interval query processor - // interval (down sampling operation) - int32_t numOfOperator = (int32_t) taosArrayGetSize(pOperator); - for(int32_t i = 0; i < numOfOperator; ++i) { - int32_t* op = taosArrayGet(pOperator, i); - - switch (*op) { - case OP_TagScan: { - pRuntimeEnv->proot = createTagScanOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - break; - } - case OP_MultiTableTimeInterval: { - pRuntimeEnv->proot = - createMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - break; - } - case OP_AllMultiTableTimeInterval: { - pRuntimeEnv->proot = - createAllMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - break; - } - case OP_TimeWindow: { - pRuntimeEnv->proot = - createIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput && opType != OP_Join) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - case OP_AllTimeWindow: { - pRuntimeEnv->proot = - createAllTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput && opType != OP_Join) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - case OP_Groupby: { - pRuntimeEnv->proot = - createGroupbyOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - case OP_SessionWindow: { - pRuntimeEnv->proot = - createSWindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - case OP_MultiTableAggregate: { - pRuntimeEnv->proot = - createMultiTableAggOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - break; - } - case OP_Aggregate: { - pRuntimeEnv->proot = - createAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput && opType != OP_Join) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - - case OP_Project: { // TODO refactor to remove arith operator. - SOperatorInfo* prev = pRuntimeEnv->proot; - if (i == 0) { - pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - if (pRuntimeEnv->proot != NULL && prev->operatorType != OP_DummyInput && prev->operatorType != OP_Join) { // TODO refactor - setTableScanFilterOperatorInfo(prev->info, pRuntimeEnv->proot); - } - } else { - prev = pRuntimeEnv->proot; - assert(pQueryAttr->pExpr2 != NULL); - pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); - } - break; - } - - case OP_StateWindow: { - pRuntimeEnv->proot = createStatewindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - - case OP_Limit: { - pRuntimeEnv->proot = createLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot); - break; - } - - case OP_Filter: { // todo refactor - int32_t numOfFilterCols = 0; - if (pQueryAttr->stableQuery) { - SColumnInfo* pColInfo = - extractColumnFilterInfo(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3, &numOfFilterCols); - pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, - pQueryAttr->numOfExpr3, pColInfo, numOfFilterCols); - freeColumnInfo(pColInfo, pQueryAttr->numOfExpr3); - } else { - SColumnInfo* pColInfo = - extractColumnFilterInfo(pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &numOfFilterCols); - pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, - pQueryAttr->numOfOutput, pColInfo, numOfFilterCols); - freeColumnInfo(pColInfo, pQueryAttr->numOfOutput); - } - - break; - } - - case OP_Fill: { - SOperatorInfo* pInfo = pRuntimeEnv->proot; - pRuntimeEnv->proot = createFillOperatorInfo(pRuntimeEnv, pInfo, pInfo->pExpr, pInfo->numOfOutput, pQueryAttr->multigroupResult); - break; - } - - case OP_MultiwayMergeSort: { - pRuntimeEnv->proot = createMultiwaySortOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, 4096, merger); - break; - } - - case OP_GlobalAggregate: { // If fill operator exists, the result rows of different group can not be in the same SSDataBlock. - bool multigroupResult = pQueryAttr->multigroupResult; - if (pQueryAttr->multigroupResult) { - multigroupResult = (pQueryAttr->fillType == TSDB_FILL_NONE); - } - - pRuntimeEnv->proot = createGlobalAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, - pQueryAttr->numOfExpr3, merger, pQueryAttr->pUdfInfo, multigroupResult); - break; - } - - case OP_SLimit: { - int32_t num = pRuntimeEnv->proot->numOfOutput; - SExprInfo* pExpr = pRuntimeEnv->proot->pExpr; - pRuntimeEnv->proot = createSLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pExpr, num, merger, pQueryAttr->multigroupResult); - break; - } - - case OP_Distinct: { - pRuntimeEnv->proot = createDistinctOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - break; - } - - case OP_Order: { - pRuntimeEnv->proot = createSortOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &pQueryAttr->order); - break; - } - - default: { - assert(0); - } - } - } - - return TSDB_CODE_SUCCESS; - -_clean: - tfree(pRuntimeEnv->sasArray); - tfree(pRuntimeEnv->pResultRowHashTable); - tfree(pRuntimeEnv->keyBuf); - tfree(pRuntimeEnv->prevRow); - tfree(pRuntimeEnv->tagVal); - - return TSDB_CODE_QRY_OUT_OF_MEMORY; -} - -static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - tsdbCleanupQueryHandle(pRuntimeEnv->pTsdbReadHandle); - pRuntimeEnv->pTsdbReadHandle = NULL; - - SMemRef* pMemRef = &pQueryAttr->memRef; - assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); -} - -static void destroyTsComp(SQueryRuntimeEnv *pRuntimeEnv, SQueryAttr *pQueryAttr) { - if (pQueryAttr->tsCompQuery && pRuntimeEnv->outputBuf && pRuntimeEnv->outputBuf->pDataBlock && taosArrayGetSize(pRuntimeEnv->outputBuf->pDataBlock) > 0) { - SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); - if (pColInfoData) { - FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor - if (f) { - fclose(f); - *(FILE **)pColInfoData->pData = NULL; - } - } - } -} - -static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - SQInfo* pQInfo = (SQInfo*) pRuntimeEnv->qinfo; - - qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); - - if (pRuntimeEnv->sasArray != NULL) { - for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - tfree(pRuntimeEnv->sasArray[i].data); - tfree(pRuntimeEnv->sasArray[i].colList); - } - - tfree(pRuntimeEnv->sasArray); - } - - destroyUdfInfo(pRuntimeEnv->pUdfInfo); - - destroyResultBuf(pRuntimeEnv->pResultBuf); - doFreeQueryHandle(pRuntimeEnv); - - destroyTsComp(pRuntimeEnv, pQueryAttr); - - pRuntimeEnv->pTsBuf = tsBufDestroy(pRuntimeEnv->pTsBuf); - - tfree(pRuntimeEnv->keyBuf); - tfree(pRuntimeEnv->prevRow); - tfree(pRuntimeEnv->tagVal); - - taosHashCleanup(pRuntimeEnv->pResultRowHashTable); - pRuntimeEnv->pResultRowHashTable = NULL; - - taosHashCleanup(pRuntimeEnv->pTableRetrieveTsMap); - pRuntimeEnv->pTableRetrieveTsMap = NULL; - - taosHashCleanup(pRuntimeEnv->pResultRowListSet); - pRuntimeEnv->pResultRowListSet = NULL; - - destroyOperatorInfo(pRuntimeEnv->proot); - - pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); - taosArrayDestroyEx(pRuntimeEnv->prevResult, freeInterResult); - taosArrayDestroy(pRuntimeEnv->pResultRowArrayList); - pRuntimeEnv->prevResult = NULL; -} - -static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { - return pQInfo->rspContext != NULL; -} - -bool isQueryKilled(SQInfo *pQInfo) { - if (IS_QUERY_KILLED(pQInfo)) { - return true; - } - - // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived - // abort current query execution. - if (pQInfo->owner != 0 && ((taosGetTimestampSec() - pQInfo->startExecTs/1000) > getMaximumIdleDurationSec()) && - (!needBuildResAfterQueryComplete(pQInfo))) { - - assert(pQInfo->startExecTs != 0); - qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 - ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec()); - return true; - } - - return false; -} - -void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;} - -//static bool isFixedOutputQuery(SQueryAttr* pQueryAttr) { -// if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { -// return false; -// } -// -// // Note:top/bottom query is fixed output query -// if (pQueryAttr->topBotQuery || pQueryAttr->groupbyColumn || pQueryAttr->tsCompQuery) { -// return true; -// } -// -// for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { -// SSqlExpr *pExpr = &pQueryAttr->pExpr1[i].base; -// -// if (pExpr->functionId == TSDB_FUNC_TS || pExpr->functionId == TSDB_FUNC_TS_DUMMY) { -// continue; -// } -// -// if (!IS_MULTIOUTPUT(aAggs[pExpr->functionId].status)) { -// return true; -// } -// } -// -// return false; -//} - -// todo refactor with isLastRowQuery -//bool isPointInterpoQuery(SQueryAttr *pQueryAttr) { -// for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { -// int32_t functionId = pQueryAttr->pExpr1[i].base.functionId; -// if (functionId == TSDB_FUNC_INTERP) { -// return true; -// } -// } -// -// return false; -//} - -static bool isFirstLastRowQuery(SQueryAttr *pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionID = pQueryAttr->pExpr1[i].base.functionId; - if (functionID == TSDB_FUNC_LAST_ROW) { - return true; - } - } - - return false; -} - -static bool isCachedLastQuery(SQueryAttr *pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionID = pQueryAttr->pExpr1[i].base.functionId; - if (functionID == TSDB_FUNC_LAST || functionID == TSDB_FUNC_LAST_DST) { - continue; - } - - return false; - } - - if (pQueryAttr->order.order != TSDB_ORDER_DESC || !TSWINDOW_IS_EQUAL(pQueryAttr->window, TSWINDOW_DESC_INITIALIZER)) { - return false; - } - - if (pQueryAttr->groupbyColumn) { - return false; - } - - if (pQueryAttr->interval.interval > 0) { - return false; - } - - if (pQueryAttr->numOfFilterCols > 0 || pQueryAttr->havingNum > 0) { - return false; - } - - return true; -} - - - -/** - * The following 4 kinds of query are treated as the tags query - * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query - */ -bool onlyQueryTags(SQueryAttr* pQueryAttr) { - for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - SExprInfo* pExprInfo = &pQueryAttr->pExpr1[i]; - - int32_t functionId = pExprInfo->base.functionId; - - if (functionId != TSDB_FUNC_TAGPRJ && - functionId != TSDB_FUNC_TID_TAG && - (!(functionId == TSDB_FUNC_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) && - (!(functionId == TSDB_FUNC_PRJ && TSDB_COL_IS_UD_COL(pExprInfo->base.colInfo.flag)))) { - return false; - } - } - - return true; -} - -///////////////////////////////////////////////////////////////////////////////////////////// - -void getAlignQueryTimeWindow(SQueryAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { - assert(key >= keyFirst && key <= keyLast && pQueryAttr->interval.sliding <= pQueryAttr->interval.interval); - win->skey = taosTimeTruncate(key, &pQueryAttr->interval, pQueryAttr->precision); - - /* - * if the realSkey > INT64_MAX - pQueryAttr->interval.interval, the query duration between - * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. - */ - if (keyFirst > (INT64_MAX - pQueryAttr->interval.interval)) { - assert(keyLast - keyFirst < pQueryAttr->interval.interval); - win->ekey = INT64_MAX; - } else if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - win->ekey = taosTimeAdd(win->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; - } else { - win->ekey = win->skey + pQueryAttr->interval.interval - 1; - } -} - -/* - * todo add more parameters to check soon.. - */ -bool colIdCheck(SQueryAttr *pQueryAttr, uint64_t qId) { - // load data column information is incorrect - for (int32_t i = 0; i < pQueryAttr->numOfCols - 1; ++i) { - if (pQueryAttr->tableCols[i].colId == pQueryAttr->tableCols[i + 1].colId) { - qError("QInfo:0x%"PRIx64" invalid data load column for query", qId); - return false; - } - } - - return true; -} - -// todo ignore the avg/sum/min/max/count/stddev/top/bottom functions, of which -// the scan order is not matter -static bool onlyOneQueryType(SQueryAttr *pQueryAttr, int32_t functId, int32_t functIdDst) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = pQueryAttr->pExpr1[i].base.functionId; - - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG || - functionId == TSDB_FUNC_TAG_DUMMY) { - continue; - } - - if (functionId != functId && functionId != functIdDst) { - return false; - } - } - - return true; -} - -static bool onlyFirstQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, TSDB_FUNC_FIRST, TSDB_FUNC_FIRST_DST); } - -static bool onlyLastQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST); } - -static bool notContainSessionOrStateWindow(SQueryAttr *pQueryAttr) { return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); } - -static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) { - bool hasFirstLastFunc = false; - bool hasOtherFunc = false; - - if (status == BLK_DATA_ALL_NEEDED || status == BLK_DATA_DISCARD) { - return status; - } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG || - functionId == TSDB_FUNC_TAG_DUMMY) { - continue; - } - - if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { - hasFirstLastFunc = true; - } else { - hasOtherFunc = true; - } - } - - if (hasFirstLastFunc && status == BLK_DATA_NO_NEEDED) { - if(!hasOtherFunc) { - return BLK_DATA_DISCARD; - } else { - return BLK_DATA_ALL_NEEDED; - } - } - - return status; -} - -static void doUpdateLastKey(SQueryAttr* pQueryAttr) { - STimeWindow* win = &pQueryAttr->window; - - size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList); - for(int32_t i = 0; i < num; ++i) { - SArray* p1 = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); - - size_t len = taosArrayGetSize(p1); - for(int32_t j = 0; j < len; ++j) { - STableKeyInfo* pInfo = taosArrayGet(p1, j); - - // update the new lastkey if it is equalled to the value of the old skey - if (pInfo->lastKey == win->ekey) { - pInfo->lastKey = win->skey; - } - } - } -} - -static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool stableQuery) { - SQueryAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - - // in case of point-interpolation query, use asc order scan - char msg[] = "QInfo:0x%"PRIx64" scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%" PRId64 - "-%" PRId64 ", new qrange:%" PRId64 "-%" PRId64; - - // todo handle the case the the order irrelevant query type mixed up with order critical query type - // descending order query for last_row query - if (isFirstLastRowQuery(pQueryAttr)) { - qDebug("QInfo:0x%"PRIx64" scan order changed for last_row query, old:%d, new:%d", pQInfo->qId, pQueryAttr->order.order, TSDB_ORDER_ASC); - - pQueryAttr->order.order = TSDB_ORDER_ASC; - if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - } - - pQueryAttr->needReverseScan = false; - return; - } - - if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) { - pQueryAttr->order.order = TSDB_ORDER_ASC; - if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - } - - pQueryAttr->needReverseScan = false; - doUpdateLastKey(pQueryAttr); - return; - } - - if (pQueryAttr->pointInterpQuery && pQueryAttr->interval.interval == 0) { - if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { - qDebug(msg, pQInfo->qId, "interp", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - } - - pQueryAttr->order.order = TSDB_ORDER_ASC; - return; - } - - if (pQueryAttr->interval.interval == 0) { - if (onlyFirstQuery(pQueryAttr)) { - if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { - qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, - pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - doUpdateLastKey(pQueryAttr); - } - - pQueryAttr->order.order = TSDB_ORDER_ASC; - pQueryAttr->needReverseScan = false; - } else if (onlyLastQuery(pQueryAttr) && notContainSessionOrStateWindow(pQueryAttr)) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, - pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - doUpdateLastKey(pQueryAttr); - } - - pQueryAttr->order.order = TSDB_ORDER_DESC; - pQueryAttr->needReverseScan = false; - } - - } else { // interval query - if (stableQuery) { - if (onlyFirstQuery(pQueryAttr)) { - if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { - qDebug(msg, pQInfo->qId, "only-first stable", pQueryAttr->order.order, TSDB_ORDER_ASC, - pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - doUpdateLastKey(pQueryAttr); - } - - pQueryAttr->order.order = TSDB_ORDER_ASC; - pQueryAttr->needReverseScan = false; - } else if (onlyLastQuery(pQueryAttr)) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - qDebug(msg, pQInfo->qId, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC, - pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - doUpdateLastKey(pQueryAttr); - } - - pQueryAttr->order.order = TSDB_ORDER_DESC; - pQueryAttr->needReverseScan = false; - } - } - } -} - -static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t MIN_ROWS_PER_PAGE = 4; - - *rowsize = (int32_t)(pQueryAttr->resultRowSize * getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); - int32_t overhead = sizeof(tFilePage); - - // one page contains at least two rows - *ps = DEFAULT_INTERN_BUF_PAGE_SIZE; - while(((*rowsize) * MIN_ROWS_PER_PAGE) > (*ps) - overhead) { - *ps = ((*ps) << 1u); - } -} - -#define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) - -static FORCE_INLINE bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) { - return true; - } - - return filterRangeExecute(pQueryAttr->pFilters, pDataStatis, pQueryAttr->numOfCols, numOfRows); -} - -static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { - STimeWindow w = {0}; - - TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); - - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); - assert(w.ekey >= pBlockInfo->window.skey); - - if (w.ekey < pBlockInfo->window.ekey) { - return true; - } - - while(1) { - getNextTimeWindow(pQueryAttr, &w); - if (w.skey > pBlockInfo->window.ekey) { - break; - } - - assert(w.ekey > pBlockInfo->window.ekey); - if (w.skey <= pBlockInfo->window.ekey && w.skey > pBlockInfo->window.skey) { - return true; - } - } - } else { - getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); - assert(w.skey <= pBlockInfo->window.ekey); - - if (w.skey > pBlockInfo->window.skey) { - return true; - } - - while(1) { - getNextTimeWindow(pQueryAttr, &w); - if (w.ekey < pBlockInfo->window.skey) { - break; - } - - assert(w.skey < pBlockInfo->window.skey); - if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) { - return true; - } - } - } - - return false; -} - -static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key, bool ascQuery) { - STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); - -#if defined(_DEBUG_VIEW) - printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", - elem.ts, key, elem.tag.i64, pQueryAttr->order.order, pRuntimeEnv->pTsBuf->tsOrder, - pRuntimeEnv->pTsBuf->cur.order, pRuntimeEnv->pTsBuf->cur.tsIndex); -#endif - - if (ascQuery) { - if (key < elem.ts) { - return TS_JOIN_TS_NOT_EQUALS; - } else if (key > elem.ts) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); - } - } else { - if (key > elem.ts) { - return TS_JOIN_TS_NOT_EQUALS; - } else if (key < elem.ts) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); - } - } - - return TS_JOIN_TS_EQUAL; -} - -bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p) { - bool all = true; - - for (int32_t i = 0; i < numOfRows; ++i) { - bool qualified = false; - - for (int32_t k = 0; k < numOfFilterCols; ++k) { - char* pElem = (char*)pFilterInfo[k].pData + pFilterInfo[k].info.bytes * i; - - qualified = false; - for (int32_t j = 0; j < pFilterInfo[k].numOfFilters; ++j) { - SColumnFilterElem* pFilterElem = &pFilterInfo[k].pFilters[j]; - - bool isnull = isNull(pElem, pFilterInfo[k].info.type); - if (isnull) { - if (pFilterElem->fp == isNullOperator) { - qualified = true; - break; - } else { - continue; - } - } else { - if (pFilterElem->fp == notNullOperator) { - qualified = true; - break; - } else if (pFilterElem->fp == isNullOperator) { - continue; - } - } - - if (pFilterElem->fp(pFilterElem, pElem, pElem, pFilterInfo[k].info.type)) { - qualified = true; - break; - } - } - - if (!qualified) { - break; - } - } - - p[i] = qualified ? 1 : 0; - if (!qualified) { - all = false; - } - } - - return all; -} - -void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p) { - int32_t len = 0; - int32_t start = 0; - for (int32_t j = 0; j < numOfRows; ++j) { - if (p[j] == 1) { - len++; - } else { - if (len > 0) { - int32_t cstart = j - len; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); - - int16_t bytes = pColumnInfoData->info.bytes; - memmove(((char*)pColumnInfoData->pData) + start * bytes, pColumnInfoData->pData + cstart * bytes, - len * bytes); - } - - start += len; - len = 0; - } - } - } - - if (len > 0) { - int32_t cstart = numOfRows - len; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); - - int16_t bytes = pColumnInfoData->info.bytes; - memmove(pColumnInfoData->pData + start * bytes, pColumnInfoData->pData + cstart * bytes, len * bytes); - } - - start += len; - len = 0; - } - - pBlock->info.rows = start; - pBlock->pBlockStatis = NULL; // clean the block statistics info - - if (start > 0) { - SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, 0); - if (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && - pColumnInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - pBlock->info.window.skey = *(int64_t*)pColumnInfoData->pData; - pBlock->info.window.ekey = *(int64_t*)(pColumnInfoData->pData + TSDB_KEYSIZE * (start - 1)); - } - } -} - -void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, - SSDataBlock* pBlock, bool ascQuery) { - int32_t numOfRows = pBlock->info.rows; - - int8_t *p = calloc(numOfRows, sizeof(int8_t)); - bool all = true; - - if (pRuntimeEnv->pTsBuf != NULL) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - - TSKEY* k = (TSKEY*) pColInfoData->pData; - for (int32_t i = 0; i < numOfRows; ++i) { - int32_t offset = ascQuery? i:(numOfRows - i - 1); - int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery); - if (ret == TS_JOIN_TAG_NOT_EQUALS) { - break; - } else if (ret == TS_JOIN_TS_NOT_EQUALS) { - all = false; - continue; - } else { - assert(ret == TS_JOIN_TS_EQUAL); - p[offset] = true; - } - - if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { - break; - } - } - - // save the cursor status - pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - } else { - all = doFilterDataBlock(pFilterInfo, numOfFilterCols, numOfRows, p); - } - - if (!all) { - doCompactSDataBlock(pBlock, numOfRows, p); - } - - tfree(p); -} - -void filterColRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) { - int32_t numOfRows = pBlock->info.rows; - - int8_t *p = NULL; - bool all = true; - - if (pRuntimeEnv->pTsBuf != NULL) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - p = calloc(numOfRows, sizeof(int8_t)); - - TSKEY* k = (TSKEY*) pColInfoData->pData; - for (int32_t i = 0; i < numOfRows; ++i) { - int32_t offset = ascQuery? i:(numOfRows - i - 1); - int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery); - if (ret == TS_JOIN_TAG_NOT_EQUALS) { - break; - } else if (ret == TS_JOIN_TS_NOT_EQUALS) { - all = false; - continue; - } else { - assert(ret == TS_JOIN_TS_EQUAL); - p[offset] = true; - } - - if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { - break; - } - } - - // save the cursor status - pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - } else { - all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, &p, pBlock->pBlockStatis, pRuntimeEnv->pQueryAttr->numOfCols); - } - - if (!all) { - if (p) { - doCompactSDataBlock(pBlock, numOfRows, p); - } else { - pBlock->info.rows = 0; - pBlock->pBlockStatis = NULL; // clean the block statistics info - } - } - - tfree(p); -} - - - -static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); -static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes); - -static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { - SQLFunctionCtx* pCtx = pTableScanInfo->pCtx; - uint32_t status = BLK_DATA_NO_NEEDED; - - int32_t numOfOutput = pTableScanInfo->numOfOutput; - for (int32_t i = 0; i < numOfOutput; ++i) { - int32_t functionId = pCtx[i].functionId; - int32_t colId = pTableScanInfo->pExpr[i].base.colInfo.colId; - - // group by + first/last should not apply the first/last block filter - if (functionId < 0) { - status |= BLK_DATA_ALL_NEEDED; - return status; - } else { - status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId); - if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) { - return status; - } - } - } - - return status; -} - -void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock) { - // set the initial static data value filter expression - for (int32_t i = 0; i < numOfFilterCols; ++i) { - for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, j); - - if (pFilterInfo[i].info.colId == pColInfo->info.colId) { - pFilterInfo[i].pData = pColInfo->pData; - break; - } - } - } -} - -int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, - uint32_t* status) { - *status = BLK_DATA_NO_NEEDED; - pBlock->pDataBlock = NULL; - pBlock->pBlockStatis = NULL; - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int64_t groupId = pRuntimeEnv->current->groupIndex; - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - - SQInfo* pQInfo = pRuntimeEnv->qinfo; - SQueryCostInfo* pCost = &pQInfo->summary; - - pCost->totalBlocks += 1; - pCost->totalRows += pBlock->info.rows; - - if (pRuntimeEnv->pTsBuf != NULL) { - (*status) = BLK_DATA_ALL_NEEDED; - - if (pQueryAttr->stableQuery) { // todo refactor - SExprInfo* pExprInfo = &pTableScanInfo->pExpr[0]; - int16_t tagId = (int16_t)pExprInfo->base.param[0].i64; - SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagId); - - // compare tag first - tVariant t = {0}; - doSetTagValueInParam(pRuntimeEnv->current->pTable, tagId, &t, pColInfo->type, pColInfo->bytes); - setTimestampListJoinInfo(pRuntimeEnv, &t, pRuntimeEnv->current); - - STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); - if (!tsBufIsValidElem(&elem) || (tsBufIsValidElem(&elem) && (tVariantCompare(&t, elem.tag) != 0))) { - (*status) = BLK_DATA_DISCARD; - return TSDB_CODE_SUCCESS; - } - } - } - - // Calculate all time windows that are overlapping or contain current data block. - // If current data block is contained by all possible time window, do not load current data block. - if (pQueryAttr->pFilters || pQueryAttr->groupbyColumn || pQueryAttr->sw.gap > 0 || - (QUERY_IS_INTERVAL_QUERY(pQueryAttr) && overlapWithTimeWindow(pQueryAttr, &pBlock->info))) { - (*status) = BLK_DATA_ALL_NEEDED; - } - - // check if this data block is required to load - if ((*status) != BLK_DATA_ALL_NEEDED) { - bool needFilter = true; - - // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, - // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer - if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { - SResultRow* pResult = NULL; - - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; - - STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); - if (pQueryAttr->pointInterpQuery) { - needFilter = chkWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, - pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, - pTableScanInfo->rowCellInfoOffset); - } else { - if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.tid, &win, masterScan, &pResult, groupId, - pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, - pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } - } else if (pQueryAttr->stableQuery && (!pQueryAttr->tsCompQuery) && (!pQueryAttr->diffQuery)) { // stable aggregate, not interval aggregate or normal column aggregate - doSetTableGroupOutputBuf(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, - pTableScanInfo->rowCellInfoOffset, pTableScanInfo->numOfOutput, - pRuntimeEnv->current->groupIndex); - } - - if (needFilter) { - (*status) = doFilterByBlockTimeWindow(pTableScanInfo, pBlock); - } else { - (*status) = BLK_DATA_ALL_NEEDED; - } - } - - SDataBlockInfo* pBlockInfo = &pBlock->info; - *status = updateBlockLoadStatus(pRuntimeEnv->pQueryAttr, *status); - - if ((*status) == BLK_DATA_NO_NEEDED || (*status) == BLK_DATA_DISCARD) { - qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, - pBlockInfo->window.ekey, pBlockInfo->rows); - pCost->skipBlocks += 1; - } else if ((*status) == BLK_DATA_STATIS_NEEDED) { - // this function never returns error? - pCost->loadBlockStatis += 1; - tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockStatis); - - if (pBlock->pBlockStatis == NULL) { // data block statistics does not exist, load data block - pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); - pCost->totalCheckedRows += pBlock->info.rows; - } - } else { - assert((*status) == BLK_DATA_ALL_NEEDED); - - // load the data block statistics to perform further filter - pCost->loadBlockStatis += 1; - tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockStatis); - - if (pQueryAttr->topBotQuery && pBlock->pBlockStatis != NULL) { - { // set previous window - if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { - SResultRow* pResult = NULL; - - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; - - STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); - if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.tid, &win, masterScan, &pResult, groupId, - pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, - pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } - } - bool load = false; - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = pTableScanInfo->pCtx[i].functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - load = topbot_datablock_filter(&pTableScanInfo->pCtx[i], (char*)&(pBlock->pBlockStatis[i].min), - (char*)&(pBlock->pBlockStatis[i].max)); - if (!load) { // current block has been discard due to filter applied - pCost->skipBlocks += 1; - qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - (*status) = BLK_DATA_DISCARD; - return TSDB_CODE_SUCCESS; - } - } - } - } - - // current block has been discard due to filter applied - if (!doFilterByBlockStatistics(pRuntimeEnv, pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) { - pCost->skipBlocks += 1; - qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, - pBlockInfo->window.ekey, pBlockInfo->rows); - (*status) = BLK_DATA_DISCARD; - return TSDB_CODE_SUCCESS; - } - - pCost->totalCheckedRows += pBlockInfo->rows; - pCost->loadBlocks += 1; - pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); - if (pBlock->pDataBlock == NULL) { - return terrno; - } - - if (pQueryAttr->pFilters != NULL) { - filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock); - } - - if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { - filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery); - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { - int32_t midPos = -1; - int32_t numOfRows; - - if (num <= 0) { - return -1; - } - - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - - TSKEY * keyList = (TSKEY *)pValue; - int32_t firstPos = 0; - int32_t lastPos = num - 1; - - if (order == TSDB_ORDER_DESC) { - // find the first position which is smaller than the key - while (1) { - if (key >= keyList[lastPos]) return lastPos; - if (key == keyList[firstPos]) return firstPos; - if (key < keyList[firstPos]) return firstPos - 1; - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1) + firstPos; - - if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { - firstPos = midPos + 1; - } else { - break; - } - } - - } else { - // find the first position which is bigger than the key - while (1) { - if (key <= keyList[firstPos]) return firstPos; - if (key == keyList[lastPos]) return lastPos; - - if (key > keyList[lastPos]) { - lastPos = lastPos + 1; - if (lastPos >= num) - return -1; - else - return lastPos; - } - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1u) + firstPos; - - if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { - firstPos = midPos + 1; - } else { - break; - } - } - } - - return midPos; -} - -/* - * set tag value in SQLFunctionCtx - * e.g.,tag information into input buffer - */ -static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes) { - tVariantDestroy(tag); - - char* val = NULL; - if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { - val = tsdbGetTableName(pTable); - assert(val != NULL); - } else { - val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); - } - - if (val == NULL || isNull(val, type)) { - tag->nType = TSDB_DATA_TYPE_NULL; - return; - } - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t maxLen = bytes - VARSTR_HEADER_SIZE; - int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val); - tVariantCreateFromBinary(tag, varDataVal(val), len, type); - //tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type); - } else { - tVariantCreateFromBinary(tag, val, bytes, type); - } -} - -static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId) { - assert(pTagColList != NULL && numOfTags > 0); - - for(int32_t i = 0; i < numOfTags; ++i) { - if (pTagColList[i].colId == colId) { - return &pTagColList[i]; - } - } - - return NULL; -} - -void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCtx, int32_t numOfOutput) { - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - - SExprInfo *pExpr = pOperatorInfo->pExpr; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - SExprInfo* pExprInfo = &pExpr[0]; - if (pQueryAttr->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP && pQueryAttr->stableQuery) { - assert(pExprInfo->base.numOfParams == 1); - - int16_t tagColId = (int16_t)pExprInfo->base.param[0].i64; - SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId); - - doSetTagValueInParam(pTable, tagColId, &pCtx[0].tag, pColInfo->type, pColInfo->bytes); - return; - } else { - // set tag value, by which the results are aggregated. - int32_t offset = 0; - memset(pRuntimeEnv->tagVal, 0, pQueryAttr->tagLen); - - for (int32_t idx = 0; idx < numOfOutput; ++idx) { - SExprInfo* pLocalExprInfo = &pExpr[idx]; - - // ts_comp column required the tag value for join filter - if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) { - continue; - } - - // todo use tag column index to optimize performance - doSetTagValueInParam(pTable, pLocalExprInfo->base.colInfo.colId, &pCtx[idx].tag, pLocalExprInfo->base.resType, - pLocalExprInfo->base.resBytes); - - if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resType) - || pLocalExprInfo->base.resType == TSDB_DATA_TYPE_BOOL - || pLocalExprInfo->base.resType == TSDB_DATA_TYPE_TIMESTAMP) { - memcpy(pRuntimeEnv->tagVal + offset, &pCtx[idx].tag.i64, pLocalExprInfo->base.resBytes); - } else { - if (pCtx[idx].tag.pz != NULL) { - memcpy(pRuntimeEnv->tagVal + offset, pCtx[idx].tag.pz, pCtx[idx].tag.nLen); - } - } - - offset += pLocalExprInfo->base.resBytes; - } - - //todo : use index to avoid iterator all possible output columns - if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddev(pRuntimeEnv, pCtx, numOfOutput, pExprInfo); - } - } - - // set the tsBuf start position before check each data block - if (pRuntimeEnv->pTsBuf != NULL) { - setCtxTagForJoin(pRuntimeEnv, &pCtx[0], pExprInfo, pTable); - } -} - -static UNUSED_FUNC void printBinaryData(int32_t functionId, char *data, int32_t srcDataType) { - if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { - switch (srcDataType) { - case TSDB_DATA_TYPE_BINARY: - printf("%" PRId64 ",%s\t", *(TSKEY *)data, (data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BOOL: - printf("%" PRId64 ",%d\t", *(TSKEY *)data, *(int8_t *)(data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%" PRId64 ",%d\t", *(TSKEY *)data, *(int16_t *)(data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - printf("%" PRId64 ",%" PRId64 "\t", *(TSKEY *)data, *(TSKEY *)(data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_INT: - printf("%" PRId64 ",%d\t", *(TSKEY *)data, *(int32_t *)(data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%" PRId64 ",%f\t", *(TSKEY *)data, *(float *)(data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf("%" PRId64 ",%lf\t", *(TSKEY *)data, *(double *)(data + TSDB_KEYSIZE + 1)); - break; - } - } else if (functionId == TSDB_FUNC_AVG) { - printf("%lf,%d\t", *(double *)data, *(int32_t *)(data + sizeof(double))); - } else if (functionId == TSDB_FUNC_SPREAD) { - printf("%lf,%lf\t", *(double *)data, *(double *)(data + sizeof(double))); - } else if (functionId == TSDB_FUNC_TWA) { - data += 1; - printf("%lf,%" PRId64 ",%" PRId64 ",%" PRId64 "\t", *(double *)data, *(int64_t *)(data + 8), - *(int64_t *)(data + 16), *(int64_t *)(data + 24)); - } else if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { - switch (srcDataType) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BOOL: - printf("%d\t", *(int8_t *)data); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%d\t", *(int16_t *)data); - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - printf("%" PRId64 "\t", *(int64_t *)data); - break; - case TSDB_DATA_TYPE_INT: - printf("%d\t", *(int *)data); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)data); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf("%f\t", *(float *)data); - break; - } - } else if (functionId == TSDB_FUNC_SUM) { - if (srcDataType == TSDB_DATA_TYPE_FLOAT || srcDataType == TSDB_DATA_TYPE_DOUBLE) { - printf("%lf\t", *(float *)data); - } else { - printf("%" PRId64 "\t", *(int64_t *)data); - } - } else { - printf("%s\t", data); - } -} - -void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntimeEnv, int32_t numOfRows) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t numOfCols = pQueryAttr->numOfOutput; - printf("super table query intermediate result, total:%d\n", numOfRows); - - for (int32_t j = 0; j < numOfRows; ++j) { - for (int32_t i = 0; i < numOfCols; ++i) { - - switch (pQueryAttr->pExpr1[i].base.resType) { - case TSDB_DATA_TYPE_BINARY: { - int32_t type = pQueryAttr->pExpr1[i].base.resType; - printBinaryData(pQueryAttr->pExpr1[i].base.functionId, pdata[i]->data + pQueryAttr->pExpr1[i].base.resBytes * j, - type); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - printf("%" PRId64 "\t", *(int64_t *)(pdata[i]->data + pQueryAttr->pExpr1[i].base.resBytes * j)); - break; - case TSDB_DATA_TYPE_INT: - printf("%d\t", *(int32_t *)(pdata[i]->data + pQueryAttr->pExpr1[i].base.resBytes * j)); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)(pdata[i]->data + pQueryAttr->pExpr1[i].base.resBytes * j)); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf("%lf\t", *(double *)(pdata[i]->data + pQueryAttr->pExpr1[i].base.resBytes * j)); - break; - } - } - printf("\n"); - } -} - -void copyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBlock* pBlock, int32_t* offset) { - SGroupResInfo* pGroupResInfo = &pRuntimeEnv->groupResInfo; - pBlock->info.rows = 0; - - int32_t code = TSDB_CODE_SUCCESS; - while (pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) { - // all results in current group have been returned to client, try next group - if ((pGroupResInfo->pRows == NULL) || taosArrayGetSize(pGroupResInfo->pRows) == 0) { - assert(pGroupResInfo->index == 0); - if ((code = mergeIntoGroupResult(&pRuntimeEnv->groupResInfo, pRuntimeEnv, offset)) != TSDB_CODE_SUCCESS) { - return; - } - } - - doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, TSDB_ORDER_ASC, pBlock); - - // current data are all dumped to result buffer, clear it - if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { - cleanupGroupResInfo(pGroupResInfo); - if (!incNextGroup(pGroupResInfo)) { - break; - } - } - - // enough results in data buffer, return - if (pBlock->info.rows >= threshold) { - break; - } - } -} - -static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo) { - if (pTableQueryInfo == NULL) { - return; - } - - TSWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); - pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; - - SWITCH_ORDER(pTableQueryInfo->cur.order); - pTableQueryInfo->cur.vgroupIndex = -1; - - // set the index to be the end slot of result rows array - SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; - if (pResultRowInfo->size > 0) { - pResultRowInfo->curPos = pResultRowInfo->size - 1; - } else { - pResultRowInfo->curPos = -1; - } -} - -static void setupQueryRangeForReverseScan(SQueryRuntimeEnv* pRuntimeEnv) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pRuntimeEnv)); - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); - SArray *tableKeyGroup = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); - - size_t t = taosArrayGetSize(group); - for (int32_t j = 0; j < t; ++j) { - STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); - updateTableQueryInfoForReverseScan(pCheckInfo); - - // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide - // the start check timestamp of tsdbQueryHandle - STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); - pTableKeyInfo->lastKey = pCheckInfo->lastKey; - - assert(pCheckInfo->pTable == pTableKeyInfo->pTable); - } - } -} - -void switchCtxOrder(SQLFunctionCtx* pCtx, int32_t numOfOutput) { - for (int32_t i = 0; i < numOfOutput; ++i) { - SWITCH_ORDER(pCtx[i].order); - } -} - -int32_t initResultRow(SResultRow *pResultRow) { - pResultRow->pCellInfo = (SResultRowCellInfo*)((char*)pResultRow + sizeof(SResultRow)); - pResultRow->pageId = -1; - pResultRow->offset = -1; - return TSDB_CODE_SUCCESS; -} - -/* - * The start of each column SResultRowCellInfo is denote by RowCellInfoOffset. - * Note that in case of top/bottom query, the whole multiple rows of result is treated as only one row of results. - * +------------+-----------------result column 1-----------+-----------------result column 2-----------+ - * + SResultRow | SResultRowCellInfo | intermediate buffer1 | SResultRowCellInfo | intermediate buffer 2| - * +------------+-------------------------------------------+-------------------------------------------+ - * offset[0] offset[1] offset[2] - */ -void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid, int32_t stage) { - SQLFunctionCtx* pCtx = pInfo->pCtx; - SSDataBlock* pDataBlock = pInfo->pRes; - int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; - SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; - - int64_t tid = 0; - pRuntimeEnv->keyBuf = realloc(pRuntimeEnv->keyBuf, sizeof(tid) + sizeof(int64_t) + POINTER_BYTES); - SResultRow* pRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, uid); - - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); - - /* - * set the output buffer information and intermediate buffer - * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. - */ - SResultRowCellInfo* pCellInfo = getResultCell(pRow, i, rowCellInfoOffset); - RESET_RESULT_INFO(pCellInfo); - - pCtx[i].resultInfo = pCellInfo; - pCtx[i].pOutput = pData->pData; - pCtx[i].currentStage = stage; - assert(pCtx[i].pOutput != NULL); - - // set the timestamp output buffer for top/bottom/diff query - int32_t fid = pCtx[i].functionId; - if (fid == TSDB_FUNC_TOP || fid == TSDB_FUNC_BOTTOM || fid == TSDB_FUNC_DIFF || fid == TSDB_FUNC_DERIVATIVE) { - if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; - } - } - - initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); -} - -void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows) { - SSDataBlock* pDataBlock = pBInfo->pRes; - - int32_t newSize = pDataBlock->info.rows + numOfInputRows + 5; // extra output buffer - if ((*bufCapacity) < newSize) { - for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - - char* p = realloc(pColInfo->pData, newSize * pColInfo->info.bytes); - if (p != NULL) { - pColInfo->pData = p; - - // it starts from the tail of the previously generated results. - pBInfo->pCtx[i].pOutput = pColInfo->pData; - (*bufCapacity) = newSize; - } else { - // longjmp - } - } - } - - - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - pBInfo->pCtx[i].pOutput = pColInfo->pData + pColInfo->info.bytes * pDataBlock->info.rows; - - // set the correct pointer after the memory buffer reallocated. - int32_t functionId = pBInfo->pCtx[i].functionId; - - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - if (i > 0) pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; - } - } -} - -void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) { - bool needCopyTs = false; - int32_t tsNum = 0; - char *src = NULL; - for (int32_t i = 0; i < numOfOutput; i++) { - int32_t functionId = pCtx[i].functionId; - if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - needCopyTs = true; - if (i > 0 && pCtx[i-1].functionId == TSDB_FUNC_TS_DUMMY){ - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i - 1); // find ts data - src = pColRes->pData; - } - }else if(functionId == TSDB_FUNC_TS_DUMMY) { - tsNum++; - } - } - - if (!needCopyTs) return; - if (tsNum < 2) return; - if (src == NULL) return; - - for (int32_t i = 0; i < numOfOutput; i++) { - int32_t functionId = pCtx[i].functionId; - if(functionId == TSDB_FUNC_TS_DUMMY) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i); - memcpy(pColRes->pData, src, pColRes->info.bytes * pRes->info.rows); - } - } -} - -void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity) { - SSDataBlock* pDataBlock = pBInfo->pRes; - - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - - int32_t functionId = pBInfo->pCtx[i].functionId; - if (functionId < 0) { - memset(pBInfo->pCtx[i].pOutput, 0, pColInfo->info.bytes * (*bufCapacity)); - } - } -} - -void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { - for (int32_t j = 0; j < size; ++j) { - SResultRowCellInfo* pResInfo = GET_RES_INFO(&pCtx[j]); - if (pResInfo->initialized) { - continue; - } - - if (pCtx[j].functionId < 0) { // todo udf initialization - continue; - } else { - aAggs[pCtx[j].functionId].init(&pCtx[j], pCtx[j].resultInfo); - } - } -} - -void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status) { - if (status == QUERY_NOT_COMPLETED) { - pRuntimeEnv->status = status; - } else { - // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first - CLEAR_QUERY_STATUS(pRuntimeEnv, QUERY_NOT_COMPLETED); - pRuntimeEnv->status |= status; - } -} - -static void setupEnvForReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, SQLFunctionCtx* pCtx, int32_t numOfOutput) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - if (pRuntimeEnv->pTsBuf) { - SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); - bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); - assert(ret); - } - - // reverse order time range - TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - - SET_REVERSE_SCAN_FLAG(pRuntimeEnv); - setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); - - switchCtxOrder(pCtx, numOfOutput); - SWITCH_ORDER(pQueryAttr->order.order); - - setupQueryRangeForReverseScan(pRuntimeEnv); -} - -void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t numOfOutput = pOperator->numOfOutput; - if (pQueryAttr->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQueryAttr) || pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow) { - // for each group result, call the finalize function for each column - if (pQueryAttr->groupbyColumn) { - closeAllResultRows(pResultRowInfo); - } - - for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - SResultRow *buf = pResultRowInfo->pResult[i]; - if (!isResultRowClosed(pResultRowInfo, i)) { - continue; - } - - setResultOutputBuf(pRuntimeEnv, buf, pCtx, numOfOutput, rowCellInfoOffset); - - for (int32_t j = 0; j < numOfOutput; ++j) { - pCtx[j].startTs = buf->win.skey; - if (pCtx[j].functionId < 0) { - doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); - } else { - aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); - } - } - - - /* - * set the number of output results for group by normal columns, the number of output rows usually is 1 except - * the top and bottom query - */ - buf->numOfRows = (uint16_t)getNumOfResult(pRuntimeEnv, pCtx, numOfOutput); - } - - } else { - for (int32_t j = 0; j < numOfOutput; ++j) { - if (pCtx[j].functionId < 0) { - doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); - } else { - aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); - } - } - } -} - -static bool hasMainOutput(SQueryAttr *pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = pQueryAttr->pExpr1[i].base.functionId; - - if (functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TAGPRJ) { - return true; - } - } - - return false; -} - -STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf) { - STableQueryInfo *pTableQueryInfo = buf; - - pTableQueryInfo->win = win; - pTableQueryInfo->lastKey = win.skey; - - pTableQueryInfo->pTable = pTable; - pTableQueryInfo->cur.vgroupIndex = -1; - - // set more initial size of interval/groupby query - if (QUERY_IS_INTERVAL_QUERY(pQueryAttr) || groupbyColumn) { - int32_t initialSize = 128; - int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); - if (code != TSDB_CODE_SUCCESS) { - return NULL; - } - } else { // in other aggregate query, do not initialize the windowResInfo - } - - return pTableQueryInfo; -} - -STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { - STableQueryInfo* pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); - - pTableQueryInfo->win = win; - pTableQueryInfo->lastKey = win.skey; - - pTableQueryInfo->pTable = NULL; - pTableQueryInfo->cur.vgroupIndex = -1; - - // set more initial size of interval/groupby query - int32_t initialSize = 16; - int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); - if (code != TSDB_CODE_SUCCESS) { - tfree(pTableQueryInfo); - return NULL; - } - - return pTableQueryInfo; -} - -void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { - if (pTableQueryInfo == NULL) { - return; - } - - tVariantDestroy(&pTableQueryInfo->tag); - cleanupResultRowInfo(&pTableQueryInfo->resInfo); -} - -void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, - int32_t numOfOutput, int32_t* rowCellInfoOffset) { - // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); - - int32_t offset = 0; - for (int32_t i = 0; i < numOfOutput; ++i) { - pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); - - SResultRowCellInfo* pResInfo = pCtx[i].resultInfo; - if (pResInfo->initialized && pResInfo->complete) { - offset += pCtx[i].outputBytes; - continue; - } - - pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQueryAttr, bufPage, pResult->offset, offset); - offset += pCtx[i].outputBytes; - - int32_t functionId = pCtx[i].functionId; - if (functionId < 0) { - continue; - } - - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - if(i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; - } - - if (!pResInfo->initialized) { - aAggs[functionId].init(&pCtx[i], pResInfo); - } - } -} - -void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, - int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId) { - // for simple group by query without interval, all the tables belong to one group result. - int64_t uid = 0; - int64_t tid = 0; - - SResultRow* pResultRow = - doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char*)&tableGroupId, sizeof(tableGroupId), true, uid); - assert (pResultRow != NULL); - - /* - * not assign result buffer yet, add new result buffer - * all group belong to one result set, and each group result has different group id so set the id to be one - */ - if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, tableGroupId, pRuntimeEnv->pQueryAttr->resultRowSize); - if (ret != TSDB_CODE_SUCCESS) { - return; - } - } - - setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); -} - -void setExecutionContext(SQueryRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, int32_t numOfOutput, int32_t tableGroupId, - TSKEY nextKey) { - STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; - - // lastKey needs to be updated - pTableQueryInfo->lastKey = nextKey; - if (pRuntimeEnv->prevGroupId != INT32_MIN && pRuntimeEnv->prevGroupId == tableGroupId) { - return; - } - - doSetTableGroupOutputBuf(pRuntimeEnv, &pInfo->resultRowInfo, pInfo->pCtx, pInfo->rowCellInfoOffset, numOfOutput, tableGroupId); - - // record the current active group id - pRuntimeEnv->prevGroupId = tableGroupId; -} - -void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, - int32_t numOfCols, int32_t* rowCellInfoOffset) { - // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); - - int16_t offset = 0; - for (int32_t i = 0; i < numOfCols; ++i) { - pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQueryAttr, page, pResult->offset, offset); - offset += pCtx[i].outputBytes; - - int32_t functionId = pCtx[i].functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) { - if(i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; - } - - /* - * set the output buffer information and intermediate buffer, - * not all queries require the interResultBuf, such as COUNT - */ - pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); - } -} - -void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - SSqlExpr* pExpr = &pExprInfo->base; - if (pQueryAttr->stableQuery && (pRuntimeEnv->pTsBuf != NULL) && - (pExpr->functionId == TSDB_FUNC_TS || pExpr->functionId == TSDB_FUNC_PRJ) && - (pExpr->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX)) { - assert(pExpr->numOfParams == 1); - - int16_t tagColId = (int16_t)pExprInfo->base.param[0].i64; - SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId); - - doSetTagValueInParam(pTable, tagColId, &pCtx->tag, pColInfo->type, pColInfo->bytes); - - int16_t tagType = pCtx[0].tag.nType; - if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_QID(pRuntimeEnv), - pExprInfo->base.param[0].i64, pCtx[0].tag.pz); - } else { - qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_QID(pRuntimeEnv), - pExprInfo->base.param[0].i64, pCtx[0].tag.i64); - } - } -} - -int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, tVariant* pTag, STableQueryInfo *pTableQueryInfo) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - assert(pRuntimeEnv->pTsBuf != NULL); - - // both the master and supplement scan needs to set the correct ts comp start position - if (pTableQueryInfo->cur.vgroupIndex == -1) { - tVariantAssign(&pTableQueryInfo->tag, pTag); - - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQueryAttr->vgId, &pTableQueryInfo->tag); - - // failed to find data with the specified tag value and vnodeId - if (!tsBufIsValidElem(&elem)) { - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_QID(pRuntimeEnv), pTag->pz); - } else { - qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_QID(pRuntimeEnv), pTag->i64); - } - - return -1; - } - - // Keep the cursor info of current table - pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } else { - qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i64, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } - - } else { - tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } else { - qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i64, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } - } - - return 0; -} - -// TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function. -void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t numOfExprs = pQueryAttr->numOfOutput; - for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExprInfo1 = &(pExprInfo[i]); - if (pExprInfo1->base.functionId != TSDB_FUNC_STDDEV_DST) { - continue; - } - - SSqlExpr* pExpr = &pExprInfo1->base; - - pCtx[i].param[0].arr = NULL; - pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int - - // TODO use hash to speedup this loop - int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); - for (int32_t j = 0; j < numOfGroup; ++j) { - SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); - if (pQueryAttr->tagLen == 0 || memcmp(p->tags, pRuntimeEnv->tagVal, pQueryAttr->tagLen) == 0) { - int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); - for (int32_t k = 0; k < numOfCols; ++k) { - SStddevInterResult* pres = taosArrayGet(p->pResult, k); - if (pres->colId == pExpr->colInfo.colId) { - pCtx[i].param[0].arr = pres->pResult; - break; - } - } - } - } - } - -} - -void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t numOfExprs = pQueryAttr->numOfOutput; - for(int32_t i = 0; i < numOfExprs; ++i) { - SSqlExpr* pExpr1 = &pExpr[i].base; - if (pExpr1->functionId != TSDB_FUNC_STDDEV_DST) { - continue; - } - - pCtx[i].param[0].arr = NULL; - pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int - - // TODO use hash to speedup this loop - int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); - for (int32_t j = 0; j < numOfGroup; ++j) { - SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); - if (bytes == 0 || memcmp(p->tags, val, bytes) == 0) { - int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); - for (int32_t k = 0; k < numOfCols; ++k) { - SStddevInterResult* pres = taosArrayGet(p->pResult, k); - if (pres->colId == pExpr1->colInfo.colId) { - pCtx[i].param[0].arr = pres->pResult; - break; - } - } - } - } - } -} - -/* - * There are two cases to handle: - * - * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including pQueryAttr->lastKey, - * pQueryAttr->window.skey, and pQueryAttr->eKey. - * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be - * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there - * is a previous result generated or not. - */ -void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; - SResultRowInfo *pResultRowInfo = &pTableQueryInfo->resInfo; - - if (pResultRowInfo->curPos != -1) { - return; - } - - pTableQueryInfo->win.skey = key; - STimeWindow win = {.skey = key, .ekey = pQueryAttr->window.ekey}; - - /** - * In handling the both ascending and descending order super table query, we need to find the first qualified - * timestamp of this table, and then set the first qualified start timestamp. - * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional - * operations involve. - */ - STimeWindow w = TSWINDOW_INITIALIZER; - - TSKEY sk = TMIN(win.skey, win.ekey); - TSKEY ek = TMAX(win.skey, win.ekey); - getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); - -// if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { -// if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { -// assert(win.ekey == pQueryAttr->window.ekey); -// } -// -// pResultRowInfo->prevSKey = w.skey; -// } - - pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; -} - -/** - * copyToOutputBuf support copy data in ascending/descending order - * For interval query of both super table and table, copy the data in ascending order, since the output results are - * ordered in SWindowResutl already. While handling the group by query for both table and super table, - * all group result are completed already. - * - * @param pQInfo - * @param result - */ - -static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); - int32_t numOfResult = pBlock->info.rows; // there are already exists result rows - - int32_t start = 0; - int32_t step = -1; - - qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_QID(pRuntimeEnv)); - assert(orderType == TSDB_ORDER_ASC || orderType == TSDB_ORDER_DESC); - - if (orderType == TSDB_ORDER_ASC) { - start = pGroupResInfo->index; - step = 1; - } else { // desc order copy all data - start = numOfRows - pGroupResInfo->index - 1; - step = -1; - } - - for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { - SResultRow* pRow = taosArrayGetP(pGroupResInfo->pRows, i); - if (pRow->numOfRows == 0) { - pGroupResInfo->index += 1; - continue; - } - - int32_t numOfRowsToCopy = pRow->numOfRows; - if (numOfResult + numOfRowsToCopy >= pRuntimeEnv->resultInfo.capacity) { - break; - } - - pGroupResInfo->index += 1; - - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRow->pageId); - - int32_t offset = 0; - for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j); - int32_t bytes = pColInfoData->info.bytes; - - char *out = pColInfoData->pData + numOfResult * bytes; - char *in = getPosInResultPage(pQueryAttr, page, pRow->offset, offset); - memcpy(out, in, bytes * numOfRowsToCopy); - - offset += bytes; - } - - numOfResult += numOfRowsToCopy; - if (numOfResult == pRuntimeEnv->resultInfo.capacity) { // output buffer is full - break; - } - } - - qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_QID(pRuntimeEnv)); - pBlock->info.rows = numOfResult; - return 0; -} - -static void toSSDataBlock(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock) { - assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); - - pBlock->info.rows = 0; - if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { - return; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t orderType = (pQueryAttr->pGroupbyExpr != NULL) ? pQueryAttr->pGroupbyExpr->orderType : TSDB_ORDER_ASC; - doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, orderType, pBlock); - - // refactor : extract method - SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0); - - //add condition (pBlock->info.rows >= 1) just to runtime happy - if (pInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && pBlock->info.rows >= 1) { - STimeWindow* w = &pBlock->info.window; - w->skey = *(int64_t*)pInfoData->pData; - w->ekey = *(int64_t*)(((char*)pInfoData->pData) + TSDB_KEYSIZE * (pBlock->info.rows - 1)); - } -} - -static void updateNumOfRowsInResultRows(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, - SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - // update the number of result for each, only update the number of rows for the corresponding window result. - if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { - return; - } - - for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - SResultRow *pResult = pResultRowInfo->pResult[i]; - - for (int32_t j = 0; j < numOfOutput; ++j) { - int32_t functionId = pCtx[j].functionId; - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { - continue; - } - - SResultRowCellInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset); - pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes)); - } - } -} - -static int32_t compressQueryColData(SColumnInfoData *pColRes, int32_t numOfRows, char *data, int8_t compressed) { - int32_t colSize = pColRes->info.bytes * numOfRows; - return (*(tDataTypes[pColRes->info.type].compFunc))(pColRes->pData, colSize, numOfRows, data, - colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); -} - -static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data, int8_t compressed, int32_t *compLen) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - SSDataBlock* pRes = pRuntimeEnv->outputBuf; - - int32_t *compSizes = NULL; - int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput; - - if (compressed) { - compSizes = tcalloc(numOfCols, sizeof(int32_t)); - } - - if (pQueryAttr->pExpr2 == NULL) { - for (int32_t col = 0; col < numOfCols; ++col) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); - if (compressed) { - compSizes[col] = compressQueryColData(pColRes, pRes->info.rows, data, compressed); - data += compSizes[col]; - *compLen += compSizes[col]; - compSizes[col] = htonl(compSizes[col]); - } else { - memmove(data, pColRes->pData, pColRes->info.bytes * pRes->info.rows); - data += pColRes->info.bytes * pRes->info.rows; - } - } - } else { - for (int32_t col = 0; col < numOfCols; ++col) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); - if (compressed) { - compSizes[col] = htonl(compressQueryColData(pColRes, numOfRows, data, compressed)); - data += compSizes[col]; - *compLen += compSizes[col]; - compSizes[col] = htonl(compSizes[col]); - } else { - memmove(data, pColRes->pData, pColRes->info.bytes * numOfRows); - data += pColRes->info.bytes * numOfRows; - } - } - } - - if (compressed) { - memmove(data, (char *)compSizes, numOfCols * sizeof(int32_t)); - data += numOfCols * sizeof(int32_t); - - tfree(compSizes); - } - - int32_t numOfTables = (int32_t) taosHashGetSize(pRuntimeEnv->pTableRetrieveTsMap); - *(int32_t*)data = htonl(numOfTables); - data += sizeof(int32_t); - - int32_t total = 0; - STableIdInfo* item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, NULL); - - while(item) { - STableIdInfo* pDst = (STableIdInfo*)data; - pDst->uid = htobe64(item->uid); - pDst->tid = htonl(item->tid); - pDst->key = htobe64(item->key); - - data += sizeof(STableIdInfo); - total++; - - qDebug("QInfo:0x%"PRIx64" set subscribe info, tid:%d, uid:%"PRIu64", skey:%"PRId64, pQInfo->qId, item->tid, item->uid, item->key); - item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, item); - } - - qDebug("QInfo:0x%"PRIx64" set %d subscribe info", pQInfo->qId, total); - - // Check if query is completed or not for stable query or normal table query respectively. - if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { - setQueryStatus(pRuntimeEnv, QUERY_OVER); - } -} - -int32_t doFillTimeIntervalGapsInResults(SFillInfo* pFillInfo, SSDataBlock *pOutput, int32_t capacity, void** p) { - for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pOutput->pDataBlock, i); - p[i] = pColInfoData->pData + (pColInfoData->info.bytes * pOutput->info.rows); - } - - int32_t numOfRows = (int32_t)taosFillResultDataBlock(pFillInfo, p, capacity - pOutput->info.rows); - pOutput->info.rows += numOfRows; - - return pOutput->info.rows; -} - -void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType) { - SQueryProfEvent event = {0}; - - event.eventType = eventType; - event.eventTime = taosGetTimestampUs(); - event.operatorType = operatorInfo->operatorType; - - if (operatorInfo->pRuntimeEnv) { - SQInfo* pQInfo = operatorInfo->pRuntimeEnv->qinfo; - if (pQInfo->summary.queryProfEvents) { - taosArrayPush(pQInfo->summary.queryProfEvents, &event); - } - } -} - -void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code) { - SQueryProfEvent event; - event.eventType = QUERY_PROF_QUERY_ABORT; - event.eventTime = taosGetTimestampUs(); - event.abortCode = code; - - if (pQInfo->summary.queryProfEvents) { - taosArrayPush(pQInfo->summary.queryProfEvents, &event); - } -} - -typedef struct { - uint8_t operatorType; - int64_t beginTime; - int64_t endTime; - int64_t selfTime; - int64_t descendantsTime; -} SOperatorStackItem; - -static void doOperatorExecProfOnce(SOperatorStackItem* item, SQueryProfEvent* event, SArray* opStack, SHashObj* profResults) { - item->endTime = event->eventTime; - item->selfTime = (item->endTime - item->beginTime) - (item->descendantsTime); - - for (int32_t j = 0; j < taosArrayGetSize(opStack); ++j) { - SOperatorStackItem* ancestor = taosArrayGet(opStack, j); - ancestor->descendantsTime += item->selfTime; - } - - uint8_t operatorType = item->operatorType; - SOperatorProfResult* result = taosHashGet(profResults, &operatorType, sizeof(operatorType)); - if (result != NULL) { - result->sumRunTimes++; - result->sumSelfTime += item->selfTime; - } else { - SOperatorProfResult opResult; - opResult.operatorType = operatorType; - opResult.sumSelfTime = item->selfTime; - opResult.sumRunTimes = 1; - taosHashPut(profResults, &(operatorType), sizeof(operatorType), - &opResult, sizeof(opResult)); - } -} - -void calculateOperatorProfResults(SQInfo* pQInfo) { - if (pQInfo->summary.queryProfEvents == NULL) { - qDebug("QInfo:0x%"PRIx64" query prof events array is null", pQInfo->qId); - return; - } - - if (pQInfo->summary.operatorProfResults == NULL) { - qDebug("QInfo:0x%"PRIx64" operator prof results hash is null", pQInfo->qId); - return; - } - - SArray* opStack = taosArrayInit(32, sizeof(SOperatorStackItem)); - if (opStack == NULL) { - return; - } - - size_t size = taosArrayGetSize(pQInfo->summary.queryProfEvents); - SHashObj* profResults = pQInfo->summary.operatorProfResults; - - for (int i = 0; i < size; ++i) { - SQueryProfEvent* event = taosArrayGet(pQInfo->summary.queryProfEvents, i); - if (event->eventType == QUERY_PROF_BEFORE_OPERATOR_EXEC) { - SOperatorStackItem opItem; - opItem.operatorType = event->operatorType; - opItem.beginTime = event->eventTime; - opItem.descendantsTime = 0; - taosArrayPush(opStack, &opItem); - } else if (event->eventType == QUERY_PROF_AFTER_OPERATOR_EXEC) { - SOperatorStackItem* item = taosArrayPop(opStack); - assert(item->operatorType == event->operatorType); - doOperatorExecProfOnce(item, event, opStack, profResults); - } else if (event->eventType == QUERY_PROF_QUERY_ABORT) { - SOperatorStackItem* item; - while ((item = taosArrayPop(opStack)) != NULL) { - doOperatorExecProfOnce(item, event, opStack, profResults); - } - } - } - - taosArrayDestroy(opStack); -} - -void queryCostStatis(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryCostInfo *pSummary = &pQInfo->summary; - - uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); - hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map); - pSummary->hashSize = hashSize; - - // add the merge time - pSummary->elapsedTime += pSummary->firstStageMergeTime; - - SResultRowPool* p = pQInfo->runtimeEnv.pool; - if (p != NULL) { - pSummary->winInfoSize = getResultRowPoolMemSize(p); - pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); - } else { - pSummary->winInfoSize = 0; - pSummary->numOfTimeWindows = 0; - } - - calculateOperatorProfResults(pQInfo); - - qDebug("QInfo:0x%"PRIx64" :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " - "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, - pQInfo->qId, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, - pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - - qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, - pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); - - if (pSummary->operatorProfResults) { - SOperatorProfResult* opRes = taosHashIterate(pSummary->operatorProfResults, NULL); - while (opRes != NULL) { - qDebug("QInfo:0x%" PRIx64 " :cost summary: operator : %d, exec times: %" PRId64 ", self time: %" PRId64, - pQInfo->qId, opRes->operatorType, opRes->sumRunTimes, opRes->sumSelfTime); - opRes = taosHashIterate(pSummary->operatorProfResults, opRes); - } - } -} - -//static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// -// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); -// -// if (pQueryAttr->limit.offset == pBlockInfo->rows) { // current block will ignore completed -// pTableQueryInfo->lastKey = QUERY_IS_ASC_QUERY(pQueryAttr) ? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; -// pQueryAttr->limit.offset = 0; -// return; -// } -// -// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { -// pQueryAttr->pos = (int32_t)pQueryAttr->limit.offset; -// } else { -// pQueryAttr->pos = pBlockInfo->rows - (int32_t)pQueryAttr->limit.offset - 1; -// } -// -// assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); -// -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); -// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); -// -// // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value -// TSKEY *keys = (TSKEY *) pColInfoData->pData; -// -// // update the offset value -// pTableQueryInfo->lastKey = keys[pQueryAttr->pos]; -// pQueryAttr->limit.offset = 0; -// -// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); -// -// qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_QID(pRuntimeEnv), -// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); -//} - -//void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// -// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0) { -// return; -// } -// -// pQueryAttr->pos = 0; -// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); -// -// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle; -// -// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pTsdbReadHandle)) { -// if (isQueryKilled(pRuntimeEnv->qinfo)) { -// longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); -// } -// -// tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo); -// -// if (pQueryAttr->limit.offset > blockInfo.rows) { -// pQueryAttr->limit.offset -= blockInfo.rows; -// pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey; -// pTableQueryInfo->lastKey += step; -// -// qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_QID(pRuntimeEnv), blockInfo.rows, -// pQuery->limit.offset); -// } else { // find the appropriated start position in current block -// updateOffsetVal(pRuntimeEnv, &blockInfo); -// break; -// } -// } -// -// if (terrno != TSDB_CODE_SUCCESS) { -// longjmp(pRuntimeEnv->env, terrno); -// } -//} - -//static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; -// -// assert(pQueryAttr->limit.offset == 0); -// STimeWindow tw = *win; -// getNextTimeWindow(pQueryAttr, &tw); -// -// if ((tw.skey <= pBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) || -// (tw.ekey >= pBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { -// -// // load the data block and check data remaining in current data block -// // TODO optimize performance -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); -// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); -// -// tw = *win; -// int32_t startPos = -// getNextQualifiedWindow(pQueryAttr, &tw, pBlockInfo, pColInfoData->pData, binarySearchForKey, -1); -// assert(startPos >= 0); -// -// // set the abort info -// pQueryAttr->pos = startPos; -// -// // reset the query start timestamp -// pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; -// pQueryAttr->window.skey = pTableQueryInfo->win.skey; -// TSKEY key = pTableQueryInfo->win.skey; -// -// pWindowResInfo->prevSKey = tw.skey; -// int32_t index = pRuntimeEnv->resultRowInfo.curIndex; -// -// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); -// pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index -// -// qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, -// GET_QID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, -// pQueryAttr->current->lastKey); -// -// return key; -// } else { // do nothing -// pQueryAttr->window.skey = tw.skey; -// pWindowResInfo->prevSKey = tw.skey; -// pTableQueryInfo->lastKey = tw.skey; -// -// return tw.skey; -// } -// -// return true; -//} - -//static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { -// assert(*start <= pRuntimeEnv->current->lastKey); -// } else { -// assert(*start >= pRuntimeEnv->current->lastKey); -// } -// -// // if queried with value filter, do NOT forward query start position -// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { -// return true; -// } -// -// /* -// * 1. for interval without interpolation query we forward pQueryAttr->interval.interval at a time for -// * pQueryAttr->limit.offset times. Since hole exists, pQueryAttr->interval.interval*pQueryAttr->limit.offset value is -// * not valid. otherwise, we only forward pQueryAttr->limit.offset number of points -// */ -// assert(pRuntimeEnv->resultRowInfo.prevSKey == TSKEY_INITIAL_VAL); -// -// STimeWindow w = TSWINDOW_INITIALIZER; -// bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); -// -// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; -// STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; -// -// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) { -// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo); -// -// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { -// if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { -// getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.skey, blockInfo.window.skey, pQueryAttr->window.ekey, &w); -// pWindowResInfo->prevSKey = w.skey; -// } -// } else { -// getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.ekey, pQueryAttr->window.ekey, blockInfo.window.ekey, &w); -// pWindowResInfo->prevSKey = w.skey; -// } -// -// // the first time window -// STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQueryAttr); -// -// while (pQueryAttr->limit.offset > 0) { -// STimeWindow tw = win; -// -// if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) { -// pQueryAttr->limit.offset -= 1; -// pWindowResInfo->prevSKey = win.skey; -// -// // current time window is aligned with blockInfo.window.ekey -// // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL; -// if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) { -// pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; -// } -// } -// -// if (pQueryAttr->limit.offset == 0) { -// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); -// return true; -// } -// -// // current window does not ended in current data block, try next data block -// getNextTimeWindow(pQueryAttr, &tw); -// -// /* -// * If the next time window still starts from current data block, -// * load the primary timestamp column first, and then find the start position for the next queried time window. -// * Note that only the primary timestamp column is required. -// * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually required -// * time window resides in current data block. -// */ -// if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { -// -// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); -// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); -// -// if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { -// pQueryAttr->limit.offset -= 1; -// } -// -// if (pQueryAttr->limit.offset == 0) { -// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); -// return true; -// } else { -// tw = win; -// int32_t startPos = -// getNextQualifiedWindow(pQueryAttr, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1); -// assert(startPos >= 0); -// -// // set the abort info -// pQueryAttr->pos = startPos; -// pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; -// pWindowResInfo->prevSKey = tw.skey; -// win = tw; -// } -// } else { -// break; // offset is not 0, and next time window begins or ends in the next block. -// } -// } -// } -// -// // check for error -// if (terrno != TSDB_CODE_SUCCESS) { -// longjmp(pRuntimeEnv->env, terrno); -// } -// -// return true; -//} - -void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream) { - if (p->upstream == NULL) { - assert(p->numOfUpstream == 0); - } - - p->upstream = realloc(p->upstream, POINTER_BYTES * (p->numOfUpstream + 1)); - p->upstream[p->numOfUpstream++] = pUpstream; -} - -static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo); - -static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64_t qId, bool isSTableQuery) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - // TODO set the tags scan handle - if (onlyQueryTags(pQueryAttr)) { - return TSDB_CODE_SUCCESS; - } - - STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - if (pQueryAttr->tsCompQuery || pQueryAttr->pointInterpQuery) { - cond.type = BLOCK_LOAD_TABLE_SEQ_ORDER; - } - - if (!isSTableQuery - && (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 1) - && (cond.order == TSDB_ORDER_ASC) - && (!QUERY_IS_INTERVAL_QUERY(pQueryAttr)) - && (!pQueryAttr->groupbyColumn) - && (!pQueryAttr->simpleAgg) - ) { - SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); - STableQueryInfo* pCheckInfo = taosArrayGetP(pa, 0); - cond.twindow = pCheckInfo->win; - } - - terrno = TSDB_CODE_SUCCESS; - if (isFirstLastRowQuery(pQueryAttr)) { - pRuntimeEnv->pTsdbReadHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); - - // update the query time window - pQueryAttr->window = cond.twindow; - if (pQueryAttr->tableGroupInfo.numOfTables == 0) { - pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0; - } else { - size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); - - size_t t = taosArrayGetSize(group); - for (int32_t j = 0; j < t; ++j) { - STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); - - pCheckInfo->win = pQueryAttr->window; - pCheckInfo->lastKey = pCheckInfo->win.skey; - } - } - } - } else if (isCachedLastQuery(pQueryAttr)) { - pRuntimeEnv->pTsdbReadHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); - } else if (pQueryAttr->pointInterpQuery) { - pRuntimeEnv->pTsdbReadHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); - } else { - pRuntimeEnv->pTsdbReadHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); - } - - return terrno; -} - -static SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, int64_t* fillVal) { - int32_t offset = 0; - - SFillColInfo* pFillCol = calloc(numOfOutput, sizeof(SFillColInfo)); - if (pFillCol == NULL) { - return NULL; - } - - for(int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExprInfo = &pExpr[i]; - - pFillCol[i].col.bytes = pExprInfo->base.resBytes; - pFillCol[i].col.type = (int8_t)pExprInfo->base.resType; - pFillCol[i].col.offset = offset; - pFillCol[i].col.colId = pExprInfo->base.resColId; - pFillCol[i].tagIndex = -2; - pFillCol[i].flag = pExprInfo->base.colInfo.flag; // always be the normal column for table query - pFillCol[i].functionId = pExprInfo->base.functionId; - pFillCol[i].fillVal.i = fillVal[i]; - - offset += pExprInfo->base.resBytes; - } - - return pFillCol; -} - -int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr, int32_t tbScanner, SArray* pOperator, - void* param) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - pQueryAttr->tsdb = tsdb; - - if (tsdb != NULL) { - int32_t code = setupQueryHandle(tsdb, pRuntimeEnv, pQInfo->qId, pQueryAttr->stableQuery); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - pQueryAttr->interBufSize = getOutputInterResultBufSize(pQueryAttr); - - pRuntimeEnv->groupResInfo.totalGroup = (int32_t) (pQueryAttr->stableQuery? GET_NUM_OF_TABLEGROUP(pRuntimeEnv):0); - pRuntimeEnv->enableGroupData = false; - - pRuntimeEnv->pQueryAttr = pQueryAttr; - pRuntimeEnv->pTsBuf = pTsBuf; - pRuntimeEnv->cur.vgroupIndex = -1; - setResultBufSize(pQueryAttr, &pRuntimeEnv->resultInfo); - - switch(tbScanner) { - case OP_TableBlockInfoScan: { - pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); - break; - } - case OP_TableSeqScan: { - pRuntimeEnv->proot = createTableSeqScanOperatorInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); - break; - } - case OP_DataBlocksOptScan: { - pRuntimeEnv->proot = createTableScanOperatorInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); - break; - } - case OP_TableScan: { - pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); - break; - } - default: { // do nothing - break; - } - } - - if (sourceOptr != NULL) { - assert(pRuntimeEnv->proot == NULL); - pRuntimeEnv->proot = sourceOptr; - } - - if (pTsBuf != NULL) { - int16_t order = (pQueryAttr->order.order == pRuntimeEnv->pTsBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - tsBufSetTraverseOrder(pRuntimeEnv->pTsBuf, order); - } - - int32_t ps = DEFAULT_PAGE_SIZE; - getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize); - - int32_t TENMB = 1024*1024*10; - int32_t code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // create runtime environment - int32_t numOfTables = (int32_t)pQueryAttr->tableGroupInfo.numOfTables; - pQInfo->summary.tableInfoSize += (numOfTables * sizeof(STableQueryInfo)); - pQInfo->summary.queryProfEvents = taosArrayInit(512, sizeof(SQueryProfEvent)); - if (pQInfo->summary.queryProfEvents == NULL) { - qDebug("QInfo:0x%"PRIx64" failed to allocate query prof events array", pQInfo->qId); - } - - pQInfo->summary.operatorProfResults = - taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TINYINT), true, HASH_NO_LOCK); - - if (pQInfo->summary.operatorProfResults == NULL) { - qDebug("QInfo:0x%"PRIx64" failed to allocate operator prof results hash", pQInfo->qId); - } - - code = setupQueryRuntimeEnv(pRuntimeEnv, (int32_t) pQueryAttr->tableGroupInfo.numOfTables, pOperator, param); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); - return TSDB_CODE_SUCCESS; -} - -static void doTableQueryInfoTimeWindowCheck(SQueryAttr* pQueryAttr, STableQueryInfo* pTableQueryInfo) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - assert( - (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && - (pTableQueryInfo->lastKey >= pTableQueryInfo->win.skey) && - (pTableQueryInfo->win.skey >= pQueryAttr->window.skey && pTableQueryInfo->win.ekey <= pQueryAttr->window.ekey)); - } else { - assert( - (pTableQueryInfo->win.skey >= pTableQueryInfo->win.ekey) && - (pTableQueryInfo->lastKey <= pTableQueryInfo->win.skey) && - (pTableQueryInfo->win.skey <= pQueryAttr->window.skey && pTableQueryInfo->win.ekey >= pQueryAttr->window.ekey)); - } -} - -STsdbQueryCond createTsdbQueryCond(SQueryAttr* pQueryAttr, STimeWindow* win) { - STsdbQueryCond cond = { - .colList = pQueryAttr->tableCols, - .order = pQueryAttr->order.order, - .numOfCols = pQueryAttr->numOfCols, - .type = BLOCK_LOAD_OFFSET_SEQ_ORDER, - .loadExternalRows = false, - }; - - TIME_WINDOW_COPY(cond.twindow, *win); - return cond; -} - -static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo) { - STableIdInfo tidInfo; - STableId* id = TSDB_TABLEID(pTableQueryInfo->pTable); - - tidInfo.uid = id->uid; - tidInfo.tid = id->tid; - tidInfo.key = pTableQueryInfo->lastKey; - - return tidInfo; -} - -static void updateTableIdInfo(STableQueryInfo* pTableQueryInfo, SSDataBlock* pBlock, SHashObj* pTableIdInfo, int32_t order) { - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - pTableQueryInfo->lastKey = ((order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey) + step; - - if (pTableQueryInfo->pTable == NULL) { - return; - } - - STableIdInfo tidInfo = createTableIdInfo(pTableQueryInfo); - STableIdInfo *idinfo = taosHashGet(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid)); - if (idinfo != NULL) { - assert(idinfo->tid == tidInfo.tid && idinfo->uid == tidInfo.uid); - idinfo->key = tidInfo.key; - } else { - taosHashPut(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); - } -} - -static void doCloseAllTimeWindow(SQueryRuntimeEnv* pRuntimeEnv) { - size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); - for (int32_t i = 0; i < numOfGroup; ++i) { - SArray* group = GET_TABLEGROUP(pRuntimeEnv, i); - - size_t num = taosArrayGetSize(group); - for (int32_t j = 0; j < num; ++j) { - STableQueryInfo* item = taosArrayGetP(group, j); - closeAllResultRows(&item->resInfo); - } - } -} - -static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo*) param; - - STableScanInfo *pTableScanInfo = pOperator->info; - SSDataBlock *pBlock = &pTableScanInfo->block; - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - STableGroupInfo *pTableGroupInfo = &pOperator->pRuntimeEnv->tableqinfoGroupInfo; - - *newgroup = false; - - while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) { - if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { - longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } - - pTableScanInfo->numOfBlocks += 1; - tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info); - - // todo opt - if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { - STableQueryInfo** pTableQueryInfo = - (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.tid, sizeof(pBlock->info.tid)); - if (pTableQueryInfo == NULL) { - break; - } - - pRuntimeEnv->current = *pTableQueryInfo; - doTableQueryInfoTimeWindowCheck(pQueryAttr, *pTableQueryInfo); - - if (pRuntimeEnv->enableGroupData) { - if(pTableScanInfo->prevGroupId != -1 && pTableScanInfo->prevGroupId != (*pTableQueryInfo)->groupIndex) { - *newgroup = true; - } - } - - pTableScanInfo->prevGroupId = (*pTableQueryInfo)->groupIndex; - } - - // this function never returns error? - uint32_t status; - int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pOperator->pRuntimeEnv->env, code); - } - - // current block is ignored according to filter result by block statistics data, continue load the next block - if (status == BLK_DATA_DISCARD || pBlock->info.rows == 0) { - continue; - } - - return pBlock; - } - - return NULL; -} - -static SSDataBlock* doTableScan(void* param, bool *newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - - STableScanInfo *pTableScanInfo = pOperator->info; - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; - *newgroup = false; - - while (pTableScanInfo->current < pTableScanInfo->times) { - SSDataBlock* p = doTableScanImpl(pOperator, newgroup); - if (p != NULL) { - return p; - } - - if (++pTableScanInfo->current >= pTableScanInfo->times) { - if (pTableScanInfo->reverseTimes <= 0 || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)) { - return NULL; - } else { - break; - } - } - - // do prepare for the next round table scan operation - STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - - setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); - pRuntimeEnv->scanFlag = REPEAT_SCAN; - - if (pRuntimeEnv->pTsBuf) { - bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); - assert(ret); - } - - if (pResultRowInfo->size > 0) { - pResultRowInfo->curPos = 0; - } - - qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, - GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); - } - - SSDataBlock *p = NULL; - if (pTableScanInfo->reverseTimes > 0) { - setupEnvForReverseScan(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); - - STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - - qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, - GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); - - pRuntimeEnv->scanFlag = REVERSE_SCAN; - - pTableScanInfo->times = 1; - pTableScanInfo->current = 0; - pTableScanInfo->reverseTimes = 0; - pTableScanInfo->order = cond.order; - - if (pResultRowInfo->size > 0) { - pResultRowInfo->curPos = pResultRowInfo->size - 1; - } - - p = doTableScanImpl(pOperator, newgroup); - } - - return p; -} - -static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo*)param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STableScanInfo *pTableScanInfo = pOperator->info; - *newgroup = false; - - STableBlockDist tableBlockDist = {0}; - tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables; - - int32_t numRowSteps = tsMaxRowsInFileBlock / TSDB_BLOCK_DIST_STEP_ROWS; - if (tsMaxRowsInFileBlock % TSDB_BLOCK_DIST_STEP_ROWS != 0) { - ++numRowSteps; - } - tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); - taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps); - tableBlockDist.maxRows = INT_MIN; - tableBlockDist.minRows = INT_MAX; - - tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist); - tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle); - - SSDataBlock* pBlock = &pTableScanInfo->block; - pBlock->info.rows = 1; - pBlock->info.numOfCols = 1; - - SBufferWriter bw = tbufInitWriter(NULL, false); - blockDistInfoToBinary(&tableBlockDist, &bw); - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); - - int32_t len = (int32_t) tbufTell(&bw); - pColInfo->pData = malloc(len + sizeof(int32_t)); - - *(int32_t*) pColInfo->pData = len; - memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); - - tbufCloseWriter(&bw); - - SArray* g = GET_TABLEGROUP(pOperator->pRuntimeEnv, 0); - pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); - - pOperator->status = OP_EXEC_DONE; - return pBlock; -} - -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime) { - assert(repeatTime > 0); - - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pTsdbReadHandle = pTsdbQueryHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = 0; - pInfo->order = pRuntimeEnv->pQueryAttr->order.order; - pInfo->current = 0; -// pInfo->prevGroupId = -1; - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableScanOperator"; - pOperator->operatorType = OP_TableScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doTableScan; - - return pOperator; -} - -SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - - pInfo->pTsdbReadHandle = pTsdbQueryHandle; - pInfo->times = 1; - pInfo->reverseTimes = 0; - pInfo->order = pRuntimeEnv->pQueryAttr->order.order; - pInfo->current = 0; - pInfo->prevGroupId = -1; - pRuntimeEnv->enableGroupData = true; - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableSeqScanOperator"; - pOperator->operatorType = OP_TableSeqScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doTableScanImpl; - - return pOperator; -} - -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - - pInfo->pTsdbReadHandle = pTsdbQueryHandle; - pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - - SColumnInfoData infoData = {{0}}; - infoData.info.type = TSDB_DATA_TYPE_BINARY; - infoData.info.bytes = 1024; - infoData.info.colId = 0; - taosArrayPush(pInfo->block.pDataBlock, &infoData); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableBlockInfoScanOperator"; - pOperator->operatorType = OP_TableBlockInfoScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->exec = doBlockInfoScan; - - return pOperator; -} - -void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream) { - assert(pTableScanInfo != NULL && pDownstream != NULL); - - pTableScanInfo->pExpr = pDownstream->pExpr; // TODO refactor to use colId instead of pExpr - pTableScanInfo->numOfOutput = pDownstream->numOfOutput; - - if (pDownstream->operatorType == OP_Aggregate || pDownstream->operatorType == OP_MultiTableAggregate) { - SAggOperatorInfo* pAggInfo = pDownstream->info; - - pTableScanInfo->pCtx = pAggInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pAggInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; - } else if (pDownstream->operatorType == OP_TimeWindow || pDownstream->operatorType == OP_AllTimeWindow) { - STableIntervalOperatorInfo *pIntervalInfo = pDownstream->info; - - pTableScanInfo->pCtx = pIntervalInfo->pCtx; - pTableScanInfo->pResultRowInfo = &pIntervalInfo->resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pIntervalInfo->rowCellInfoOffset; - - } else if (pDownstream->operatorType == OP_Groupby) { - SGroupbyOperatorInfo *pGroupbyInfo = pDownstream->info; - - pTableScanInfo->pCtx = pGroupbyInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pGroupbyInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pGroupbyInfo->binfo.rowCellInfoOffset; - - } else if (pDownstream->operatorType == OP_MultiTableTimeInterval || pDownstream->operatorType == OP_AllMultiTableTimeInterval) { - STableIntervalOperatorInfo *pInfo = pDownstream->info; - - pTableScanInfo->pCtx = pInfo->pCtx; - pTableScanInfo->pResultRowInfo = &pInfo->resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pInfo->rowCellInfoOffset; - - } else if (pDownstream->operatorType == OP_Project) { - SProjectOperatorInfo *pInfo = pDownstream->info; - - pTableScanInfo->pCtx = pInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; - } else if (pDownstream->operatorType == OP_SessionWindow) { - SSWindowOperatorInfo* pInfo = pDownstream->info; - - pTableScanInfo->pCtx = pInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; - } else if (pDownstream->operatorType == OP_StateWindow) { - SStateWindowOperatorInfo* pInfo = pDownstream->info; - - pTableScanInfo->pCtx = pInfo->binfo.pCtx; - pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; - pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; - } else { - assert(0); - } -} - -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) { - assert(repeatTime > 0); - - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pTsdbReadHandle = pTsdbQueryHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = reverseTime; - pInfo->current = 0; - pInfo->order = pRuntimeEnv->pQueryAttr->order.order; - - SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo)); - pOptr->name = "TableScanOperator"; - pOptr->operatorType = OP_DataBlocksOptScan; - pOptr->pRuntimeEnv = pRuntimeEnv; - pOptr->blockingOptr = false; - pOptr->info = pInfo; - pOptr->exec = doTableScan; - - return pOptr; -} - -SArray* getOrderCheckColumns(SQueryAttr* pQuery) { - int32_t numOfCols = pQuery->pGroupbyExpr == NULL? 0: pQuery->pGroupbyExpr->numOfGroupCols; - - SArray* pOrderColumns = NULL; - if (numOfCols > 0) { - pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo); - } else { - pOrderColumns = taosArrayInit(4, sizeof(SColIndex)); - } - - if (pQuery->interval.interval > 0) { - if (pOrderColumns == NULL) { - pOrderColumns = taosArrayInit(1, sizeof(SColIndex)); - } - - SColIndex colIndex = {.colIndex = 0, .colId = 0, .flag = TSDB_COL_NORMAL}; - taosArrayPush(pOrderColumns, &colIndex); - } - - { - numOfCols = (int32_t) taosArrayGetSize(pOrderColumns); - for(int32_t i = 0; i < numOfCols; ++i) { - SColIndex* index = taosArrayGet(pOrderColumns, i); - for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SSqlExpr* pExpr = &pQuery->pExpr1[j].base; - int32_t functionId = pExpr->functionId; - - if (index->colId == pExpr->colInfo.colId && - (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS)) { - index->colIndex = j; - index->colId = pExpr->resColId; - } - } - } - } - - return pOrderColumns; -} - -SArray* getResultGroupCheckColumns(SQueryAttr* pQuery) { - int32_t numOfCols = pQuery->pGroupbyExpr == NULL? 0 : pQuery->pGroupbyExpr->numOfGroupCols; - - SArray* pOrderColumns = NULL; - if (numOfCols > 0) { - pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo); - } else { - pOrderColumns = taosArrayInit(4, sizeof(SColIndex)); - } - - for(int32_t i = 0; i < numOfCols; ++i) { - SColIndex* index = taosArrayGet(pOrderColumns, i); - - bool found = false; - for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SSqlExpr* pExpr = &pQuery->pExpr1[j].base; - - // TSDB_FUNC_TAG_DUMMY function needs to be ignored - if (index->colId == pExpr->colInfo.colId && - ((TSDB_COL_IS_TAG(pExpr->colInfo.flag) && pExpr->functionId == TSDB_FUNC_TAG) || - (TSDB_COL_IS_NORMAL_COL(pExpr->colInfo.flag) && pExpr->functionId == TSDB_FUNC_PRJ))) { - index->colIndex = j; - index->colId = pExpr->resColId; - found = true; - break; - } - } - - assert(found && index->colIndex >= 0 && index->colIndex < pQuery->numOfOutput); - } - - return pOrderColumns; -} - -static void destroyGlobalAggOperatorInfo(void* param, int32_t numOfOutput) { - SMultiwayMergeInfo *pInfo = (SMultiwayMergeInfo*) param; - destroyBasicOperatorInfo(&pInfo->binfo, numOfOutput); - - taosArrayDestroy(pInfo->orderColumnList); - taosArrayDestroy(pInfo->groupColumnList); - tfree(pInfo->prevRow); - tfree(pInfo->currentGroupColData); -} -static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { - SSLimitOperatorInfo *pInfo = (SSLimitOperatorInfo*) param; - taosArrayDestroy(pInfo->orderColumnList); - pInfo->pRes = blockDataDestroy(pInfo->pRes); - tfree(pInfo->prevRow); -} - -SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, - SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp) { - SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); - - pInfo->resultRowFactor = - (int32_t)(getRowNumForMultioutput(pRuntimeEnv->pQueryAttr, pRuntimeEnv->pQueryAttr->topBotQuery, false)); - - pRuntimeEnv->scanFlag = MERGE_STAGE; // TODO init when creating pCtx - - pInfo->multiGroupResults = groupResultMixedUp; - pInfo->pMerge = param; - pInfo->bufCapacity = 4096; - pInfo->udfInfo = pUdfInfo; - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity * pInfo->resultRowFactor); - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->orderColumnList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); - pInfo->groupColumnList = getResultGroupCheckColumns(pRuntimeEnv->pQueryAttr); - - // TODO refactor - int32_t len = 0; - for(int32_t i = 0; i < numOfOutput; ++i) { - len += pExpr[i].base.colBytes; - } - - int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; - pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); - int32_t offset = POINTER_BYTES * numOfCols; - - for(int32_t i = 0; i < numOfCols; ++i) { - pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; - - SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); - offset += pExpr[index->colIndex].base.resBytes; - } - - numOfCols = (pInfo->groupColumnList != NULL)? (int32_t)taosArrayGetSize(pInfo->groupColumnList):0; - pInfo->currentGroupColData = calloc(1, (POINTER_BYTES * numOfCols + len)); - offset = POINTER_BYTES * numOfCols; - - for(int32_t i = 0; i < numOfCols; ++i) { - pInfo->currentGroupColData[i] = (char*)pInfo->currentGroupColData + offset; - - SColIndex* index = taosArrayGet(pInfo->groupColumnList, i); - offset += pExpr[index->colIndex].base.resBytes; - } - - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - pInfo->seed = rand(); - setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MERGE_STAGE); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "GlobalAggregate"; - pOperator->operatorType = OP_GlobalAggregate; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; - - pOperator->exec = doGlobalAggregate; - pOperator->cleanup = destroyGlobalAggOperatorInfo; - appendUpstream(pOperator, upstream); - - return pOperator; -} - -SOperatorInfo *createMultiwaySortOperatorInfo(SQueryRuntimeEnv *pRuntimeEnv, SExprInfo *pExpr, int32_t numOfOutput, - int32_t numOfRows, void *merger) { - SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); - - pInfo->pMerge = merger; - pInfo->bufCapacity = numOfRows; - pInfo->orderColumnList = getResultGroupCheckColumns(pRuntimeEnv->pQueryAttr); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); - - { // todo extract method to create prev compare buffer - int32_t len = 0; - for(int32_t i = 0; i < numOfOutput; ++i) { - len += pExpr[i].base.colBytes; - } - - int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; - pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); - - int32_t offset = POINTER_BYTES * numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { - pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; - - SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); - offset += pExpr[index->colIndex].base.colBytes; - } - } - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "MultiwaySortOperator"; - pOperator->operatorType = OP_MultiwayMergeSort; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->numOfOutput = numOfOutput; - pOperator->pExpr = pExpr; - pOperator->exec = doMultiwayMergeSort; - pOperator->cleanup = destroyGlobalAggOperatorInfo; - return pOperator; -} - -static int32_t doMergeSDatablock(SSDataBlock* pDest, SSDataBlock* pSrc) { - assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols); - - int32_t numOfCols = pSrc->info.numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); - SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); - - int32_t newSize = (pDest->info.rows + pSrc->info.rows) * pCol2->info.bytes; - char* tmp = realloc(pCol2->pData, newSize); - if (tmp != NULL) { - pCol2->pData = tmp; - int32_t offset = pCol2->info.bytes * pDest->info.rows; - memcpy(pCol2->pData + offset, pCol1->pData, pSrc->info.rows * pCol2->info.bytes); - } else { - return TSDB_CODE_VND_OUT_OF_MEMORY; - } - } - - pDest->info.rows += pSrc->info.rows; - - return TSDB_CODE_SUCCESS; -} - -static SSDataBlock* doSort(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SSortOperatorInfo* pInfo = pOperator->info; - - SSDataBlock* pBlock = NULL; - while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - // start to flush data into disk and try do multiway merge sort - if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); - break; - } - - int32_t code = doMergeSDatablock(pInfo->pDataBlock, pBlock); - if (code != TSDB_CODE_SUCCESS) { - // todo handle error - } - } - - int32_t numOfCols = pInfo->pDataBlock->info.numOfCols; - void** pCols = calloc(numOfCols, POINTER_BYTES); - SSchema* pSchema = calloc(numOfCols, sizeof(SSchema)); - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* p1 = taosArrayGet(pInfo->pDataBlock->pDataBlock, i); - pCols[i] = p1->pData; - pSchema[i].colId = p1->info.colId; - pSchema[i].bytes = p1->info.bytes; - pSchema[i].type = (uint8_t) p1->info.type; - } - - __compar_fn_t comp = getKeyComparFunc(pSchema[pInfo->colIndex].type, pInfo->order); - taoscQSort(pCols, pSchema, numOfCols, pInfo->pDataBlock->info.rows, pInfo->colIndex, comp); - - tfree(pCols); - tfree(pSchema); - return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; -} - -SOperatorInfo *createSortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal) { - SSortOperatorInfo* pInfo = calloc(1, sizeof(SSortOperatorInfo)); - - { - SSDataBlock* pDataBlock = calloc(1, sizeof(SSDataBlock)); - pDataBlock->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); - for(int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData col = {{0}}; - col.info.colId = pExpr[i].base.colInfo.colId; - col.info.bytes = pExpr[i].base.colBytes; - col.info.type = pExpr[i].base.colType; - taosArrayPush(pDataBlock->pDataBlock, &col); - - if (col.info.colId == pOrderVal->orderColId) { - pInfo->colIndex = i; - } - } - - pDataBlock->info.numOfCols = numOfOutput; - pInfo->order = pOrderVal->order; - pInfo->pDataBlock = pDataBlock; - } - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "InMemoryOrder"; - pOperator->operatorType = OP_Order; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->exec = doSort; - pOperator->cleanup = destroyOrderOperatorInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -static int32_t getTableScanOrder(STableScanInfo* pTableScanInfo) { - return pTableScanInfo->order; -} - -// this is a blocking operator -static SSDataBlock* doAggregate(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SAggOperatorInfo* pAggInfo = pOperator->info; - SOptrBasicInfo* pInfo = &pAggInfo->binfo; - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - if (pRuntimeEnv->current != NULL) { - setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - } - - if (upstream->operatorType == OP_DataBlocksOptScan) { - STableScanInfo* pScanInfo = upstream->info; - order = getTableScanOrder(pScanInfo); - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - doAggregateImpl(pOperator, pQueryAttr->window.skey, pInfo->pCtx, pBlock); - } - - doSetOperatorCompleted(pOperator); - - finalizeQueryResult(pOperator, pInfo->pCtx, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset); - pInfo->pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); - - return pInfo->pRes; -} - -static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SAggOperatorInfo* pAggInfo = pOperator->info; - SOptrBasicInfo* pInfo = &pAggInfo->binfo; - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); - - if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pInfo->pRes; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - - if (upstream->operatorType == OP_DataBlocksOptScan) { - STableScanInfo* pScanInfo = upstream->info; - order = getTableScanOrder(pScanInfo); - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - - TSKEY key = 0; - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - key = pBlock->info.window.ekey; - TSKEY_MAX_ADD(key, 1); - } else { - key = pBlock->info.window.skey; - TSKEY_MIN_SUB(key, -1); - } - - setExecutionContext(pRuntimeEnv, pInfo, pOperator->numOfOutput, pRuntimeEnv->current->groupIndex, key); - doAggregateImpl(pOperator, pQueryAttr->window.skey, pInfo->pCtx, pBlock); - } - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pInfo->resultRowInfo); - - updateNumOfRowsInResultRows(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput, &pInfo->resultRowInfo, - pInfo->rowCellInfoOffset); - - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->resultRowInfo); - - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); - if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pInfo->pRes; -} - -static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - - SProjectOperatorInfo* pProjectInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - SOptrBasicInfo *pInfo = &pProjectInfo->binfo; - - SSDataBlock* pRes = pInfo->pRes; - int32_t order = pRuntimeEnv->pQueryAttr->order.order; - - pRes->info.rows = 0; - - if (pProjectInfo->existDataBlock) { // TODO refactor - STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - - SSDataBlock* pBlock = pProjectInfo->existDataBlock; - pProjectInfo->existDataBlock = NULL; - *newgroup = true; - - // todo dynamic set tags - if (pTableQueryInfo != NULL) { - setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); - - projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); - if (pTableQueryInfo != NULL) { - updateTableIdInfo(pTableQueryInfo, pBlock, pRuntimeEnv->pTableRetrieveTsMap, order); - } - - pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); - if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) { - copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); - clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); - return pRes; - } - } - - while(1) { - bool prevVal = *newgroup; - - // The upstream exec may change the value of the newgroup, so use a local variable instead. - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - assert(*newgroup == false); - - *newgroup = prevVal; - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - break; - } - - // Return result of the previous group in the firstly. - if (*newgroup) { - if (pRes->info.rows > 0) { - pProjectInfo->existDataBlock = pBlock; - break; - } else { // init output buffer for a new group data - for (int32_t j = 0; j < pOperator->numOfOutput; ++j) { - aAggs[pInfo->pCtx[j].functionId].xFinalize(&pInfo->pCtx[j]); - } - initCtxOutputBuffer(pInfo->pCtx, pOperator->numOfOutput); - } - } - - STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - - // todo dynamic set tags - if (pTableQueryInfo != NULL) { - setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); - - projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); - if (pTableQueryInfo != NULL) { - updateTableIdInfo(pTableQueryInfo, pBlock, pRuntimeEnv->pTableRetrieveTsMap, order); - } - - pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); - if (pRes->info.rows >= 1000/*pRuntimeEnv->resultInfo.threshold*/) { - break; - } - } - copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); - clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); - return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; -} - -static SSDataBlock* doLimit(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*)param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SLimitOperatorInfo* pInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - SSDataBlock* pBlock = NULL; - while (1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); - return NULL; - } - - if (pRuntimeEnv->currentOffset == 0) { - break; - } else if (pRuntimeEnv->currentOffset >= pBlock->info.rows) { - pRuntimeEnv->currentOffset -= pBlock->info.rows; - } else { - int32_t remain = (int32_t)(pBlock->info.rows - pRuntimeEnv->currentOffset); - pBlock->info.rows = remain; - - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - - int16_t bytes = pColInfoData->info.bytes; - memmove(pColInfoData->pData, pColInfoData->pData + bytes * pRuntimeEnv->currentOffset, remain * bytes); - } - - pRuntimeEnv->currentOffset = 0; - break; - } - } - - if (pInfo->total + pBlock->info.rows >= pInfo->limit) { - pBlock->info.rows = (int32_t)(pInfo->limit - pInfo->total); - pInfo->total = pInfo->limit; - - doSetOperatorCompleted(pOperator); - } else { - pInfo->total += pBlock->info.rows; - } - - return pBlock; -} - -static SSDataBlock* doFilter(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo *)param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SFilterOperatorInfo* pCondInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - while (1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock *pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - doSetFilterColumnInfo(pCondInfo->pFilterInfo, pCondInfo->numOfFilterCols, pBlock); - assert(pRuntimeEnv->pTsBuf == NULL); - filterRowsInDataBlock(pRuntimeEnv, pCondInfo->pFilterInfo, pCondInfo->numOfFilterCols, pBlock, true); - - if (pBlock->info.rows > 0) { - return pBlock; - } - } - - doSetOperatorCompleted(pOperator); - return NULL; -} - -static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pIntervalInfo->pRes; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - STimeWindow win = pQueryAttr->window; - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); - hashIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); - } - - // restore the value - pQueryAttr->order.order = order; - pQueryAttr->window = win; - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pIntervalInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); - - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; -} - -static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pIntervalInfo->pRes; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - STimeWindow win = pQueryAttr->window; - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); - hashAllIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); - } - - // restore the value - pQueryAttr->order.order = order; - pQueryAttr->window = win; - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pIntervalInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); - - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; -} - -static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - if (pOperator->status == OP_RES_TO_RETURN) { - int64_t st = taosGetTimestampUs(); - - copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - SQInfo* pQInfo = pRuntimeEnv->qinfo; - pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); - - return pIntervalInfo->pRes; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - - setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); - setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); - - hashIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); - } - - pOperator->status = OP_RES_TO_RETURN; - pQueryAttr->order.order = order; // TODO : restore the order - doCloseAllTimeWindow(pRuntimeEnv); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - - copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pIntervalInfo->pRes; -} - -static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - if (pOperator->status == OP_RES_TO_RETURN) { - copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pIntervalInfo->pRes; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - - setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); - setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); - - hashAllIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); - } - - pOperator->status = OP_RES_TO_RETURN; - pQueryAttr->order.order = order; // TODO : restore the order - doCloseAllTimeWindow(pRuntimeEnv); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - - int64_t st = taosGetTimestampUs(); - copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - SQInfo* pQInfo = pRuntimeEnv->qinfo; - pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); - - return pIntervalInfo->pRes; -} - -static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - STableQueryInfo* item = pRuntimeEnv->current; - SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); - - SOptrBasicInfo* pBInfo = &pInfo->binfo; - - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - int16_t bytes = pColInfoData->info.bytes; - int16_t type = pColInfoData->info.type; - - SColumnInfoData* pTsColInfoData = taosArrayGet(pSDataBlock->pDataBlock, 0); - TSKEY* tsList = (TSKEY*)pTsColInfoData->pData; - if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) { - pInfo->reptScan = true; - tfree(pInfo->prevData); - } - - pInfo->numOfRows = 0; - for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { - char* val = ((char*)pColInfoData->pData) + bytes * j; - if (isNull(val, type)) { - continue; - } - if (pInfo->prevData == NULL) { - pInfo->prevData = malloc(bytes); - memcpy(pInfo->prevData, val, bytes); - pInfo->numOfRows = 1; - pInfo->curWindow.skey = tsList[j]; - pInfo->curWindow.ekey = tsList[j]; - pInfo->start = j; - - } else if (memcmp(pInfo->prevData, val, bytes) == 0) { - pInfo->curWindow.ekey = tsList[j]; - pInfo->numOfRows += 1; - //pInfo->start = j; - if (j == 0 && pInfo->start != 0) { - pInfo->numOfRows = 1; - pInfo->start = 0; - } - } else { - SResultRow* pResult = NULL; - pInfo->curWindow.ekey = pInfo->curWindow.skey; - int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.tid, &pInfo->curWindow, masterScan, - &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, - pBInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); - - pInfo->curWindow.skey = tsList[j]; - pInfo->curWindow.ekey = tsList[j]; - memcpy(pInfo->prevData, val, bytes); - pInfo->numOfRows = 1; - pInfo->start = j; - - } - } - - SResultRow* pResult = NULL; - - pInfo->curWindow.ekey = pInfo->curWindow.skey; - int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.tid, &pInfo->curWindow, masterScan, - &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, - pBInfo->rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); -} - -static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SStateWindowOperatorInfo* pWindowInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); - - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pBInfo->pRes; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - STimeWindow win = pQueryAttr->window; - SOperatorInfo* upstream = pOperator->upstream[0]; - while (1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQueryAttr->order.order); - if (pWindowInfo->colIndex == -1) { - pWindowInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock); - } - doStateWindowAggImpl(pOperator, pWindowInfo, pBlock); - } - - // restore the value - pQueryAttr->order.order = order; - pQueryAttr->window = win; - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pBInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); - - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); - - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes; -} - -static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SSWindowOperatorInfo* pWindowInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; - - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); - - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pBInfo->pRes; - } - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - //pQueryAttr->order.order = TSDB_ORDER_ASC; - int32_t order = pQueryAttr->order.order; - STimeWindow win = pQueryAttr->window; - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQueryAttr->order.order); - doSessionWindowAggImpl(pOperator, pWindowInfo, pBlock); - } - - // restore the value - pQueryAttr->order.order = order; - pQueryAttr->window = win; - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pBInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); - - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); - - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes; -} - -static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SGroupbyOperatorInfo *pInfo = pOperator->info; - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); - - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pInfo->binfo.pRes; - } - - SOperatorInfo* upstream = pOperator->upstream[0]; - - while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pRuntimeEnv->pQueryAttr->order.order); - setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); - if (pInfo->colIndex == -1) { - pInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock); - } - - doHashGroupbyAgg(pOperator, pInfo, pBlock); - } - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pInfo->binfo.resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - - if (!pRuntimeEnv->pQueryAttr->stableQuery) { // finalize include the update of result rows - finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - } else { - updateNumOfRowsInResultRows(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - } - - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->binfo.resultRowInfo); - if (!pRuntimeEnv->pQueryAttr->stableQuery) { - sortGroupResByOrderList(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); - } - - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); - - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pInfo->binfo.pRes; -} - -static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, SQueryRuntimeEnv* pRuntimeEnv, bool* newgroup) { - pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; - int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; - taosResetFillInfo(pInfo->pFillInfo, pInfo->pFillInfo->start); - - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); - taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->existNewGroupBlock); - - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity, pInfo->p); - pInfo->existNewGroupBlock = NULL; - *newgroup = true; -} - -static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo *pInfo, SQueryRuntimeEnv *pRuntimeEnv, bool *newgroup) { - if (taosFillHasMoreResults(pInfo->pFillInfo)) { - *newgroup = false; - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pRuntimeEnv->resultInfo.capacity, pInfo->p); - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult)) { - return; - } - } - - // handle the cached new group data block - if (pInfo->existNewGroupBlock) { - doHandleRemainBlockForNewGroupImpl(pInfo, pRuntimeEnv, newgroup); - } -} - -static SSDataBlock* doFill(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - - SFillOperatorInfo *pInfo = pOperator->info; - pInfo->pRes->info.rows = 0; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup); - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult && pInfo->pRes->info.rows > 0)) { - return pInfo->pRes; - } - - while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (*newgroup) { - assert(pBlock != NULL); - } - - if (*newgroup && pInfo->totalInputRows > 0) { // there are already processed current group data block - pInfo->existNewGroupBlock = pBlock; - *newgroup = false; - - // Fill the previous group data block, before handle the data block of new group. - // Close the fill operation for previous group data block - taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQueryAttr->window.ekey); - } else { - if (pBlock == NULL) { - if (pInfo->totalInputRows == 0) { - pOperator->status = OP_EXEC_DONE; - return NULL; - } - - taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQueryAttr->window.ekey); - } else { - pInfo->totalInputRows += pBlock->info.rows; - taosFillSetStartInfo(pInfo->pFillInfo, pBlock->info.rows, pBlock->info.window.ekey); - taosFillSetInputDataBlock(pInfo->pFillInfo, pBlock); - } - } - - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity, pInfo->p); - - // current group has no more result to return - if (pInfo->pRes->info.rows > 0) { - // 1. The result in current group not reach the threshold of output result, continue - // 2. If multiple group results existing in one SSDataBlock is not allowed, return immediately - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || pBlock == NULL || (!pInfo->multigroupResult)) { - return pInfo->pRes; - } - - doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup); - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || pBlock == NULL) { - return pInfo->pRes; - } - } else if (pInfo->existNewGroupBlock) { // try next group - assert(pBlock != NULL); - doHandleRemainBlockForNewGroupImpl(pInfo, pRuntimeEnv, newgroup); - - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold) { - return pInfo->pRes; - } - } else { - return NULL; - } - } -} - -// todo set the attribute of query scan count -static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr) { - for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = pQueryAttr->pExpr1[i].base.functionId; - if (functionId == TSDB_FUNC_STDDEV || functionId == TSDB_FUNC_PERCT) { - return 2; - } - } - - return 1; -} - -static void destroyOperatorInfo(SOperatorInfo* pOperator) { - if (pOperator == NULL) { - return; - } - - if (pOperator->cleanup != NULL) { - pOperator->cleanup(pOperator->info, pOperator->numOfOutput); - } - - if (pOperator->upstream != NULL) { - for(int32_t i = 0; i < pOperator->numOfUpstream; ++i) { - destroyOperatorInfo(pOperator->upstream[i]); - } - - tfree(pOperator->upstream); - pOperator->numOfUpstream = 0; - } - - tfree(pOperator->info); - tfree(pOperator); -} - -SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t numOfRows = (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); - - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - pInfo->seed = rand(); - setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MASTER_SCAN); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableAggregate"; - pOperator->operatorType = OP_Aggregate; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; - - pOperator->exec = doAggregate; - pOperator->cleanup = destroyAggOperatorInfo; - appendUpstream(pOperator, upstream); - - return pOperator; -} - -static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { - assert(pInfo != NULL); - - destroySQLFunctionCtx(pInfo->pCtx, numOfOutput); - tfree(pInfo->rowCellInfoOffset); - - cleanupResultRowInfo(&pInfo->resultRowInfo); - pInfo->pRes = blockDataDestroy(pInfo->pRes); -} - -static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { - SOptrBasicInfo* pInfo = (SOptrBasicInfo*) param; - doDestroyBasicInfo(pInfo, numOfOutput); -} -static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { - SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*) param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - tfree(pInfo->prevData); -} -static void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { - SAggOperatorInfo* pInfo = (SAggOperatorInfo*) param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); -} -static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { - SSWindowOperatorInfo* pInfo = (SSWindowOperatorInfo*) param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); -} - -static void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { - SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param; - pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); - pInfo->pRes = blockDataDestroy(pInfo->pRes); - tfree(pInfo->p); -} - -static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { - SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*) param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - tfree(pInfo->prevData); -} - -static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { - SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*) param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); -} - -static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { - STagScanInfo* pInfo = (STagScanInfo*) param; - pInfo->pRes = blockDataDestroy(pInfo->pRes); -} - -static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { - SSortOperatorInfo* pInfo = (SSortOperatorInfo*) param; - pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock); -} - -static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) { - SFilterOperatorInfo* pInfo = (SFilterOperatorInfo*) param; - doDestroyFilterInfo(pInfo->pFilterInfo, pInfo->numOfFilterCols); -} - -static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { - SDistinctOperatorInfo* pInfo = (SDistinctOperatorInfo*) param; - taosHashCleanup(pInfo->pSet); - tfree(pInfo->buf); - taosArrayDestroy(pInfo->pDistinctDataInfo); - pInfo->pRes = blockDataDestroy(pInfo->pRes); -} - -SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - - size_t tableGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); - - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) tableGroup); - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup, TSDB_DATA_TYPE_INT); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "MultiTableAggregate"; - pOperator->operatorType = OP_MultiTableAggregate; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; - - pOperator->exec = doSTableAggregate; - pOperator->cleanup = destroyAggOperatorInfo; - appendUpstream(pOperator, upstream); - - return pOperator; -} - -SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo)); - - pInfo->seed = rand(); - pInfo->bufCapacity = pRuntimeEnv->resultInfo.capacity; - - SOptrBasicInfo* pBInfo = &pInfo->binfo; - pBInfo->pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity); - pBInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset); - - initResultRowInfo(&pBInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); - setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MASTER_SCAN); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "ProjectOperator"; - pOperator->operatorType = OP_Project; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; - - pOperator->exec = doProjectOperation; - pOperator->cleanup = destroyProjectOperatorInfo; - appendUpstream(pOperator, upstream); - - return pOperator; -} - -SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols) { - SColumnInfo* pCols = calloc(numOfOutput, sizeof(SColumnInfo)); - - int32_t numOfFilter = 0; - for(int32_t i = 0; i < numOfOutput; ++i) { - if (pExpr[i].base.flist.numOfFilters > 0) { - numOfFilter += 1; - } - - pCols[i].type = pExpr[i].base.resType; - pCols[i].bytes = pExpr[i].base.resBytes; - pCols[i].colId = pExpr[i].base.resColId; - - pCols[i].flist.numOfFilters = pExpr[i].base.flist.numOfFilters; - if (pCols[i].flist.numOfFilters != 0) { - pCols[i].flist.filterInfo = calloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo)); - memcpy(pCols[i].flist.filterInfo, pExpr[i].base.flist.filterInfo, pCols[i].flist.numOfFilters * sizeof(SColumnFilterInfo)); - } else { - // avoid runtime error - pCols[i].flist.filterInfo = NULL; - } - } - - assert(numOfFilter > 0); - - *numOfFilterCols = numOfFilter; - return pCols; -} - -SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, - int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter) { - SFilterOperatorInfo* pInfo = calloc(1, sizeof(SFilterOperatorInfo)); - - assert(numOfFilter > 0 && pCols != NULL); - doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); - pInfo->numOfFilterCols = numOfFilter; - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - - pOperator->name = "FilterOperator"; - pOperator->operatorType = OP_Filter; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->numOfOutput = numOfOutput; - pOperator->pExpr = pExpr; - pOperator->exec = doFilter; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroyConditionOperatorInfo; - appendUpstream(pOperator, upstream); - - return pOperator; -} - -SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream) { - SLimitOperatorInfo* pInfo = calloc(1, sizeof(SLimitOperatorInfo)); - pInfo->limit = pRuntimeEnv->pQueryAttr->limit.limit; - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - - pOperator->name = "LimitOperator"; - pOperator->operatorType = OP_Limit; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->exec = doLimit; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - appendUpstream(pOperator, upstream); - - return pOperator; -} - -SOperatorInfo* createIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - - pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - - pOperator->name = "TimeIntervalAggOperator"; - pOperator->operatorType = OP_TimeWindow; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - - -SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - - pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - - pOperator->name = "AllTimeIntervalAggOperator"; - pOperator->operatorType = OP_AllTimeWindow; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doAllIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SStateWindowOperatorInfo* pInfo = calloc(1, sizeof(SStateWindowOperatorInfo)); - pInfo->colIndex = -1; - pInfo->reptScan = false; - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "StateWindowOperator"; - pOperator->operatorType = OP_StateWindow; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doStateWindowAgg; - pOperator->cleanup = destroyStateWindowOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} -SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo)); - - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - pInfo->prevTs = INT64_MIN; - pInfo->reptScan = false; - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - - pOperator->name = "SessionWindowAggOperator"; - pOperator->operatorType = OP_SessionWindow; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doSessionWindowAgg; - pOperator->cleanup = destroySWindowOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - - pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "MultiTableTimeIntervalOperator"; - pOperator->operatorType = OP_MultiTableTimeInterval; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - - pOperator->exec = doSTableIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - - pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "AllMultiTableTimeIntervalOperator"; - pOperator->operatorType = OP_AllMultiTableTimeInterval; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - - pOperator->exec = doAllSTableIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; - - appendUpstream(pOperator, upstream); - - return pOperator; -} - - -SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); - pInfo->colIndex = -1; // group by column index - - - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - pQueryAttr->resultRowSize = (pQueryAttr->resultRowSize * - (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery))); - - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "GroupbyAggOperator"; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->operatorType = OP_Groupby; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = hashGroupbyAggregate; - pOperator->cleanup = destroyGroupbyOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult) { - SFillOperatorInfo* pInfo = calloc(1, sizeof(SFillOperatorInfo)); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - pInfo->multigroupResult = multigroupResult; - - { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal); - STimeWindow w = TSWINDOW_INITIALIZER; - - TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); - getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); - - pInfo->pFillInfo = - taosCreateFillInfo(pQueryAttr->order.order, w.skey, 0, (int32_t)pRuntimeEnv->resultInfo.capacity, numOfOutput, - pQueryAttr->interval.sliding, pQueryAttr->interval.slidingUnit, - (int8_t)pQueryAttr->precision, pQueryAttr->fillType, pColInfo, pRuntimeEnv->qinfo); - - pInfo->p = calloc(pInfo->pFillInfo->numOfCols, POINTER_BYTES); - } - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - - pOperator->name = "FillOperator"; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->operatorType = OP_Fill; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doFill; - pOperator->cleanup = destroySFillOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* pMerger, bool multigroupResult) { - SSLimitOperatorInfo* pInfo = calloc(1, sizeof(SSLimitOperatorInfo)); - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - pInfo->orderColumnList = getResultGroupCheckColumns(pQueryAttr); - pInfo->slimit = pQueryAttr->slimit; - pInfo->limit = pQueryAttr->limit; - pInfo->capacity = pRuntimeEnv->resultInfo.capacity; - pInfo->threshold = (int64_t)(pInfo->capacity * 0.8); - pInfo->currentOffset = pQueryAttr->limit.offset; - pInfo->currentGroupOffset = pQueryAttr->slimit.offset; - pInfo->multigroupResult= multigroupResult; - - // TODO refactor - int32_t len = 0; - for(int32_t i = 0; i < numOfOutput; ++i) { - len += pExpr[i].base.resBytes; - } - - int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; - pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); - - int32_t offset = POINTER_BYTES * numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { - pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; - - SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); - offset += pExpr[index->colIndex].base.resBytes; - } - - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - - pOperator->name = "SLimitOperator"; - pOperator->operatorType = OP_SLimit; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->exec = doSLimit; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroySlimitOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -static SSDataBlock* doTagScan(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - int32_t maxNumOfTables = (int32_t)pRuntimeEnv->resultInfo.capacity; - - STagScanInfo *pInfo = pOperator->info; - SSDataBlock *pRes = pInfo->pRes; - *newgroup = false; - - int32_t count = 0; - SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); - - int32_t functionId = pOperator->pExpr[0].base.functionId; - if (functionId == TSDB_FUNC_TID_TAG) { // return the tags & table Id - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - assert(pQueryAttr->numOfOutput == 1); - - SExprInfo* pExprInfo = &pOperator->pExpr[0]; - int32_t rsize = pExprInfo->base.resBytes; - - count = 0; - - int16_t bytes = pExprInfo->base.resBytes; - int16_t type = pExprInfo->base.resType; - - for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { - if (pQueryAttr->tagColList[i].colId == pExprInfo->base.colInfo.colId) { - bytes = pQueryAttr->tagColList[i].bytes; - type = pQueryAttr->tagColList[i].type; - break; - } - } - - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; - STableQueryInfo *item = taosArrayGetP(pa, i); - - char *output = pColInfo->pData + count * rsize; - varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); - - output = varDataVal(output); - STableId* id = TSDB_TABLEID(item->pTable); - - *(int16_t *)output = 0; - output += sizeof(int16_t); - - *(int64_t *)output = id->uid; // memory align problem, todo serialize - output += sizeof(id->uid); - - *(int32_t *)output = id->tid; - output += sizeof(id->tid); - - *(int32_t *)output = pQueryAttr->vgId; - output += sizeof(pQueryAttr->vgId); - - char* data = NULL; - if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); - } - - doSetTagValueToResultBuf(output, data, type, bytes); - count += 1; - } - - qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_QID(pRuntimeEnv), count); - } else if (functionId == TSDB_FUNC_COUNT) {// handle the "count(tbname)" query - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - *(int64_t*)pColInfo->pData = pInfo->totalTables; - count = 1; - - pOperator->status = OP_EXEC_DONE; - qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_QID(pRuntimeEnv), count); - } else { // return only the tags|table name etc. - SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo - - count = 0; - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; - - STableQueryInfo* item = taosArrayGetP(pa, i); - - char *data = NULL, *dst = NULL; - int16_t type = 0, bytes = 0; - for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { - // not assign value in case of user defined constant output column - if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.colInfo.flag)) { - continue; - } - - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); - type = pExprInfo[j].base.resType; - bytes = pExprInfo[j].base.resBytes; - - if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); - } - - dst = pColInfo->pData + count * pExprInfo[j].base.resBytes; - doSetTagValueToResultBuf(dst, data, type, bytes); - } - - count += 1; - } - - if (pInfo->curPos >= pInfo->totalTables) { - pOperator->status = OP_EXEC_DONE; - } - - qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_QID(pRuntimeEnv), count); - } - - if (pOperator->status == OP_EXEC_DONE) { - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); - } - - pRes->info.rows = count; - return (pRes->info.rows == 0)? NULL:pInfo->pRes; -} - -SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { - STagScanInfo* pInfo = calloc(1, sizeof(STagScanInfo)); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - - size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); - assert(numOfGroup == 0 || numOfGroup == 1); - - pInfo->totalTables = pRuntimeEnv->tableqinfoGroupInfo.numOfTables; - pInfo->curPos = 0; - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "SeqTableTagScan"; - pOperator->operatorType = OP_TagScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->exec = doTagScan; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroyTagScanOperatorInfo; - - return pOperator; -} -static bool initMultiDistinctInfo(SDistinctOperatorInfo *pInfo, SOperatorInfo* pOperator, SSDataBlock *pBlock) { - if (taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput) { - // distinct info already inited - return true; - } - for (int i = 0; i < pOperator->numOfOutput; i++) { - pInfo->totalBytes += pOperator->pExpr[i].base.colBytes; - } - for (int i = 0; i < pOperator->numOfOutput; i++) { - int numOfBlock = (int)(taosArrayGetSize(pBlock->pDataBlock)); - assert(i < numOfBlock); - for (int j = 0; j < numOfBlock; j++) { - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, j); - if (pColDataInfo->info.colId == pOperator->pExpr[i].base.resColId) { - SDistinctDataInfo item = {.index = j, .type = pColDataInfo->info.type, .bytes = pColDataInfo->info.bytes}; - taosArrayInsert(pInfo->pDistinctDataInfo, i, &item); - } - } - } - pInfo->totalBytes += (int32_t)strlen(MULTI_KEY_DELIM) * (pOperator->numOfOutput); - pInfo->buf = calloc(1, pInfo->totalBytes); - return taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput ? true : false; -} - -static void buildMultiDistinctKey(SDistinctOperatorInfo *pInfo, SSDataBlock *pBlock, int32_t rowId) { - char *p = pInfo->buf; - memset(p, 0, pInfo->totalBytes); - - for (int i = 0; i < taosArrayGetSize(pInfo->pDistinctDataInfo); i++) { - SDistinctDataInfo* pDistDataInfo = (SDistinctDataInfo *)taosArrayGet(pInfo->pDistinctDataInfo, i); - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); - char *val = ((char *)pColDataInfo->pData) + pColDataInfo->info.bytes * rowId; - if (isNull(val, pDistDataInfo->type)) { - p += pDistDataInfo->bytes; - continue; - } - if (IS_VAR_DATA_TYPE(pDistDataInfo->type)) { - memcpy(p, varDataVal(val), varDataLen(val)); - p += varDataLen(val); - } else { - memcpy(p, val, pDistDataInfo->bytes); - p += pDistDataInfo->bytes; - } - memcpy(p, MULTI_KEY_DELIM, strlen(MULTI_KEY_DELIM)); - p += strlen(MULTI_KEY_DELIM); - } -} - -static SSDataBlock* hashDistinct(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SDistinctOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pRes = pInfo->pRes; - - pRes->info.rows = 0; - SSDataBlock* pBlock = NULL; - - while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); - break; - } - if (!initMultiDistinctInfo(pInfo, pOperator, pBlock)) { - doSetOperatorCompleted(pOperator); - break; - } - // ensure result output buf - if (pRes->info.rows + pBlock->info.rows > pInfo->outputCapacity) { - int32_t newSize = pRes->info.rows + pBlock->info.rows; - for (int i = 0; i < taosArrayGetSize(pRes->pDataBlock); i++) { - SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, i); - SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, i); - char* tmp = realloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes); - if (tmp == NULL) { - return NULL; - } else { - pResultColInfoData->pData = tmp; - } - } - pInfo->outputCapacity = newSize; - } - - for (int32_t i = 0; i < pBlock->info.rows; i++) { - buildMultiDistinctKey(pInfo, pBlock, i); - if (taosHashGet(pInfo->pSet, pInfo->buf, pInfo->totalBytes) == NULL) { - int32_t dummy; - taosHashPut(pInfo->pSet, pInfo->buf, pInfo->totalBytes, &dummy, sizeof(dummy)); - for (int j = 0; j < taosArrayGetSize(pRes->pDataBlock); j++) { - SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, j); // distinct meta info - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); //src - SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, j); // dist - - char* val = ((char*)pColInfoData->pData) + pDistDataInfo->bytes * i; - char *start = pResultColInfoData->pData + pDistDataInfo->bytes * pInfo->pRes->info.rows; - memcpy(start, val, pDistDataInfo->bytes); - } - pRes->info.rows += 1; - } - } - - if (pRes->info.rows >= pInfo->threshold) { - break; - } - } - return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; -} - -SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SDistinctOperatorInfo* pInfo = calloc(1, sizeof(SDistinctOperatorInfo)); - pInfo->totalBytes = 0; - pInfo->buf = NULL; - pInfo->threshold = tsMaxNumOfDistinctResults; // distinct result threshold - pInfo->outputCapacity = 4096; - pInfo->pDistinctDataInfo = taosArrayInit(numOfOutput, sizeof(SDistinctDataInfo)); - pInfo->pSet = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) pInfo->outputCapacity); - - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "DistinctOperator"; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->operatorType = OP_Distinct; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = hashDistinct; - pOperator->pExpr = pExpr; - pOperator->cleanup = destroyDistinctOperatorInfo; - - appendUpstream(pOperator, upstream); - return pOperator; -} - -static int32_t getColumnIndexInSource(SQueriedTableInfo *pTableInfo, SSqlExpr *pExpr, SColumnInfo* pTagCols) { - int32_t j = 0; - - if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { - if (pExpr->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - return TSDB_TBNAME_COLUMN_INDEX; - } - - while(j < pTableInfo->numOfTags) { - if (pExpr->colInfo.colId == pTagCols[j].colId) { - return j; - } - - j += 1; - } - - } else if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) { // user specified column data - return TSDB_UD_COLUMN_INDEX; - } else { - while (j < pTableInfo->numOfCols) { - if (pExpr->colInfo.colId == pTableInfo->colList[j].colId) { - return j; - } - - j += 1; - } - } - - return INT32_MIN; // return a less than TSDB_TBNAME_COLUMN_INDEX value -} - -bool validateExprColumnInfo(SQueriedTableInfo *pTableInfo, SSqlExpr *pExpr, SColumnInfo* pTagCols) { - int32_t j = getColumnIndexInSource(pTableInfo, pExpr, pTagCols); - return j != INT32_MIN; -} - -static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { - if (pQueryMsg->interval.interval < 0) { - qError("qmsg:%p illegal value of interval time %" PRId64, pQueryMsg, pQueryMsg->interval.interval); - return false; - } - - if (pQueryMsg->sw.gap < 0 || pQueryMsg->sw.primaryColId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - qError("qmsg:%p illegal value of session window time %" PRId64, pQueryMsg, pQueryMsg->sw.gap); - return false; - } - - if (pQueryMsg->sw.gap > 0 && pQueryMsg->interval.interval > 0) { - qError("qmsg:%p illegal value of session window time %" PRId64" and interval value %"PRId64, pQueryMsg, - pQueryMsg->sw.gap, pQueryMsg->interval.interval); - return false; - } - - if (pQueryMsg->numOfTables <= 0) { - qError("qmsg:%p illegal value of numOfTables %d", pQueryMsg, pQueryMsg->numOfTables); - return false; - } - - if (pQueryMsg->numOfGroupCols < 0) { - qError("qmsg:%p illegal value of numOfGroupbyCols %d", pQueryMsg, pQueryMsg->numOfGroupCols); - return false; - } - - if (pQueryMsg->numOfOutput > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutput <= 0) { - qError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutput); - return false; - } - - return true; -} - -static bool validateQueryTableCols(SQueriedTableInfo* pTableInfo, SSqlExpr** pExpr, int32_t numOfOutput, - SColumnInfo* pTagCols, void* pMsg) { - int32_t numOfTotal = pTableInfo->numOfCols + pTableInfo->numOfTags; - if (pTableInfo->numOfCols < 0 || pTableInfo->numOfTags < 0 || numOfTotal > TSDB_MAX_COLUMNS) { - qError("qmsg:%p illegal value of numOfCols %d numOfTags:%d", pMsg, pTableInfo->numOfCols, pTableInfo->numOfTags); - return false; - } - - if (numOfTotal == 0) { // table total columns are not required. - for(int32_t i = 0; i < numOfOutput; ++i) { - SSqlExpr* p = pExpr[i]; - if ((p->functionId == TSDB_FUNC_TAGPRJ) || - (p->functionId == TSDB_FUNC_TID_TAG && p->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) || - (p->functionId == TSDB_FUNC_COUNT && p->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) || - (p->functionId == TSDB_FUNC_BLKINFO)) { - continue; - } - - return false; - } - } - - for(int32_t i = 0; i < numOfOutput; ++i) { - if (!validateExprColumnInfo(pTableInfo, pExpr[i], pTagCols)) { - return TSDB_CODE_QRY_INVALID_MSG; - } - } - - return true; -} - -static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **pTableIdList) { - assert(pQueryMsg->numOfTables > 0); - - *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableIdInfo)); - - for (int32_t j = 0; j < pQueryMsg->numOfTables; ++j) { - STableIdInfo* pTableIdInfo = (STableIdInfo *)pMsg; - - pTableIdInfo->tid = htonl(pTableIdInfo->tid); - pTableIdInfo->uid = htobe64(pTableIdInfo->uid); - pTableIdInfo->key = htobe64(pTableIdInfo->key); - - taosArrayPush(*pTableIdList, pTableIdInfo); - pMsg += sizeof(STableIdInfo); - } - - return pMsg; -} - -static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) { - for (int32_t f = 0; f < numOfFilters; ++f) { - SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)(*pMsg); - - SColumnFilterInfo *pColFilter = &pColFilters[f]; - pColFilter->filterstr = htons(pFilterMsg->filterstr); - - (*pMsg) += sizeof(SColumnFilterInfo); - - if (pColFilter->filterstr) { - pColFilter->len = htobe64(pFilterMsg->len); - - pColFilter->pz = (int64_t)calloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator - if (pColFilter->pz == 0) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - memcpy((void *)pColFilter->pz, (*pMsg), (size_t)pColFilter->len); - (*pMsg) += (pColFilter->len + 1); - } else { - pColFilter->lowerBndi = htobe64(pFilterMsg->lowerBndi); - pColFilter->upperBndi = htobe64(pFilterMsg->upperBndi); - } - - pColFilter->lowerRelOptr = htons(pFilterMsg->lowerRelOptr); - pColFilter->upperRelOptr = htons(pFilterMsg->upperRelOptr); - } - - return TSDB_CODE_SUCCESS; -} - -/** - * pQueryMsg->head has been converted before this function is called. - * - * @param pQueryMsg - * @param pTableIdList - * @param pExpr - * @return - */ -int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { - int32_t code = TSDB_CODE_SUCCESS; - - if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { - return TSDB_CODE_QRY_INVALID_MSG; - } - - pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); - pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); - pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); - pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval); - pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding); - pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset); - pQueryMsg->limit = htobe64(pQueryMsg->limit); - pQueryMsg->offset = htobe64(pQueryMsg->offset); - pQueryMsg->vgroupLimit = htobe64(pQueryMsg->vgroupLimit); - - pQueryMsg->order = htons(pQueryMsg->order); - pQueryMsg->orderColId = htons(pQueryMsg->orderColId); - pQueryMsg->queryType = htonl(pQueryMsg->queryType); - pQueryMsg->tagNameRelType = htons(pQueryMsg->tagNameRelType); - - pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); - pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); - pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); - - pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); - pQueryMsg->colCondLen = htons(pQueryMsg->colCondLen); - - pQueryMsg->tsBuf.tsOffset = htonl(pQueryMsg->tsBuf.tsOffset); - pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); - pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); - pQueryMsg->tsBuf.tsOrder = htonl(pQueryMsg->tsBuf.tsOrder); - - pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags); - pQueryMsg->tbnameCondLen = htonl(pQueryMsg->tbnameCondLen); - pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput); - pQueryMsg->sqlstrLen = htonl(pQueryMsg->sqlstrLen); - pQueryMsg->prevResultLen = htonl(pQueryMsg->prevResultLen); - pQueryMsg->sw.gap = htobe64(pQueryMsg->sw.gap); - pQueryMsg->sw.primaryColId = htonl(pQueryMsg->sw.primaryColId); - pQueryMsg->tableScanOperator = htonl(pQueryMsg->tableScanOperator); - pQueryMsg->numOfOperator = htonl(pQueryMsg->numOfOperator); - pQueryMsg->udfContentOffset = htonl(pQueryMsg->udfContentOffset); - pQueryMsg->udfContentLen = htonl(pQueryMsg->udfContentLen); - pQueryMsg->udfNum = htonl(pQueryMsg->udfNum); - - // query msg safety check - if (!validateQueryMsg(pQueryMsg)) { - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - - char *pMsg = (char *)(pQueryMsg->tableCols) + sizeof(SColumnInfo) * pQueryMsg->numOfCols; - for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) { - SColumnInfo *pColInfo = &pQueryMsg->tableCols[col]; - - pColInfo->colId = htons(pColInfo->colId); - pColInfo->type = htons(pColInfo->type); - pColInfo->bytes = htons(pColInfo->bytes); - pColInfo->flist.numOfFilters = 0; - - if (!isValidDataType(pColInfo->type)) { - qDebug("qmsg:%p, invalid data type in source column, index:%d, type:%d", pQueryMsg, col, pColInfo->type); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - -/* - int32_t numOfFilters = pColInfo->flist.numOfFilters; - if (numOfFilters > 0) { - pColInfo->flist.filterInfo = calloc(numOfFilters, sizeof(SColumnFilterInfo)); - if (pColInfo->flist.filterInfo == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - } - - code = deserializeColFilterInfo(pColInfo->flist.filterInfo, numOfFilters, &pMsg); - if (code != TSDB_CODE_SUCCESS) { - goto _cleanup; - } -*/ - } - - if (pQueryMsg->colCondLen > 0) { - param->colCond = calloc(1, pQueryMsg->colCondLen); - if (param->colCond == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - memcpy(param->colCond, pMsg, pQueryMsg->colCondLen); - pMsg += pQueryMsg->colCondLen; - } - - - param->tableScanOperator = pQueryMsg->tableScanOperator; - param->pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); - if (param->pExpr == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - SSqlExpr *pExprMsg = (SSqlExpr *)pMsg; - - for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { - param->pExpr[i] = pExprMsg; - - pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); - pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); - pExprMsg->colBytes = htons(pExprMsg->colBytes); - pExprMsg->colType = htons(pExprMsg->colType); - - pExprMsg->resType = htons(pExprMsg->resType); - pExprMsg->resBytes = htons(pExprMsg->resBytes); - pExprMsg->interBytes = htonl(pExprMsg->interBytes); - - pExprMsg->functionId = htons(pExprMsg->functionId); - pExprMsg->numOfParams = htons(pExprMsg->numOfParams); - pExprMsg->resColId = htons(pExprMsg->resColId); - pExprMsg->flist.numOfFilters = htons(pExprMsg->flist.numOfFilters); - pMsg += sizeof(SSqlExpr); - - for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { - pExprMsg->param[j].nType = htonl(pExprMsg->param[j].nType); - pExprMsg->param[j].nLen = htonl(pExprMsg->param[j].nLen); - - if (pExprMsg->param[j].nType == TSDB_DATA_TYPE_BINARY) { - pExprMsg->param[j].pz = pMsg; - pMsg += pExprMsg->param[j].nLen; // one more for the string terminated char. - } else { - pExprMsg->param[j].i64 = htobe64(pExprMsg->param[j].i64); - } - } - - int16_t functionId = pExprMsg->functionId; - if (functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG_DUMMY) { - if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - } - - if (pExprMsg->flist.numOfFilters > 0) { - pExprMsg->flist.filterInfo = calloc(pExprMsg->flist.numOfFilters, sizeof(SColumnFilterInfo)); - } - - deserializeColFilterInfo(pExprMsg->flist.filterInfo, pExprMsg->flist.numOfFilters, &pMsg); - pExprMsg = (SSqlExpr *)pMsg; - } - - if (pQueryMsg->secondStageOutput) { - pExprMsg = (SSqlExpr *)pMsg; - param->pSecExpr = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES); - - for (int32_t i = 0; i < pQueryMsg->secondStageOutput; ++i) { - param->pSecExpr[i] = pExprMsg; - - pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); - pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); - pExprMsg->resType = htons(pExprMsg->resType); - pExprMsg->resBytes = htons(pExprMsg->resBytes); - pExprMsg->colBytes = htons(pExprMsg->colBytes); - pExprMsg->colType = htons(pExprMsg->colType); - - pExprMsg->functionId = htons(pExprMsg->functionId); - pExprMsg->numOfParams = htons(pExprMsg->numOfParams); - - pMsg += sizeof(SSqlExpr); - - for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { - pExprMsg->param[j].nType = htonl(pExprMsg->param[j].nType); - pExprMsg->param[j].nLen = htonl(pExprMsg->param[j].nLen); - - if (pExprMsg->param[j].nType == TSDB_DATA_TYPE_BINARY) { - pExprMsg->param[j].pz = pMsg; - pMsg += pExprMsg->param[j].nLen; // one more for the string terminated char. - } else { - pExprMsg->param[j].i64 = htobe64(pExprMsg->param[j].i64); - } - } - - int16_t functionId = pExprMsg->functionId; - if (functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG_DUMMY) { - if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - } - - pExprMsg = (SSqlExpr *)pMsg; - } - } - - pMsg = createTableIdList(pQueryMsg, pMsg, &(param->pTableIdList)); - - if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns - param->pGroupColIndex = malloc(pQueryMsg->numOfGroupCols * sizeof(SColIndex)); - if (param->pGroupColIndex == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { - param->pGroupColIndex[i].colId = htons(*(int16_t *)pMsg); - pMsg += sizeof(param->pGroupColIndex[i].colId); - - param->pGroupColIndex[i].colIndex = htons(*(int16_t *)pMsg); - pMsg += sizeof(param->pGroupColIndex[i].colIndex); - - param->pGroupColIndex[i].flag = htons(*(int16_t *)pMsg); - pMsg += sizeof(param->pGroupColIndex[i].flag); - - memcpy(param->pGroupColIndex[i].name, pMsg, tListLen(param->pGroupColIndex[i].name)); - pMsg += tListLen(param->pGroupColIndex[i].name); - } - - pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); - pQueryMsg->orderType = htons(pQueryMsg->orderType); - } - - pQueryMsg->fillType = htons(pQueryMsg->fillType); - if (pQueryMsg->fillType != TSDB_FILL_NONE) { - pQueryMsg->fillVal = (uint64_t)(pMsg); - - int64_t *v = (int64_t *)pMsg; - for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { - v[i] = htobe64(v[i]); - } - - pMsg += sizeof(int64_t) * pQueryMsg->numOfOutput; - } - - if (pQueryMsg->numOfTags > 0) { - param->pTagColumnInfo = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); - if (param->pTagColumnInfo == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { - SColumnInfo* pTagCol = (SColumnInfo*) pMsg; - - pTagCol->colId = htons(pTagCol->colId); - pTagCol->bytes = htons(pTagCol->bytes); - pTagCol->type = htons(pTagCol->type); - pTagCol->flist.numOfFilters = 0; - - param->pTagColumnInfo[i] = *pTagCol; - pMsg += sizeof(SColumnInfo); - } - } - - // the tag query condition expression string is located at the end of query msg - if (pQueryMsg->tagCondLen > 0) { - param->tagCond = calloc(1, pQueryMsg->tagCondLen); - if (param->tagCond == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - memcpy(param->tagCond, pMsg, pQueryMsg->tagCondLen); - pMsg += pQueryMsg->tagCondLen; - } - - if (pQueryMsg->prevResultLen > 0) { - param->prevResult = calloc(1, pQueryMsg->prevResultLen); - if (param->prevResult == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - memcpy(param->prevResult, pMsg, pQueryMsg->prevResultLen); - pMsg += pQueryMsg->prevResultLen; - } - - if (pQueryMsg->tbnameCondLen > 0) { - param->tbnameCond = calloc(1, pQueryMsg->tbnameCondLen + 1); - if (param->tbnameCond == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - strncpy(param->tbnameCond, pMsg, pQueryMsg->tbnameCondLen); - pMsg += pQueryMsg->tbnameCondLen; - } - - //skip ts buf - if ((pQueryMsg->tsBuf.tsOffset + pQueryMsg->tsBuf.tsLen) > 0) { - pMsg = (char *)pQueryMsg + pQueryMsg->tsBuf.tsOffset + pQueryMsg->tsBuf.tsLen; - } - - param->pOperator = taosArrayInit(pQueryMsg->numOfOperator, sizeof(int32_t)); - for(int32_t i = 0; i < pQueryMsg->numOfOperator; ++i) { - int32_t op = htonl(*(int32_t*)pMsg); - taosArrayPush(param->pOperator, &op); - - pMsg += sizeof(int32_t); - } - - if (pQueryMsg->udfContentLen > 0) { - param->pUdfInfo = calloc(1, sizeof(SUdfInfo)); - param->pUdfInfo->contLen = pQueryMsg->udfContentLen; - - pMsg = (char*)pQueryMsg + pQueryMsg->udfContentOffset; - param->pUdfInfo->resType = *(int8_t*) pMsg; - pMsg += sizeof(int8_t); - - param->pUdfInfo->resBytes = htons(*(int16_t*)pMsg); - pMsg += sizeof(int16_t); - - tstr* name = (tstr*)(pMsg); - param->pUdfInfo->name = strndup(name->data, name->len); - - pMsg += varDataTLen(name); - param->pUdfInfo->funcType = htonl(*(int32_t*)pMsg); - pMsg += sizeof(int32_t); - - param->pUdfInfo->bufSize = htonl(*(int32_t*)pMsg); - pMsg += sizeof(int32_t); - - param->pUdfInfo->content = malloc(pQueryMsg->udfContentLen); - memcpy(param->pUdfInfo->content, pMsg, pQueryMsg->udfContentLen); - - pMsg += pQueryMsg->udfContentLen; - } - - param->sql = strndup(pMsg, pQueryMsg->sqlstrLen); - - SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; - if (!validateQueryTableCols(&info, param->pExpr, pQueryMsg->numOfOutput, param->pTagColumnInfo, pQueryMsg)) { - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - - qDebug("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, " - "outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, compNumOfBlocks:%d, limit:%" PRId64 ", offset:%" PRId64, - pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->queryType, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols, - pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->interval.interval, - pQueryMsg->fillType, pQueryMsg->tsBuf.tsLen, pQueryMsg->tsBuf.tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset); - - qDebug("qmsg:%p, sql:%s", pQueryMsg, param->sql); - return TSDB_CODE_SUCCESS; - -_cleanup: - freeParam(param); - return code; -} - -int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int32_t filterNum) { - if (filterNum <= 0) { - return TSDB_CODE_SUCCESS; - } - - *dst = calloc(filterNum, sizeof(*src)); - if (*dst == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - memcpy(*dst, src, sizeof(*src) * filterNum); - - for (int32_t i = 0; i < filterNum; i++) { - if ((*dst)[i].filterstr && dst[i]->len > 0) { - void *pz = calloc(1, (size_t)(*dst)[i].len + 1); - - if (pz == NULL) { - if (i == 0) { - free(*dst); - } else { - freeColumnFilterInfo(*dst, i); - } - - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - memcpy(pz, (void *)src->pz, (size_t)src->len + 1); - - (*dst)[i].pz = (int64_t)pz; - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t buildArithmeticExprFromMsg(SExprInfo *pExprInfo, void *pQueryMsg) { - qDebug("qmsg:%p create arithmetic expr from binary", pQueryMsg); - - tExprNode* pExprNode = NULL; - TRY(TSDB_MAX_TAG_CONDITIONS) { - pExprNode = exprTreeFromBinary(pExprInfo->base.param[0].pz, pExprInfo->base.param[0].nLen); - } CATCH( code ) { - CLEANUP_EXECUTE(); - qError("qmsg:%p failed to create arithmetic expression string from:%s, reason: %s", pQueryMsg, pExprInfo->base.param[0].pz, tstrerror(code)); - return code; - } END_TRY - - if (pExprNode == NULL) { - qError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pExprInfo->base.param[0].pz); - return TSDB_CODE_QRY_APP_ERROR; - } - - pExprInfo->pExpr = pExprNode; - return TSDB_CODE_SUCCESS; -} - - -static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SColumnInfo* pTagCols, SExprInfo* pExprs, int32_t numOfOutput, int32_t tagLen, bool superTable) { - for (int32_t i = 0; i < numOfOutput; ++i) { - int16_t functId = pExprs[i].base.functionId; - - if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { - int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols); - if (j < 0 || j >= pTableInfo->numOfCols) { - return TSDB_CODE_QRY_INVALID_MSG; - } else { - SColumnInfo* pCol = &pTableInfo->colList[j]; - int32_t ret = getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.param[0].i64, - &pExprs[i].base.resType, &pExprs[i].base.resBytes, &pExprs[i].base.interBytes, tagLen, superTable, NULL); - assert(ret == TSDB_CODE_SUCCESS); - } - } - } - - return TSDB_CODE_SUCCESS; -} - -void destroyUdfInfo(SUdfInfo* pUdfInfo) { - if (pUdfInfo == NULL) { - return; - } - - if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) { - if (pUdfInfo->isScript) { - (*(scriptDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(pUdfInfo->pScriptCtx); - tfree(pUdfInfo->content); - }else{ - (*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init); - } - } - - tfree(pUdfInfo->name); - - if (pUdfInfo->path) { - unlink(pUdfInfo->path); - } - - tfree(pUdfInfo->path); - tfree(pUdfInfo->content); - taosCloseDll(pUdfInfo->handle); - tfree(pUdfInfo); -} - -static char* getUdfFuncName(char* funcname, char* name, int type) { - switch (type) { - case TSDB_UDF_FUNC_NORMAL: - strcpy(funcname, name); - break; - case TSDB_UDF_FUNC_INIT: - sprintf(funcname, "%s_init", name); - break; - case TSDB_UDF_FUNC_FINALIZE: - sprintf(funcname, "%s_finalize", name); - break; - case TSDB_UDF_FUNC_MERGE: - sprintf(funcname, "%s_merge", name); - break; - case TSDB_UDF_FUNC_DESTROY: - sprintf(funcname, "%s_destroy", name); - break; - default: - assert(0); - break; - } - - return funcname; -} - -int32_t initUdfInfo(SUdfInfo* pUdfInfo) { - if (pUdfInfo == NULL) { - return TSDB_CODE_SUCCESS; - } - //qError("script len: %d", pUdfInfo->contLen); - if (isValidScript(pUdfInfo->content, pUdfInfo->contLen)) { - pUdfInfo->isScript = 1; - pUdfInfo->pScriptCtx = createScriptCtx(pUdfInfo->content, pUdfInfo->resType, pUdfInfo->resBytes); - if (pUdfInfo->pScriptCtx == NULL) { - return TSDB_CODE_QRY_SYS_ERROR; - } - tfree(pUdfInfo->content); - - pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadScriptInit; - if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] == NULL - || (*(scriptInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(pUdfInfo->pScriptCtx) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_QRY_SYS_ERROR; - } - - pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadScriptNormal; - - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadScriptFinalize; - pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadScriptMerge; - } - pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadScriptDestroy; - - } else { - char path[PATH_MAX] = {0}; - taosGetTmpfilePath("script", path); - - FILE* file = fopen(path, "w+"); - - // TODO check for failure of flush to disk - /*size_t t = */ fwrite(pUdfInfo->content, pUdfInfo->contLen, 1, file); - fclose(file); - tfree(pUdfInfo->content); - - pUdfInfo->path = strdup(path); - - pUdfInfo->handle = taosLoadDll(path); - - if (NULL == pUdfInfo->handle) { - return TSDB_CODE_QRY_SYS_ERROR; - } - - char funcname[TSDB_FUNCTIONS_NAME_MAX_LENGTH + 10] = {0}; - pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_NORMAL)); - if (NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL]) { - return TSDB_CODE_QRY_SYS_ERROR; - } - - pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_INIT)); - - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE)); - pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_MERGE)); - } - - pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_DESTROY)); - - if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT]) { - return (*(udfInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(&pUdfInfo->init); - } - } - - return TSDB_CODE_SUCCESS; -} - -// TODO tag length should be passed from client, refactor -int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, - SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, SUdfInfo* pUdfInfo) { - *pExprInfo = NULL; - int32_t code = TSDB_CODE_SUCCESS; - - code = initUdfInfo(pUdfInfo); - if (code) { - return code; - } - - SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo)); - if (pExprs == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - bool isSuperTable = QUERY_IS_STABLE_QUERY(queryType); - int16_t tagLen = 0; - - for (int32_t i = 0; i < numOfOutput; ++i) { - pExprs[i].base = *pExprMsg[i]; - - memset(pExprs[i].base.param, 0, sizeof(tVariant) * tListLen(pExprs[i].base.param)); - for (int32_t j = 0; j < pExprMsg[i]->numOfParams; ++j) { - tVariantAssign(&pExprs[i].base.param[j], &pExprMsg[i]->param[j]); - } - - int16_t type = 0; - int16_t bytes = 0; - - // parse the arithmetic expression - if (pExprs[i].base.functionId == TSDB_FUNC_ARITHM) { - code = buildArithmeticExprFromMsg(&pExprs[i], pMsg); - - if (code != TSDB_CODE_SUCCESS) { - tfree(pExprs); - return code; - } - - type = TSDB_DATA_TYPE_DOUBLE; - bytes = tDataTypes[type].bytes; - } else if (pExprs[i].base.functionId == TSDB_FUNC_BLKINFO) { - SSchema s = {.type=TSDB_DATA_TYPE_BINARY, .bytes=TSDB_MAX_BINARY_LEN}; - type = s.type; - bytes = s.bytes; - } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) { // parse the normal column - SSchema* s = tGetTbnameColumnSchema(); - type = s->type; - bytes = s->bytes; - } else if (pExprs[i].base.colInfo.colId <= TSDB_UD_COLUMN_INDEX && pExprs[i].base.colInfo.colId > TSDB_RES_COL_ID) { - // it is a user-defined constant value column - assert(pExprs[i].base.functionId == TSDB_FUNC_PRJ); - - type = pExprs[i].base.param[1].nType; - bytes = pExprs[i].base.param[1].nLen; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - bytes += VARSTR_HEADER_SIZE; - } - } else { - int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols); - if (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag)) { - if (j < TSDB_TBNAME_COLUMN_INDEX || j >= pTableInfo->numOfTags) { - tfree(pExprs); - return TSDB_CODE_QRY_INVALID_MSG; - } - } else { - if (j < PRIMARYKEY_TIMESTAMP_COL_INDEX || j >= pTableInfo->numOfCols) { - tfree(pExprs); - return TSDB_CODE_QRY_INVALID_MSG; - } - } - - if (pExprs[i].base.colInfo.colId != TSDB_TBNAME_COLUMN_INDEX && j >= 0) { - SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag))? &pTagCols[j]:&pTableInfo->colList[j]; - type = pCol->type; - bytes = pCol->bytes; - } else { - SSchema* s = tGetTbnameColumnSchema(); - - type = s->type; - bytes = s->bytes; - } - - if (pExprs[i].base.flist.numOfFilters > 0) { - int32_t ret = cloneExprFilterInfo(&pExprs[i].base.flist.filterInfo, pExprMsg[i]->flist.filterInfo, - pExprMsg[i]->flist.numOfFilters); - if (ret) { - tfree(pExprs); - return ret; - } - } - } - - int32_t param = (int32_t)pExprs[i].base.param[0].i64; - if (pExprs[i].base.functionId != TSDB_FUNC_ARITHM && - (type != pExprs[i].base.colType || bytes != pExprs[i].base.colBytes)) { - tfree(pExprs); - return TSDB_CODE_QRY_INVALID_MSG; - } - - // todo remove it - if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].base.resType, &pExprs[i].base.resBytes, - &pExprs[i].base.interBytes, 0, isSuperTable, pUdfInfo) != TSDB_CODE_SUCCESS) { - tfree(pExprs); - return TSDB_CODE_QRY_INVALID_MSG; - } - - if (pExprs[i].base.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].base.functionId == TSDB_FUNC_TS_DUMMY) { - tagLen += pExprs[i].base.resBytes; - } - - assert(isValidDataType(pExprs[i].base.resType)); - } - - // the tag length is affected by other tag columns, so this should be update. - updateOutputBufForTopBotQuery(pTableInfo, pTagCols, pExprs, numOfOutput, tagLen, isSuperTable); - - *pExprInfo = pExprs; - return TSDB_CODE_SUCCESS; -} - -int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) { - tExprNode* expr = NULL; - - TRY(TSDB_MAX_TAG_CONDITIONS) { - expr = exprTreeFromBinary(data, len); - } CATCH( code ) { - CLEANUP_EXECUTE(); - return code; - } END_TRY - - if (expr == NULL) { - qError("failed to create expr tree"); - return TSDB_CODE_QRY_APP_ERROR; - } - - int32_t ret = filterInitFromTree(expr, pFilters, 0); - tExprTreeDestroy(expr, NULL); - - return ret; -} - - -// todo refactor -int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo, - SSqlExpr** pExpr, SExprInfo* prevExpr, SUdfInfo *pUdfInfo) { - *pExprInfo = NULL; - int32_t code = TSDB_CODE_SUCCESS; - - SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo)); - if (pExprs == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); - - for (int32_t i = 0; i < numOfOutput; ++i) { - pExprs[i].base = *pExpr[i]; - memset(pExprs[i].base.param, 0, sizeof(tVariant) * tListLen(pExprs[i].base.param)); - - for (int32_t j = 0; j < pExpr[i]->numOfParams; ++j) { - tVariantAssign(&pExprs[i].base.param[j], &pExpr[i]->param[j]); - } - - pExprs[i].base.resType = 0; - - int16_t type = 0; - int16_t bytes = 0; - - // parse the arithmetic expression - if (pExprs[i].base.functionId == TSDB_FUNC_ARITHM) { - code = buildArithmeticExprFromMsg(&pExprs[i], pQueryMsg); - - if (code != TSDB_CODE_SUCCESS) { - tfree(pExprs); - return code; - } - - type = TSDB_DATA_TYPE_DOUBLE; - bytes = tDataTypes[type].bytes; - } else { - int32_t index = pExprs[i].base.colInfo.colIndex; - assert(prevExpr[index].base.resColId == pExprs[i].base.colInfo.colId); - - type = prevExpr[index].base.resType; - bytes = prevExpr[index].base.resBytes; - } - - int32_t param = (int32_t)pExprs[i].base.param[0].i64; - if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].base.resType, &pExprs[i].base.resBytes, - &pExprs[i].base.interBytes, 0, isSuperTable, pUdfInfo) != TSDB_CODE_SUCCESS) { - tfree(pExprs); - return TSDB_CODE_QRY_INVALID_MSG; - } - - assert(isValidDataType(pExprs[i].base.resType)); - } - - *pExprInfo = pExprs; - return TSDB_CODE_SUCCESS; -} - -SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code) { - if (pQueryMsg->numOfGroupCols == 0) { - return NULL; - } - - // using group by tag columns - SGroupbyExpr *pGroupbyExpr = (SGroupbyExpr *)calloc(1, sizeof(SGroupbyExpr)); - if (pGroupbyExpr == NULL) { - *code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pGroupbyExpr->numOfGroupCols = pQueryMsg->numOfGroupCols; - pGroupbyExpr->orderType = pQueryMsg->orderType; - pGroupbyExpr->orderIndex = pQueryMsg->orderByIdx; - - pGroupbyExpr->columnInfo = taosArrayInit(pQueryMsg->numOfGroupCols, sizeof(SColIndex)); - for(int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { - taosArrayPush(pGroupbyExpr->columnInfo, &pColIndex[i]); - } - - return pGroupbyExpr; -} - -int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId) { - *pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfFilterCols); - if (*pFilterInfo == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t i = 0, j = 0; i < numOfCols; ++i) { - if (pCols[i].flist.numOfFilters > 0) { - SSingleColumnFilterInfo* pFilter = &((*pFilterInfo)[j]); - - memcpy(&pFilter->info, &pCols[i], sizeof(SColumnInfo)); - pFilter->info = pCols[i]; - - pFilter->numOfFilters = pCols[i].flist.numOfFilters; - pFilter->pFilters = calloc(pFilter->numOfFilters, sizeof(SColumnFilterElem)); - if (pFilter->pFilters == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t f = 0; f < pFilter->numOfFilters; ++f) { - SColumnFilterElem* pSingleColFilter = &pFilter->pFilters[f]; - pSingleColFilter->filterInfo = pCols[i].flist.filterInfo[f]; - - int32_t lower = pSingleColFilter->filterInfo.lowerRelOptr; - int32_t upper = pSingleColFilter->filterInfo.upperRelOptr; - if (lower == TSDB_RELATION_INVALID && upper == TSDB_RELATION_INVALID) { - qError("QInfo:0x%"PRIx64" invalid filter info", qId); - return TSDB_CODE_QRY_INVALID_MSG; - } - - pSingleColFilter->fp = getFilterOperator(lower, upper); - if (pSingleColFilter->fp == NULL) { - qError("QInfo:0x%"PRIx64" invalid filter info", qId); - return TSDB_CODE_QRY_INVALID_MSG; - } - - pSingleColFilter->bytes = pCols[i].bytes; - - if (lower == TSDB_RELATION_IN) { - buildFilterSetFromBinary(&pSingleColFilter->q, (char *)(pSingleColFilter->filterInfo.pz), (int32_t)(pSingleColFilter->filterInfo.len)); - } - } - - j++; - } - } - - return TSDB_CODE_SUCCESS; -} - -void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) { - for (int32_t i = 0; i < numOfFilterCols; ++i) { - if (pFilterInfo[i].numOfFilters > 0) { - if (pFilterInfo[i].pFilters->filterInfo.lowerRelOptr == TSDB_RELATION_IN) { - taosHashCleanup((SHashObj *)(pFilterInfo[i].pFilters->q)); - } - tfree(pFilterInfo[i].pFilters); - } - } - - tfree(pFilterInfo); - return NULL; -} - -int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId) { - for (int32_t i = 0; i < pQueryAttr->numOfCols; ++i) { - if (pQueryAttr->tableCols[i].flist.numOfFilters > 0 && pQueryAttr->tableCols[i].flist.filterInfo != NULL) { - pQueryAttr->numOfFilterCols++; - } - } - - if (pQueryAttr->numOfFilterCols == 0) { - return TSDB_CODE_SUCCESS; - } - - doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, - &pQueryAttr->pFilterInfo, qId); - - pQueryAttr->createFilterOperator = true; - - return TSDB_CODE_SUCCESS; -} - -static void doUpdateExprColumnIndex(SQueryAttr *pQueryAttr) { - assert(pQueryAttr->pExpr1 != NULL && pQueryAttr != NULL); - - for (int32_t k = 0; k < pQueryAttr->numOfOutput; ++k) { - SSqlExpr *pSqlExprMsg = &pQueryAttr->pExpr1[k].base; - if (pSqlExprMsg->functionId == TSDB_FUNC_ARITHM) { - continue; - } - - // todo opt performance - SColIndex *pColIndex = &pSqlExprMsg->colInfo; - if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { - int32_t f = 0; - for (f = 0; f < pQueryAttr->numOfCols; ++f) { - if (pColIndex->colId == pQueryAttr->tableCols[f].colId) { - pColIndex->colIndex = f; - break; - } - } - - assert(f < pQueryAttr->numOfCols); - } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) { - // do nothing for user-defined constant value result columns - } else { - int32_t f = 0; - for (f = 0; f < pQueryAttr->numOfTags; ++f) { - if (pColIndex->colId == pQueryAttr->tagColList[f].colId) { - pColIndex->colIndex = f; - break; - } - } - - assert(f < pQueryAttr->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX); - } - } -} - -void setResultBufSize(SQueryAttr* pQueryAttr, SRspResultInfo* pResultInfo) { - const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); - - // the minimum number of rows for projection query - const int32_t MIN_ROWS_FOR_PRJ_QUERY = 8192; - const int32_t DEFAULT_MIN_ROWS = 4096; - - const float THRESHOLD_RATIO = 0.85f; - - if (isProjQuery(pQueryAttr)) { - int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize; - if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) { - numOfRes = MIN_ROWS_FOR_PRJ_QUERY; - } - - pResultInfo->capacity = numOfRes; - } else { // in case of non-prj query, a smaller output buffer will be used. - pResultInfo->capacity = DEFAULT_MIN_ROWS; - } - - pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO); - pResultInfo->total = 0; -} - -FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) { - return ((SQInfo *)qHandle)->qId == qId; -} - -SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs, - SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, - char* sql, uint64_t qId, SUdfInfo* pUdfInfo) { - int16_t numOfCols = pQueryMsg->numOfCols; - int16_t numOfOutput = pQueryMsg->numOfOutput; - - SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); - if (pQInfo == NULL) { - goto _cleanup_qinfo; - } - - pQInfo->qId = qId; - pQInfo->startExecTs = 0; - - pQInfo->runtimeEnv.pUdfInfo = pUdfInfo; - - // to make sure third party won't overwrite this structure - pQInfo->signature = pQInfo; - SQueryAttr* pQueryAttr = &pQInfo->query; - pQInfo->runtimeEnv.pQueryAttr = pQueryAttr; - - pQueryAttr->tableGroupInfo = *pTableGroupInfo; - pQueryAttr->numOfCols = numOfCols; - pQueryAttr->numOfOutput = numOfOutput; - pQueryAttr->limit.limit = pQueryMsg->limit; - pQueryAttr->limit.offset = pQueryMsg->offset; - pQueryAttr->order.order = pQueryMsg->order; - pQueryAttr->order.orderColId = pQueryMsg->orderColId; - pQueryAttr->pExpr1 = pExprs; - pQueryAttr->pExpr2 = pSecExprs; - pQueryAttr->numOfExpr2 = pQueryMsg->secondStageOutput; - pQueryAttr->pGroupbyExpr = pGroupbyExpr; - memcpy(&pQueryAttr->interval, &pQueryMsg->interval, sizeof(pQueryAttr->interval)); - pQueryAttr->fillType = pQueryMsg->fillType; - pQueryAttr->numOfTags = pQueryMsg->numOfTags; - pQueryAttr->tagColList = pTagCols; - pQueryAttr->prjInfo.vgroupLimit = pQueryMsg->vgroupLimit; - pQueryAttr->prjInfo.ts = (pQueryMsg->order == TSDB_ORDER_ASC)? INT64_MIN:INT64_MAX; - pQueryAttr->sw = pQueryMsg->sw; - pQueryAttr->vgId = vgId; - - pQueryAttr->stableQuery = pQueryMsg->stableQuery; - pQueryAttr->topBotQuery = pQueryMsg->topBotQuery; - pQueryAttr->groupbyColumn = pQueryMsg->groupbyColumn; - pQueryAttr->hasTagResults = pQueryMsg->hasTagResults; - pQueryAttr->timeWindowInterpo = pQueryMsg->timeWindowInterpo; - pQueryAttr->queryBlockDist = pQueryMsg->queryBlockDist; - pQueryAttr->stabledev = pQueryMsg->stabledev; - pQueryAttr->tsCompQuery = pQueryMsg->tsCompQuery; - pQueryAttr->simpleAgg = pQueryMsg->simpleAgg; - pQueryAttr->pointInterpQuery = pQueryMsg->pointInterpQuery; - pQueryAttr->needReverseScan = pQueryMsg->needReverseScan; - pQueryAttr->stateWindow = pQueryMsg->stateWindow; - pQueryAttr->vgId = vgId; - pQueryAttr->pFilters = pFilters; - - pQueryAttr->tableCols = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); - if (pQueryAttr->tableCols == NULL) { - goto _cleanup; - } - - pQueryAttr->srcRowSize = 0; - pQueryAttr->maxTableColumnWidth = 0; - for (int16_t i = 0; i < numOfCols; ++i) { - pQueryAttr->tableCols[i] = pQueryMsg->tableCols[i]; - pQueryAttr->tableCols[i].flist.filterInfo = tFilterInfoDup(pQueryMsg->tableCols[i].flist.filterInfo, pQueryAttr->tableCols[i].flist.numOfFilters); - - pQueryAttr->srcRowSize += pQueryAttr->tableCols[i].bytes; - if (pQueryAttr->maxTableColumnWidth < pQueryAttr->tableCols[i].bytes) { - pQueryAttr->maxTableColumnWidth = pQueryAttr->tableCols[i].bytes; - } - } - - for (int16_t col = 0; col < numOfOutput; ++col) { - assert(pExprs[col].base.resBytes > 0); - pQueryAttr->resultRowSize += pExprs[col].base.resBytes; - - // keep the tag length - if (TSDB_COL_IS_TAG(pExprs[col].base.colInfo.flag)) { - pQueryAttr->tagLen += pExprs[col].base.resBytes; - } - - if (pExprs[col].base.flist.filterInfo) { - ++pQueryAttr->havingNum; - } - } - - doUpdateExprColumnIndex(pQueryAttr); - - if (pSecExprs != NULL) { - int32_t resultRowSize = 0; - - // calculate the result row size - for (int16_t col = 0; col < pQueryAttr->numOfExpr2; ++col) { - assert(pSecExprs[col].base.resBytes > 0); - resultRowSize += pSecExprs[col].base.resBytes; - } - - if (resultRowSize > pQueryAttr->resultRowSize) { - pQueryAttr->resultRowSize = resultRowSize; - } - } - - if (pQueryAttr->fillType != TSDB_FILL_NONE) { - pQueryAttr->fillVal = malloc(sizeof(int64_t) * pQueryAttr->numOfOutput); - if (pQueryAttr->fillVal == NULL) { - goto _cleanup; - } - - // the first column is the timestamp - memcpy(pQueryAttr->fillVal, (char *)pQueryMsg->fillVal, pQueryAttr->numOfOutput * sizeof(int64_t)); - } - - size_t numOfGroups = 0; - if (pTableGroupInfo->pGroupList != NULL) { - numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList); - STableGroupInfo* pTableqinfo = &pQInfo->runtimeEnv.tableqinfoGroupInfo; - - pTableqinfo->pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); - pTableqinfo->numOfTables = pTableGroupInfo->numOfTables; - pTableqinfo->map = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - } - - pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); - if (pQInfo->pBuf == NULL) { - goto _cleanup; - } - - pQInfo->dataReady = QUERY_RESULT_NOT_READY; - pQInfo->rspContext = NULL; - pQInfo->sql = sql; - pthread_mutex_init(&pQInfo->lock, NULL); - tsem_init(&pQInfo->ready, 0, 0); - - pQueryAttr->window = pQueryMsg->window; - updateDataCheckOrder(pQInfo, pQueryMsg, pQueryAttr->stableQuery); - - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - STimeWindow window = pQueryAttr->window; - - int32_t index = 0; - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray* pa = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); - - size_t s = taosArrayGetSize(pa); - SArray* p1 = taosArrayInit(s, POINTER_BYTES); - if (p1 == NULL) { - goto _cleanup; - } - - taosArrayPush(pRuntimeEnv->tableqinfoGroupInfo.pGroupList, &p1); - - for(int32_t j = 0; j < s; ++j) { - STableKeyInfo* info = taosArrayGet(pa, j); - window.skey = info->lastKey; - - void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); - STableQueryInfo* item = createTableQueryInfo(pQueryAttr, info->pTable, pQueryAttr->groupbyColumn, window, buf); - if (item == NULL) { - goto _cleanup; - } - - item->groupIndex = i; - taosArrayPush(p1, &item); - - STableId* id = TSDB_TABLEID(info->pTable); - taosHashPut(pRuntimeEnv->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); - index += 1; - } - } - - colIdCheck(pQueryAttr, pQInfo->qId); - - // todo refactor - pQInfo->query.queryBlockDist = (numOfOutput == 1 && pExprs[0].base.functionId == TSDB_FUNC_BLKINFO); - - qDebug("qmsg:%p vgId:%d, QInfo:0x%" PRIx64 "-%p created", pQueryMsg, pQInfo->query.vgId, pQInfo->qId, pQInfo); - return pQInfo; - -_cleanup_qinfo: - tsdbDestroyTableGroup(pTableGroupInfo); - - if (pGroupbyExpr != NULL) { - taosArrayDestroy(pGroupbyExpr->columnInfo); - free(pGroupbyExpr); - } - - tfree(pTagCols); - for (int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExprInfo = &pExprs[i]; - if (pExprInfo->pExpr != NULL) { - tExprTreeDestroy(pExprInfo->pExpr, NULL); - pExprInfo->pExpr = NULL; - } - - if (pExprInfo->base.flist.filterInfo) { - freeColumnFilterInfo(pExprInfo->base.flist.filterInfo, pExprInfo->base.flist.numOfFilters); - } - } - - tfree(pExprs); - - filterFreeInfo(pFilters); - -_cleanup: - freeQInfo(pQInfo); - return NULL; -} - -bool isValidQInfo(void *param) { - SQInfo *pQInfo = (SQInfo *)param; - if (pQInfo == NULL) { - return false; - } - - /* - * pQInfo->signature may be changed by another thread, so we assign value of signature - * into local variable, then compare by using local variable - */ - uint64_t sig = (uint64_t)pQInfo->signature; - return (sig == (uint64_t)pQInfo); -} - -int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, - int32_t prevResultLen, void* merger) { - int32_t code = TSDB_CODE_SUCCESS; - - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - pRuntimeEnv->qinfo = pQInfo; - - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - STSBuf *pTsBuf = NULL; - - if (pTsBufInfo->tsLen > 0) { // open new file to save the result - char* tsBlock = start + pTsBufInfo->tsOffset; - pTsBuf = tsBufCreateFromCompBlocks(tsBlock, pTsBufInfo->tsNumOfBlocks, pTsBufInfo->tsLen, pTsBufInfo->tsOrder, - pQueryAttr->vgId); - - if (pTsBuf == NULL) { - code = TSDB_CODE_QRY_NO_DISKSPACE; - goto _error; - } - tsBufResetPos(pTsBuf); - bool ret = tsBufNextPos(pTsBuf); - UNUSED(ret); - } - - SArray* prevResult = NULL; - if (prevResultLen > 0) { - prevResult = interResFromBinary(param->prevResult, prevResultLen); - pRuntimeEnv->prevResult = prevResult; - } - - pRuntimeEnv->currentOffset = pQueryAttr->limit.offset; - if (tsdb != NULL) { - pQueryAttr->precision = tsdbGetCfg(tsdb)->precision; - } - - if ((QUERY_IS_ASC_QUERY(pQueryAttr) && (pQueryAttr->window.skey > pQueryAttr->window.ekey)) || - (!QUERY_IS_ASC_QUERY(pQueryAttr) && (pQueryAttr->window.ekey > pQueryAttr->window.skey))) { - qDebug("QInfo:0x%"PRIx64" no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo->qId, pQueryAttr->window.skey, - pQueryAttr->window.ekey, pQueryAttr->order.order); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0; - // todo free memory - return TSDB_CODE_SUCCESS; - } - - if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { - qDebug("QInfo:0x%"PRIx64" no table qualified for tag filter, abort query", pQInfo->qId); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - return TSDB_CODE_SUCCESS; - } - - // filter the qualified - if ((code = doInitQInfo(pQInfo, pTsBuf, tsdb, sourceOptr, param->tableScanOperator, param->pOperator, merger)) != TSDB_CODE_SUCCESS) { - goto _error; - } - - return code; - -_error: - // table query ref will be decrease during error handling - freeQInfo(pQInfo); - return code; -} - -//TODO refactor -void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) { - if (pFilter == NULL || numOfFilters == 0) { - return; - } - - for (int32_t i = 0; i < numOfFilters; i++) { - if (pFilter[i].filterstr && pFilter[i].pz) { - free((void*)(pFilter[i].pz)); - } - } - - free(pFilter); -} - -static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo) { - if (pTableqinfoGroupInfo->pGroupList != NULL) { - int32_t numOfGroups = (int32_t) taosArrayGetSize(pTableqinfoGroupInfo->pGroupList); - for (int32_t i = 0; i < numOfGroups; ++i) { - SArray *p = taosArrayGetP(pTableqinfoGroupInfo->pGroupList, i); - - size_t num = taosArrayGetSize(p); - for(int32_t j = 0; j < num; ++j) { - STableQueryInfo* item = taosArrayGetP(p, j); - destroyTableQueryInfoImpl(item); - } - - taosArrayDestroy(p); - } - } - - taosArrayDestroy(pTableqinfoGroupInfo->pGroupList); - taosHashCleanup(pTableqinfoGroupInfo->map); - - pTableqinfoGroupInfo->pGroupList = NULL; - pTableqinfoGroupInfo->map = NULL; - pTableqinfoGroupInfo->numOfTables = 0; -} - -void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr) { - if (pExprInfo == NULL) { - assert(numOfExpr == 0); - return NULL; - } - - for (int32_t i = 0; i < numOfExpr; ++i) { - if (pExprInfo[i].pExpr != NULL) { - tExprTreeDestroy(pExprInfo[i].pExpr, NULL); - } - - if (pExprInfo[i].base.flist.filterInfo) { - freeColumnFilterInfo(pExprInfo[i].base.flist.filterInfo, pExprInfo[i].base.flist.numOfFilters); - } - - for(int32_t j = 0; j < pExprInfo[i].base.numOfParams; ++j) { - tVariantDestroy(&pExprInfo[i].base.param[j]); - } - } - - tfree(pExprInfo); - return NULL; -} - -void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols) { - if (pColumnInfo != NULL) { - assert(numOfCols >= 0); - - for (int32_t i = 0; i < numOfCols; i++) { - freeColumnFilterInfo(pColumnInfo[i].flist.filterInfo, pColumnInfo[i].flist.numOfFilters); - } - - tfree(pColumnInfo); - } - - return NULL; -} - -void freeQInfo(SQInfo *pQInfo) { - if (!isValidQInfo(pQInfo)) { - return; - } - - qDebug("QInfo:0x%"PRIx64" start to free QInfo", pQInfo->qId); - - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - releaseQueryBuf(pRuntimeEnv->tableqinfoGroupInfo.numOfTables); - - doDestroyTableQueryInfo(&pRuntimeEnv->tableqinfoGroupInfo); - teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); - - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - freeQueryAttr(pQueryAttr); - - tsdbDestroyTableGroup(&pQueryAttr->tableGroupInfo); - - tfree(pQInfo->pBuf); - tfree(pQInfo->sql); - - taosArrayDestroy(pQInfo->summary.queryProfEvents); - taosHashCleanup(pQInfo->summary.operatorProfResults); - - taosArrayDestroy(pRuntimeEnv->groupResInfo.pRows); - pQInfo->signature = 0; - - qDebug("QInfo:0x%"PRIx64" QInfo is freed", pQInfo->qId); - - tfree(pQInfo); -} - -int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen) { - // the remained number of retrieved rows, not the interpolated result - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - - // load data from file to msg buffer - if (pQueryAttr->tsCompQuery) { - SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); - FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor - - // make sure file exist - if (f) { - off_t s = lseek(fileno(f), 0, SEEK_END); - assert(s == pRuntimeEnv->outputBuf->info.rows); - - qDebug("QInfo:0x%"PRIx64" ts comp data return, file:%p, size:%"PRId64, pQInfo->qId, f, (uint64_t)s); - if (fseek(f, 0, SEEK_SET) >= 0) { - size_t sz = fread(data, 1, s, f); - if(sz < s) { // todo handle error - qError("fread(f:%p,%d) failed, rsize:%" PRId64 ", expect size:%" PRId64, f, fileno(f), (uint64_t)sz, (uint64_t)s); - assert(0); - } - } else { - UNUSED(s); - qError("fseek(f:%p,%d) failed, error:%s", f, fileno(f), strerror(errno)); - assert(0); - } - - // dump error info - if (s <= (sizeof(STSBufFileHeader) + sizeof(STSGroupBlockInfo) + 6 * sizeof(int32_t))) { - qDump(data, s); - assert(0); - } - - fclose(f); - *(FILE **)pColInfoData->pData = NULL; - } - - // all data returned, set query over - if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)) { - setQueryStatus(pRuntimeEnv, QUERY_OVER); - } - } else { - doCopyQueryResultToMsg(pQInfo, (int32_t)pRuntimeEnv->outputBuf->info.rows, data, compressed, compLen); - } - - qDebug("QInfo:0x%"PRIx64" current numOfRes rows:%d, total:%" PRId64, pQInfo->qId, - pRuntimeEnv->outputBuf->info.rows, pRuntimeEnv->resultInfo.total); - - if (pQueryAttr->limit.limit > 0 && pQueryAttr->limit.limit == pRuntimeEnv->resultInfo.total) { - qDebug("QInfo:0x%"PRIx64" results limitation reached, limitation:%"PRId64, pQInfo->qId, pQueryAttr->limit.limit); - setQueryStatus(pRuntimeEnv, QUERY_OVER); - } - - return TSDB_CODE_SUCCESS; -} - -bool doBuildResCheck(SQInfo* pQInfo) { - bool buildRes = false; - - pthread_mutex_lock(&pQInfo->lock); - - pQInfo->dataReady = QUERY_RESULT_READY; - buildRes = needBuildResAfterQueryComplete(pQInfo); - - // clear qhandle owner, it must be in the secure area. other thread may run ahead before current, after it is - // put into task to be executed. - assert(pQInfo->owner == taosGetSelfPthreadId()); - pQInfo->owner = 0; - - pthread_mutex_unlock(&pQInfo->lock); - - // used in retrieve blocking model. - tsem_post(&pQInfo->ready); - return buildRes; -} - -static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) { - if (val == NULL) { - setNull(output, type, bytes); - return; - } - - if (IS_VAR_DATA_TYPE(type)) { - // Binary data overflows for sort of unknown reasons. Let trim the overflow data - if (varDataTLen(val) > bytes) { - int32_t maxLen = bytes - VARSTR_HEADER_SIZE; - int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val); - memcpy(varDataVal(output), varDataVal(val), len); - varDataSetLen(output, len); - } else { - varDataCopy(output, val); - } - } else { - memcpy(output, val, bytes); - } -} - -static int64_t getQuerySupportBufSize(size_t numOfTables) { - size_t s1 = sizeof(STableQueryInfo); - size_t s2 = sizeof(SHashNode); - -// size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb - return (int64_t)((s1 + s2) * 1.5 * numOfTables); -} - -int32_t checkForQueryBuf(size_t numOfTables) { - int64_t t = getQuerySupportBufSize(numOfTables); - if (tsQueryBufferSizeBytes < 0) { - return TSDB_CODE_SUCCESS; - } else if (tsQueryBufferSizeBytes > 0) { - - while(1) { - int64_t s = tsQueryBufferSizeBytes; - int64_t remain = s - t; - if (remain >= 0) { - if (atomic_val_compare_exchange_64(&tsQueryBufferSizeBytes, s, remain) == s) { - return TSDB_CODE_SUCCESS; - } - } else { - return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; - } - } - } - - // disable query processing if the value of tsQueryBufferSize is zero. - return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; -} - -bool checkNeedToCompressQueryCol(SQInfo *pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - SSDataBlock* pRes = pRuntimeEnv->outputBuf; - - if (GET_NUM_OF_RESULTS(&(pQInfo->runtimeEnv)) <= 0) { - return false; - } - - int32_t numOfRows = pQueryAttr->pExpr2 ? GET_NUM_OF_RESULTS(pRuntimeEnv) : pRes->info.rows; - int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput; - - for (int32_t col = 0; col < numOfCols; ++col) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); - int32_t colSize = pColRes->info.bytes * numOfRows; - if (NEEDTO_COMPRESS_QUERY(colSize)) { - return true; - } - } - - return false; -} - -void releaseQueryBuf(size_t numOfTables) { - if (tsQueryBufferSizeBytes < 0) { - return; - } - - int64_t t = getQuerySupportBufSize(numOfTables); - - // restore value is not enough buffer available - atomic_add_fetch_64(&tsQueryBufferSizeBytes, t); -} - -void freeQueryAttr(SQueryAttr* pQueryAttr) { - if (pQueryAttr != NULL) { - if (pQueryAttr->fillVal != NULL) { - tfree(pQueryAttr->fillVal); - } - - pQueryAttr->pFilterInfo = doDestroyFilterInfo(pQueryAttr->pFilterInfo, pQueryAttr->numOfFilterCols); - - pQueryAttr->pExpr1 = destroyQueryFuncExpr(pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - pQueryAttr->pExpr2 = destroyQueryFuncExpr(pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); - pQueryAttr->pExpr3 = destroyQueryFuncExpr(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3); - - tfree(pQueryAttr->tagColList); - tfree(pQueryAttr->pFilterInfo); - - pQueryAttr->tableCols = freeColumnInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols); - - if (pQueryAttr->pGroupbyExpr != NULL) { - taosArrayDestroy(pQueryAttr->pGroupbyExpr->columnInfo); - tfree(pQueryAttr->pGroupbyExpr); - } - - filterFreeInfo(pQueryAttr->pFilters); - } -} - diff --git a/2.0/src/query/src/qExtbuffer.c b/2.0/src/query/src/qExtbuffer.c deleted file mode 100644 index 35e95a684568221597f66c37e94dd415328167da..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qExtbuffer.c +++ /dev/null @@ -1,1159 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "qExtbuffer.h" -#include "../../../include/client/taos.h" -#include "os.h" -#include "qAggMain.h" -#include "qExecutor.h" -#include "queryLog.h" -#include "taosdef.h" -#include "tmsg.h" -#include "tcompare.h" -#include "tulog.h" - -#define COLMODEL_GET_VAL(data, schema, allrow, rowId, colId) \ - (data + (schema)->pFields[colId].offset * (allrow) + (rowId) * (schema)->pFields[colId].field.bytes) - -/* - * SColumnModel is deeply copy - */ -tExtMemBuffer* createExtMemBuffer(int32_t inMemSize, int32_t elemSize, int32_t pagesize, SColumnModel *pModel) { - tExtMemBuffer* pMemBuffer = (tExtMemBuffer *)calloc(1, sizeof(tExtMemBuffer)); - - pMemBuffer->pageSize = pagesize; - pMemBuffer->inMemCapacity = ALIGN8(inMemSize) / pMemBuffer->pageSize; - pMemBuffer->nElemSize = elemSize; - - pMemBuffer->numOfElemsPerPage = (pMemBuffer->pageSize - sizeof(tFilePage)) / pMemBuffer->nElemSize; - - char name[MAX_TMPFILE_PATH_LENGTH] = {0}; - taosGetTmpfilePath("extbuf", name); - - pMemBuffer->path = strdup(name); - uDebug("create tmp file:%s", pMemBuffer->path); - - SExtFileInfo *pFMeta = &pMemBuffer->fileMeta; - - pFMeta->pageSize = DEFAULT_PAGE_SIZE; - - pFMeta->flushoutData.nAllocSize = 4; - pFMeta->flushoutData.nLength = 0; - pFMeta->flushoutData.pFlushoutInfo = (tFlushoutInfo *)calloc(4, sizeof(tFlushoutInfo)); - - pMemBuffer->pColumnModel = cloneColumnModel(pModel); - pMemBuffer->pColumnModel->capacity = pMemBuffer->numOfElemsPerPage; - - return pMemBuffer; -} - -void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) { - if (pMemBuffer == NULL) { - return NULL; - } - - // release flush out info link - SExtFileInfo *pFileMeta = &pMemBuffer->fileMeta; - if (pFileMeta->flushoutData.nAllocSize != 0 && pFileMeta->flushoutData.pFlushoutInfo != NULL) { - tfree(pFileMeta->flushoutData.pFlushoutInfo); - } - - // release all in-memory buffer pages - tFilePagesItem *pFilePages = pMemBuffer->pHead; - while (pFilePages != NULL) { - tFilePagesItem *pTmp = pFilePages; - pFilePages = pFilePages->pNext; - tfree(pTmp); - } - - // close temp file - if (pMemBuffer->file != 0) { - if (fclose(pMemBuffer->file) != 0) { - uError("failed to close file:%s, reason:%s", pMemBuffer->path, strerror(errno)); - } - - uDebug("remove temp file:%s for external buffer", pMemBuffer->path); - unlink(pMemBuffer->path); - } - - destroyColumnModel(pMemBuffer->pColumnModel); - - tfree(pMemBuffer->path); - tfree(pMemBuffer); - - return NULL; -} - -/* - * alloc more memory for flush out info entries. - */ -static bool allocFlushoutInfoEntries(SExtFileInfo *pFileMeta) { - pFileMeta->flushoutData.nAllocSize = pFileMeta->flushoutData.nAllocSize << 1; - - tFlushoutInfo *tmp = (tFlushoutInfo *)realloc(pFileMeta->flushoutData.pFlushoutInfo, - sizeof(tFlushoutInfo) * pFileMeta->flushoutData.nAllocSize); - if (tmp == NULL) { - uError("out of memory!\n"); - return false; - } - - pFileMeta->flushoutData.pFlushoutInfo = tmp; - return true; -} - -static bool tExtMemBufferAlloc(tExtMemBuffer *pMemBuffer) { - /* - * the in-mem buffer is full. - * To flush data to disk to accommodate more data - */ - if (pMemBuffer->numOfInMemPages > 0 && pMemBuffer->numOfInMemPages == pMemBuffer->inMemCapacity) { - if (tExtMemBufferFlush(pMemBuffer) != 0) { - return false; - } - } - - /* - * We do not recycle the file page structure. And in flush data operations, all - * file page that are full of data are destroyed after data being flushed to disk. - * - * The memory buffer pages may be recycle in order to avoid unnecessary memory - * allocation later. - */ - tFilePagesItem *item = (tFilePagesItem *)calloc(1, pMemBuffer->pageSize + sizeof(tFilePagesItem)); - if (item == NULL) { - return false; - } - - item->pNext = NULL; - item->item.num = 0; - - if (pMemBuffer->pTail != NULL) { - pMemBuffer->pTail->pNext = item; - pMemBuffer->pTail = item; - } else { - pMemBuffer->pTail = item; - pMemBuffer->pHead = item; - } - - pMemBuffer->numOfInMemPages += 1; - return true; -} - -/* - * put elements into buffer - */ -int16_t tExtMemBufferPut(tExtMemBuffer *pMemBuffer, void *data, int32_t numOfRows) { - if (numOfRows == 0) { - return pMemBuffer->numOfInMemPages; - } - - tFilePagesItem *pLast = pMemBuffer->pTail; - if (pLast == NULL) { - if (!tExtMemBufferAlloc(pMemBuffer)) { - return -1; - } - - pLast = pMemBuffer->pTail; - } - - if (pLast->item.num + numOfRows <= pMemBuffer->numOfElemsPerPage) { // enough space for records - tColModelAppend(pMemBuffer->pColumnModel, &pLast->item, data, 0, numOfRows, numOfRows); - - pMemBuffer->numOfElemsInBuffer += numOfRows; - pMemBuffer->numOfTotalElems += numOfRows; - } else { - int32_t numOfRemainEntries = pMemBuffer->numOfElemsPerPage - (int32_t)pLast->item.num; - tColModelAppend(pMemBuffer->pColumnModel, &pLast->item, data, 0, numOfRemainEntries, numOfRows); - - pMemBuffer->numOfElemsInBuffer += numOfRemainEntries; - pMemBuffer->numOfTotalElems += numOfRemainEntries; - - int32_t hasWritten = numOfRemainEntries; - int32_t remain = numOfRows - numOfRemainEntries; - - while (remain > 0) { - if (!tExtMemBufferAlloc(pMemBuffer)) { // failed to allocate memory buffer - return -1; - } - - int32_t numOfWriteElems = 0; - if (remain > pMemBuffer->numOfElemsPerPage) { - numOfWriteElems = pMemBuffer->numOfElemsPerPage; - } else { - numOfWriteElems = remain; - } - - pMemBuffer->numOfTotalElems += numOfWriteElems; - - pLast = pMemBuffer->pTail; - tColModelAppend(pMemBuffer->pColumnModel, &pLast->item, data, hasWritten, numOfWriteElems, numOfRows); - - remain -= numOfWriteElems; - pMemBuffer->numOfElemsInBuffer += numOfWriteElems; - hasWritten += numOfWriteElems; - } - } - - return pMemBuffer->numOfInMemPages; -} - -static bool tExtMemBufferUpdateFlushoutInfo(tExtMemBuffer *pMemBuffer) { - SExtFileInfo *pFileMeta = &pMemBuffer->fileMeta; - - if (pMemBuffer->flushModel == MULTIPLE_APPEND_MODEL) { - if (pFileMeta->flushoutData.nLength == pFileMeta->flushoutData.nAllocSize && !allocFlushoutInfoEntries(pFileMeta)) { - return false; - } - - tFlushoutInfo *pFlushoutInfo = &pFileMeta->flushoutData.pFlushoutInfo[pFileMeta->flushoutData.nLength]; - if (pFileMeta->flushoutData.nLength == 0) { - pFlushoutInfo->startPageId = 0; - } else { - pFlushoutInfo->startPageId = - pFileMeta->flushoutData.pFlushoutInfo[pFileMeta->flushoutData.nLength - 1].startPageId + - pFileMeta->flushoutData.pFlushoutInfo[pFileMeta->flushoutData.nLength - 1].numOfPages; - } - - // only the page still in buffer is flushed out to disk - pFlushoutInfo->numOfPages = pMemBuffer->numOfInMemPages; - pFileMeta->flushoutData.nLength += 1; - } else { - // always update the first flush out array in single_flush_model - pFileMeta->flushoutData.nLength = 1; - tFlushoutInfo *pFlushoutInfo = &pFileMeta->flushoutData.pFlushoutInfo[0]; - pFlushoutInfo->numOfPages += pMemBuffer->numOfInMemPages; - } - - return true; -} - -static void tExtMemBufferClearFlushoutInfo(tExtMemBuffer *pMemBuffer) { - SExtFileInfo *pFileMeta = &pMemBuffer->fileMeta; - - pFileMeta->flushoutData.nLength = 0; - memset(pFileMeta->flushoutData.pFlushoutInfo, 0, sizeof(tFlushoutInfo) * pFileMeta->flushoutData.nAllocSize); -} - -int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer) { - int32_t ret = 0; - if (pMemBuffer->numOfTotalElems == 0) { - return ret; - } - - if (pMemBuffer->file == NULL) { - if ((pMemBuffer->file = fopen(pMemBuffer->path, "wb+")) == NULL) { - ret = TAOS_SYSTEM_ERROR(errno); - return ret; - } - } - - /* all data has been flushed to disk, ignore flush operation */ - if (pMemBuffer->numOfElemsInBuffer == 0) { - return ret; - } - - tFilePagesItem *first = pMemBuffer->pHead; - while (first != NULL) { - size_t retVal = fwrite((char *)&(first->item), pMemBuffer->pageSize, 1, pMemBuffer->file); - if (retVal <= 0) { // failed to write to buffer, may be not enough space - ret = TAOS_SYSTEM_ERROR(errno); - pMemBuffer->pHead = first; - return ret; - } - - pMemBuffer->fileMeta.numOfElemsInFile += (uint32_t)first->item.num; - pMemBuffer->fileMeta.nFileSize += 1; - - tFilePagesItem *ptmp = first; - first = first->pNext; - - tfree(ptmp); // release all data in memory buffer - } - - fflush(pMemBuffer->file); // flush to disk - - tExtMemBufferUpdateFlushoutInfo(pMemBuffer); - - pMemBuffer->numOfElemsInBuffer = 0; - pMemBuffer->numOfInMemPages = 0; - pMemBuffer->pHead = NULL; - pMemBuffer->pTail = NULL; - - return ret; -} - -void tExtMemBufferClear(tExtMemBuffer *pMemBuffer) { - if (pMemBuffer == NULL || pMemBuffer->numOfTotalElems == 0) { - return; - } - - //release all data in memory buffer - tFilePagesItem *first = pMemBuffer->pHead; - while (first != NULL) { - tFilePagesItem *ptmp = first; - first = first->pNext; - tfree(ptmp); - } - - pMemBuffer->fileMeta.numOfElemsInFile = 0; - pMemBuffer->fileMeta.nFileSize = 0; - - pMemBuffer->numOfElemsInBuffer = 0; - pMemBuffer->numOfInMemPages = 0; - - pMemBuffer->pHead = NULL; - pMemBuffer->pTail = NULL; - - tExtMemBufferClearFlushoutInfo(pMemBuffer); - - // reset the write pointer to the header - if (pMemBuffer->file != NULL) { - fseek(pMemBuffer->file, 0, SEEK_SET); - } -} - -bool tExtMemBufferLoadData(tExtMemBuffer *pMemBuffer, tFilePage *pFilePage, int32_t flushoutId, int32_t pageIdx) { - if (flushoutId < 0 || flushoutId > (int32_t)pMemBuffer->fileMeta.flushoutData.nLength) { - return false; - } - - tFlushoutInfo *pInfo = &(pMemBuffer->fileMeta.flushoutData.pFlushoutInfo[flushoutId]); - if (pageIdx > (int32_t)pInfo->numOfPages) { - return false; - } - - size_t ret = fseek(pMemBuffer->file, (pInfo->startPageId + pageIdx) * pMemBuffer->pageSize, SEEK_SET); - ret = fread(pFilePage, pMemBuffer->pageSize, 1, pMemBuffer->file); - - return (ret > 0); -} - -bool tExtMemBufferIsAllDataInMem(tExtMemBuffer *pMemBuffer) { return (pMemBuffer->fileMeta.nFileSize == 0); } - -////////////////////////////////////////////////////////////////////////////////////////////////////////////// -static FORCE_INLINE int32_t primaryKeyComparator(int64_t f1, int64_t f2, int32_t colIdx, int32_t tsOrder) { - if (f1 == f2) { - return 0; - } - - if (tsOrder == TSDB_ORDER_DESC) { // primary column desc order - return (f1 < f2) ? 1 : -1; - } else { // asc - return (f1 < f2) ? -1 : 1; - } -} - -static int32_t tsCompareFunc(TSKEY k1, TSKEY k2, int32_t order) { - if (k1 == k2) { - return 0; - } - - if (order == TSDB_ORDER_DESC) { - return (k1 < k2)? 1:-1; - } else { - return (k1 < k2)? -1:1; - } -} - -int32_t columnValueAscendingComparator(char *f1, char *f2, int32_t type, int32_t bytes) { - switch (type) { - case TSDB_DATA_TYPE_INT: DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); - case TSDB_DATA_TYPE_DOUBLE: DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); - case TSDB_DATA_TYPE_FLOAT: DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); - case TSDB_DATA_TYPE_BIGINT: DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2)); - case TSDB_DATA_TYPE_SMALLINT:DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2)); - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2)); - - case TSDB_DATA_TYPE_BINARY: { - int32_t len1 = varDataLen(f1); - int32_t len2 = varDataLen(f2); - - if (len1 != len2) { - return len1 > len2? 1:-1; - } else { - int32_t ret = strncmp(varDataVal(f1), varDataVal(f2), len1); - if (ret == 0) { - return 0; - } - return (ret < 0) ? -1 : 1; - } - - }; - case TSDB_DATA_TYPE_NCHAR: { // todo handle the var string compare - int32_t len1 = varDataLen(f1); - int32_t len2 = varDataLen(f2); - - if (len1 != len2) { - return len1 > len2 ? 1 : -1; - } else { - int32_t ret = tasoUcs4Compare(varDataVal(f1), varDataVal(f2), len1, TSDB_NCHAR_SIZE); - if (ret == 0) { - return 0; - } - return (ret < 0) ? -1 : 1; - } - }; - case TSDB_DATA_TYPE_UTINYINT: DEFAULT_COMP(GET_UINT8_VAL(f1), GET_UINT8_VAL(f2)); - case TSDB_DATA_TYPE_USMALLINT: DEFAULT_COMP(GET_UINT16_VAL(f1), GET_UINT16_VAL(f2)); - case TSDB_DATA_TYPE_UINT: DEFAULT_COMP(GET_UINT32_VAL(f1), GET_UINT32_VAL(f2)); - case TSDB_DATA_TYPE_UBIGINT: DEFAULT_COMP(GET_UINT64_VAL(f1), GET_UINT64_VAL(f2)); - } - - return 0; -} - -int32_t compare_a(tOrderDescriptor *pDescriptor, int32_t numOfRows1, int32_t s1, char *data1, int32_t numOfRows2, - int32_t s2, char *data2) { - assert(numOfRows1 == numOfRows2); - - int32_t cmpCnt = pDescriptor->orderInfo.numOfCols; - for (int32_t i = 0; i < cmpCnt; ++i) { - int32_t colIdx = pDescriptor->orderInfo.colIndex[i]; - - char *f1 = COLMODEL_GET_VAL(data1, pDescriptor->pColumnModel, numOfRows1, s1, colIdx); - char *f2 = COLMODEL_GET_VAL(data2, pDescriptor->pColumnModel, numOfRows2, s2, colIdx); - - if (pDescriptor->pColumnModel->pFields[colIdx].field.type == TSDB_DATA_TYPE_TIMESTAMP) { - int32_t ret = primaryKeyComparator(*(int64_t *)f1, *(int64_t *)f2, colIdx, pDescriptor->tsOrder); - if (ret == 0) { - continue; - } else { - return ret; - } - } else { - SSchemaEx *pSchema = &pDescriptor->pColumnModel->pFields[colIdx]; - int32_t ret = columnValueAscendingComparator(f1, f2, pSchema->field.type, pSchema->field.bytes); - if (ret == 0) { - continue; - } else { - return ret; - } - } - } - - return 0; -} - -int32_t compare_aRv(SSDataBlock* pBlock, SArray* colIndex, int32_t numOfCols, int32_t rowIndex, char** buffer, int32_t order) { - for (int32_t i = 0; i < numOfCols; ++i) { - SColIndex* pColIndex = taosArrayGet(colIndex, i); - int32_t index = pColIndex->colIndex; - - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, index); - assert(pColIndex->colId == pColInfo->info.colId); - - char* data = pColInfo->pData + rowIndex * pColInfo->info.bytes; - if (pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - int32_t ret = tsCompareFunc(GET_INT64_VAL(data), GET_INT64_VAL(buffer[i]), order); - if (ret == 0) { - continue; // The timestamps are identical - } else { - return ret; - } - } else { - int32_t ret = columnValueAscendingComparator(data, buffer[i], pColInfo->info.type, pColInfo->info.bytes); - if (ret == 0) { - continue; - } else { - return ret; - } - } - } - - return 0; -} - -int32_t compare_d(tOrderDescriptor *pDescriptor, int32_t numOfRows1, int32_t s1, char *data1, int32_t numOfRows2, - int32_t s2, char *data2) { - assert(numOfRows1 == numOfRows2); - - int32_t cmpCnt = pDescriptor->orderInfo.numOfCols; - for (int32_t i = 0; i < cmpCnt; ++i) { - int32_t colIdx = pDescriptor->orderInfo.colIndex[i]; - - char *f1 = COLMODEL_GET_VAL(data1, pDescriptor->pColumnModel, numOfRows1, s1, colIdx); - char *f2 = COLMODEL_GET_VAL(data2, pDescriptor->pColumnModel, numOfRows2, s2, colIdx); - - if (pDescriptor->pColumnModel->pFields[colIdx].field.type == TSDB_DATA_TYPE_TIMESTAMP) { - int32_t ret = primaryKeyComparator(*(int64_t *)f1, *(int64_t *)f2, colIdx, pDescriptor->tsOrder); - if (ret == 0) { - continue; - } else { - return ret; - } - } else { - SSchemaEx *pSchema = &pDescriptor->pColumnModel->pFields[colIdx]; - int32_t ret = columnValueAscendingComparator(f1, f2, pSchema->field.type, pSchema->field.bytes); - if (ret == 0) { - continue; - } else { - return -ret; // descending order - } - } - } - - return 0; -} -FORCE_INLINE int32_t compare_sa(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t idx1, int32_t idx2, - char *data) { - return compare_a(pDescriptor, numOfRows, idx1, data, numOfRows, idx2, data); -} - -FORCE_INLINE int32_t compare_sd(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t idx1, int32_t idx2, - char *data) { - return compare_d(pDescriptor, numOfRows, idx1, data, numOfRows, idx2, data); -} - -static void swap(SColumnModel *pColumnModel, int32_t count, int32_t s1, char *data1, int32_t s2, void* buf) { - for (int32_t i = 0; i < pColumnModel->numOfCols; ++i) { - void *first = COLMODEL_GET_VAL(data1, pColumnModel, count, s1, i); - void *second = COLMODEL_GET_VAL(data1, pColumnModel, count, s2, i); - - SSchema* pSchema = &pColumnModel->pFields[i].field; - tsDataSwap(first, second, pSchema->type, pSchema->bytes, buf); - } -} - -static void tColDataInsertSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, - __col_compar_fn_t compareFn, void* buf) { - for (int32_t i = start + 1; i <= end; ++i) { - for (int32_t j = i; j > start; --j) { - if (compareFn(pDescriptor, numOfRows, j, j - 1, data) == -1) { - swap(pDescriptor->pColumnModel, numOfRows, j - 1, data, j, buf); - } else { - break; - } - } - } -} - -static void UNUSED_FUNC tSortDataPrint(int32_t type, char *prefix, char *startx, char *midx, char *endx) { - switch (type) { - case TSDB_DATA_TYPE_INT: - printf("%s:(%d, %d, %d)\n", prefix, *(int32_t *)startx, *(int32_t *)midx, *(int32_t *)endx); - break; - case TSDB_DATA_TYPE_TINYINT: - printf("%s:(%d, %d, %d)\n", prefix, *(int8_t *)startx, *(int8_t *)midx, *(int8_t *)endx); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%s:(%d, %d, %d)\n", prefix, *(int16_t *)startx, *(int16_t *)midx, *(int16_t *)endx); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - printf("%s:(%" PRId64 ", %" PRId64 ", %" PRId64 ")\n", prefix, *(int64_t *)startx, *(int64_t *)midx, *(int64_t *)endx); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%s:(%f, %f, %f)\n", prefix, *(float *)startx, *(float *)midx, *(float *)endx); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf("%s:(%lf, %lf, %lf)\n", prefix, *(double *)startx, *(double *)midx, *(double *)endx); - break; - case TSDB_DATA_TYPE_BINARY: - printf("%s:(%s, %s, %s)\n", prefix, startx, midx, endx); - break; - } -} - -static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, - __col_compar_fn_t compareFn, void* buf) { - int32_t midIdx = ((end - start) >> 1) + start; - -#if defined(_DEBUG_VIEW) - int32_t f = pDescriptor->orderInfo.colIndex[0]; - - char *midx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, midIdx, f); - char *startx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, start, f); - char *endx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, end, f); - - int32_t colIdx = pDescriptor->orderInfo.colIndex[0]; - tSortDataPrint(pDescriptor->pColumnModel->pFields[colIdx].field.type, "before", startx, midx, endx); -#endif - - SColumnModel* pModel = pDescriptor->pColumnModel; - if (compareFn(pDescriptor, numOfRows, midIdx, start, data) == 1) { - swap(pModel, numOfRows, start, data, midIdx, buf); - } - - if (compareFn(pDescriptor, numOfRows, midIdx, end, data) == 1) { - swap(pModel, numOfRows, midIdx, data, start, buf); - swap(pModel, numOfRows, midIdx, data, end, buf); - } else if (compareFn(pDescriptor, numOfRows, start, end, data) == 1) { - swap(pModel, numOfRows, start, data, end, buf); - } - - assert(compareFn(pDescriptor, numOfRows, midIdx, start, data) <= 0 && - compareFn(pDescriptor, numOfRows, start, end, data) <= 0); - -#if defined(_DEBUG_VIEW) - midx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, midIdx, f); - startx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, start, f); - endx = COLMODEL_GET_VAL(data, pDescriptor->pColumnModel, numOfRows, end, f); - tSortDataPrint(pDescriptor->pColumnModel->pFields[colIdx].field.type, "after", startx, midx, endx); -#endif -} - -static UNUSED_FUNC void tRowModelDisplay(tOrderDescriptor *pDescriptor, int32_t numOfRows, char *d, int32_t len) { - int32_t colIdx = pDescriptor->orderInfo.colIndex[0]; - - for (int32_t i = 0; i < len; ++i) { - char *startx = COLMODEL_GET_VAL(d, pDescriptor->pColumnModel, numOfRows, i, colIdx); - - switch (pDescriptor->pColumnModel->pFields[colIdx].field.type) { - case TSDB_DATA_TYPE_DOUBLE: - printf("%lf\t", *(double *)startx); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)startx); - break; - case TSDB_DATA_TYPE_INT: - printf("%d\t", *(int32_t *)startx); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%d\t", *(int16_t *)startx); - break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BOOL: - printf("%d\t", *(int8_t *)startx); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - printf("%" PRId64 "\t", *(int64_t *)startx); - break; - case TSDB_DATA_TYPE_BINARY: - printf("%s\t", startx); - break; - default: - assert(false); - } - } - printf("\n"); -} - -static void columnwiseQSortImpl(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, - int32_t orderType, __col_compar_fn_t compareFn, void* buf) { -#ifdef _DEBUG_VIEW - printf("before sort:\n"); - tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); -#endif - - int32_t s = start, e = end; - median(pDescriptor, numOfRows, start, end, data, compareFn, buf); - -#ifdef _DEBUG_VIEW - // printf("%s called: %d\n", __FUNCTION__, qsort_call++); -#endif - - int32_t end_same = end; - int32_t start_same = start; - - while (s < e) { - while (e > s) { - int32_t ret = compareFn(pDescriptor, numOfRows, e, s, data); - if (ret < 0) { - break; - } - - if (ret == 0 && e != end_same) { - swap(pDescriptor->pColumnModel, numOfRows, e, data, end_same--, buf); - } - e--; - } - - if (e != s) { - swap(pDescriptor->pColumnModel, numOfRows, s, data, e, buf); - } - -#ifdef _DEBUG_VIEW - // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); -#endif - - while (s < e) { - int32_t ret = compareFn(pDescriptor, numOfRows, s, e, data); - if (ret > 0) { - break; - } - - if (ret == 0 && s != start_same) { - swap(pDescriptor->pColumnModel, numOfRows, s, data, start_same++, buf); - } - s++; - } - - if (s != e) { - swap(pDescriptor->pColumnModel, numOfRows, s, data, e, buf); - } -#ifdef _DEBUG_VIEW - // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); -#endif - } - - int32_t rightx = e + 1; - if (end_same != end && e < end) { // move end data to around the pivotal - int32_t left = e + 1; - int32_t right = end; - - while (right > end_same && left <= end_same) { - swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--, buf); - } - - // (pivotal+1) + steps of number that are identical pivotal - rightx += (end - end_same); - -#ifdef _DEBUG_VIEW - // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); -#endif - } - - int32_t leftx = e - 1; - if (start_same != start && s > start) { - int32_t left = start; - int32_t right = e - 1; - - while (left < start_same && right >= start_same) { - swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--, buf); - } - - // (pivotal-1) - steps of number that are identical pivotal - leftx -= (start_same - start); - -#ifdef _DEBUG_VIEW - // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); -#endif - } - - if (leftx > start) { - columnwiseQSortImpl(pDescriptor, numOfRows, start, leftx, data, orderType, compareFn, buf); - } - - if (rightx < end) { - columnwiseQSortImpl(pDescriptor, numOfRows, rightx, end, data, orderType, compareFn, buf); - } -} - -void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, int32_t order) { - // short array sort, incur another sort procedure instead of quick sort process - __col_compar_fn_t compareFn = (order == TSDB_ORDER_ASC) ? compare_sa : compare_sd; - - SColumnModel* pModel = pDescriptor->pColumnModel; - - size_t width = 0; - for(int32_t i = 0; i < pModel->numOfCols; ++i) { - SSchema* pSchema = &pModel->pFields[i].field; - if (width < pSchema->bytes) { - width = pSchema->bytes; - } - } - - char* buf = malloc(width); - assert(width > 0 && buf != NULL); - - if (end - start + 1 <= 8) { - tColDataInsertSort(pDescriptor, numOfRows, start, end, data, compareFn, buf); - } else { - columnwiseQSortImpl(pDescriptor, numOfRows, start, end, data, order, compareFn, buf); - } - - free(buf); -} - -/* - * deep copy of sschema - */ -SColumnModel *createColumnModel(SSchema *fields, int32_t numOfCols, int32_t blockCapacity) { - SColumnModel *pColumnModel = (SColumnModel *)calloc(1, sizeof(SColumnModel) + numOfCols * sizeof(SSchemaEx)); - if (pColumnModel == NULL) { - return NULL; - } - - pColumnModel->pFields = (SSchemaEx *)(&pColumnModel[1]); - - for(int32_t i = 0; i < numOfCols; ++i) { - SSchemaEx* pSchemaEx = &pColumnModel->pFields[i]; - pSchemaEx->field = fields[i]; - pSchemaEx->offset = pColumnModel->rowSize; - - pColumnModel->rowSize += pSchemaEx->field.bytes; - } - - pColumnModel->numOfCols = numOfCols; - pColumnModel->capacity = blockCapacity; - - return pColumnModel; -} - -SColumnModel *cloneColumnModel(SColumnModel *pSrc) { - if (pSrc == NULL) { - return NULL; - } - - SColumnModel *pColumnModel = (SColumnModel *)calloc(1, sizeof(SColumnModel) + pSrc->numOfCols * sizeof(SSchemaEx)); - if (pColumnModel == NULL) { - return NULL; - } - - *pColumnModel = *pSrc; - - pColumnModel->pFields = (SSchemaEx*) (&pColumnModel[1]); - memcpy(pColumnModel->pFields, pSrc->pFields, pSrc->numOfCols * sizeof(SSchemaEx)); - - return pColumnModel; -} - -void destroyColumnModel(SColumnModel *pModel) { - if (pModel == NULL) { - return; - } - - tfree(pModel); -} - -static void printBinaryData(char *data, int32_t len) { - bool isCharString = true; - for (int32_t i = 0; i < len; ++i) { - if ((data[i] <= 'Z' && data[i] >= 'A') || (data[i] <= 'z' && data[i] >= 'a') || - (data[i] >= '0' && data[i] <= '9')) { - continue; - } else if (data[i] == 0) { - break; - } else { - isCharString = false; - break; - } - } - - if (len == 50) { // probably the avg intermediate result - printf("%lf,%" PRId64 "\t", *(double *)data, *(int64_t *)(data + sizeof(double))); - } else if (data[8] == ',') { // in TSDB_FUNC_FIRST_DST/TSDB_FUNC_LAST_DST, - // the value is seperated by ',' - //printf("%" PRId64 ",%0x\t", *(int64_t *)data, data + sizeof(int64_t) + 1); - printf("%" PRId64 ", HEX: ", *(int64_t *)data); - int32_t tmp_len = len - sizeof(int64_t) - 1; - for (int32_t i = 0; i < tmp_len; ++i) { - printf("%0x ", *(data + sizeof(int64_t) + 1 + i)); - } - printf("\t"); - } else if (isCharString) { - printf("%s\t", data); - } -} - -// todo cast to struct to extract data -static void printBinaryDataEx(char *data, int32_t len, SSrcColumnInfo *param) { - if (param->functionId == TSDB_FUNC_LAST_DST) { - switch (param->type) { - case TSDB_DATA_TYPE_TINYINT:printf("%" PRId64 ",%d\t", *(int64_t *) data, *(int8_t *) (data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_SMALLINT:printf("%" PRId64 ",%d\t", *(int64_t *) data, *(int16_t *) (data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT:printf("%" PRId64 ",%" PRId64 "\t", *(int64_t *) data, *(int64_t *) (data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_FLOAT:printf("%" PRId64 ",%f\t", *(int64_t *) data, *(float *) (data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_DOUBLE:printf("%" PRId64 ",%f\t", *(int64_t *) data, *(double *) (data + TSDB_KEYSIZE + 1)); - break; - case TSDB_DATA_TYPE_BINARY:printf("%" PRId64 ",%s\t", *(int64_t *) data, (data + TSDB_KEYSIZE + 1)); - break; - - case TSDB_DATA_TYPE_INT: - default:printf("%" PRId64 ",%d\t", *(int64_t *) data, *(int32_t *) (data + TSDB_KEYSIZE + 1)); - break; - } - } else if (param->functionId == TSDB_FUNC_AVG) { - printf("%f,%" PRId64 "\t", *(double *) data, *(int64_t *) (data + sizeof(double) + 1)); - } else { - // functionId == TSDB_FUNC_MAX_DST | TSDB_FUNC_TAG - switch (param->type) { - case TSDB_DATA_TYPE_TINYINT: - printf("%d\t", *(int8_t *)data); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%d\t", *(int16_t *)data); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - printf("%" PRId64 "\t", *(int64_t *)data); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)data); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf("%f\t", *(double *)data); - break; - case TSDB_DATA_TYPE_BINARY: - printf("%s\t", data); - break; - - case TSDB_DATA_TYPE_INT: - default: - printf("%f\t", *(double *)data); - break; - } - } -} - -void tColModelDisplay(SColumnModel *pModel, void *pData, int32_t numOfRows, int32_t totalCapacity) { - for (int32_t i = 0; i < numOfRows; ++i) { - for (int32_t j = 0; j < pModel->numOfCols; ++j) { - char *val = COLMODEL_GET_VAL((char *)pData, pModel, totalCapacity, i, j); - - int type = pModel->pFields[j].field.type; - printf("type:%d ", type); - - switch (type) { - case TSDB_DATA_TYPE_BIGINT: - printf("%" PRId64 "\t", *(int64_t *)val); - break; - case TSDB_DATA_TYPE_INT: - printf("%d\t", *(int32_t *)val); - break; - case TSDB_DATA_TYPE_NCHAR: { - char buf[4096] = {0}; - taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf); - printf("%s\t", buf); - break; - } - case TSDB_DATA_TYPE_BINARY: { - printBinaryData(val, pModel->pFields[j].field.bytes); - break; - } - case TSDB_DATA_TYPE_DOUBLE: - printf("%lf\t", *(double *)val); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - printf("%" PRId64 "\t", *(int64_t *)val); - break; - case TSDB_DATA_TYPE_TINYINT: - printf("%d\t", *(int8_t *)val); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%d\t", *(int16_t *)val); - break; - case TSDB_DATA_TYPE_BOOL: - printf("%d\t", *(int8_t *)val); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)val); - break; - default: - assert(false); - } - } - printf("\n"); - } - printf("\n"); -} - -void tColModelDisplayEx(SColumnModel *pModel, void *pData, int32_t numOfRows, int32_t totalCapacity, - SSrcColumnInfo *param) { - for (int32_t i = 0; i < numOfRows; ++i) { - for (int32_t j = 0; j < pModel->numOfCols; ++j) { - char *val = COLMODEL_GET_VAL((char *)pData, pModel, totalCapacity, i, j); - - printf("type:%d\t", pModel->pFields[j].field.type); - - switch (pModel->pFields[j].field.type) { - case TSDB_DATA_TYPE_BIGINT: - printf("%" PRId64 "\t", *(int64_t *)val); - break; - case TSDB_DATA_TYPE_INT: - printf("%d\t", *(int32_t *)val); - break; - case TSDB_DATA_TYPE_NCHAR: { - char buf[128] = {0}; - taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf); - printf("%s\t", buf); - break; - } - case TSDB_DATA_TYPE_BINARY: { - printBinaryDataEx(val, pModel->pFields[j].field.bytes, ¶m[j]); - break; - } - case TSDB_DATA_TYPE_DOUBLE: - printf("%lf\t", *(double *)val); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - printf("%" PRId64 "\t", *(int64_t *)val); - break; - case TSDB_DATA_TYPE_TINYINT: - printf("%d\t", *(int8_t *)val); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%d\t", *(int16_t *)val); - break; - case TSDB_DATA_TYPE_BOOL: - printf("%d\t", *(int8_t *)val); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)val); - break; - default: - assert(false); - } - } - printf("\n"); - } - printf("\n"); -} - -//////////////////////////////////////////////////////////////////////////////////////////// -void tColModelCompact(SColumnModel *pModel, tFilePage *inputBuffer, int32_t maxElemsCapacity) { - if (inputBuffer->num == 0 || maxElemsCapacity == inputBuffer->num) { - return; - } - - /* start from the second column */ - for (int32_t i = 1; i < pModel->numOfCols; ++i) { - SSchemaEx* pSchemaEx = &pModel->pFields[i]; - memmove(inputBuffer->data + pSchemaEx->offset * inputBuffer->num, - inputBuffer->data + pSchemaEx->offset * maxElemsCapacity, - (size_t)(pSchemaEx->field.bytes * inputBuffer->num)); - } -} - -SSchema* getColumnModelSchema(SColumnModel *pColumnModel, int32_t index) { - assert(pColumnModel != NULL && index >= 0 && index < pColumnModel->numOfCols); - return &pColumnModel->pFields[index].field; -} - -int16_t getColumnModelOffset(SColumnModel *pColumnModel, int32_t index) { - assert(pColumnModel != NULL && index >= 0 && index < pColumnModel->numOfCols); - return pColumnModel->pFields[index].offset; -} - -void tColModelErase(SColumnModel *pModel, tFilePage *inputBuffer, int32_t blockCapacity, int32_t s, int32_t e) { - if (inputBuffer->num == 0 || (e - s + 1) <= 0) { - return; - } - - int32_t removed = e - s + 1; - int32_t remain = (int32_t)inputBuffer->num - removed; - int32_t secPart = (int32_t)inputBuffer->num - e - 1; - - /* start from the second column */ - for (int32_t i = 0; i < pModel->numOfCols; ++i) { - int16_t offset = getColumnModelOffset(pModel, i); - SSchema* pSchema = getColumnModelSchema(pModel, i); - - char *startPos = inputBuffer->data + offset * blockCapacity + s * pSchema->bytes; - char *endPos = startPos + pSchema->bytes * removed; - - memmove(startPos, endPos, pSchema->bytes * secPart); - } - - inputBuffer->num = remain; -} - -/* - * column format data block append function - * used in write record(s) to exist column-format block - * - * data in srcData must has the same schema as data in dstPage, that can be - * described by dstModel - */ -void tColModelAppend(SColumnModel *dstModel, tFilePage *dstPage, void *srcData, int32_t start, int32_t numOfRows, - int32_t srcCapacity) { - assert(dstPage->num + numOfRows <= dstModel->capacity); - - for (int32_t col = 0; col < dstModel->numOfCols; ++col) { - char *dst = COLMODEL_GET_VAL(dstPage->data, dstModel, dstModel->capacity, dstPage->num, col); - char *src = COLMODEL_GET_VAL((char *)srcData, dstModel, srcCapacity, start, col); - - memmove(dst, src, dstModel->pFields[col].field.bytes * numOfRows); - } - - dstPage->num += numOfRows; -} - -tOrderDescriptor *tOrderDesCreate(const int32_t *orderColIdx, int32_t numOfOrderCols, SColumnModel *pModel, - int32_t tsOrderType) { - tOrderDescriptor *desc = (tOrderDescriptor *)calloc(1, sizeof(tOrderDescriptor) + sizeof(int32_t) * numOfOrderCols); - if (desc == NULL) { - return NULL; - } - - desc->pColumnModel = pModel; - desc->tsOrder = tsOrderType; - - desc->orderInfo.numOfCols = numOfOrderCols; - for (int32_t i = 0; i < numOfOrderCols; ++i) { - desc->orderInfo.colIndex[i] = orderColIdx[i]; - } - - return desc; -} - -void tOrderDescDestroy(tOrderDescriptor *pDesc) { - if (pDesc == NULL) { - return; - } - - destroyColumnModel(pDesc->pColumnModel); - tfree(pDesc); -} - -void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn) { - assert(numOfRows > 0 && numOfCols > 0 && index >= 0 && index < numOfCols); - - int32_t bytes = pSchema[index].bytes; - int32_t size = bytes + sizeof(int32_t); - - char* buf = calloc(1, size * numOfRows); - - for(int32_t i = 0; i < numOfRows; ++i) { - char* dest = buf + size * i; - memcpy(dest, ((char*) pCols[index]) + bytes * i, bytes); - *(int32_t*)(dest+bytes) = i; - } - - qsort(buf, numOfRows, size, compareFn); - - int32_t prevLength = 0; - char* p = NULL; - - for(int32_t i = 0; i < numOfCols; ++i) { - int32_t bytes1 = pSchema[i].bytes; - - if (i == index) { - for(int32_t j = 0; j < numOfRows; ++j){ - char* src = buf + (j * size); - char* dest = ((char*)pCols[i]) + (j * bytes1); - memcpy(dest, src, bytes1); - } - } else { - // make sure memory buffer is enough - if (prevLength < bytes1) { - char *tmp = realloc(p, bytes1 * numOfRows); - assert(tmp); - - p = tmp; - prevLength = bytes1; - } - - memcpy(p, pCols[i], bytes1 * numOfRows); - - for(int32_t j = 0; j < numOfRows; ++j){ - char* dest = ((char*)pCols[i]) + bytes1 * j; - - int32_t newPos = *(int32_t*)(buf + (j * size) + bytes); - char* src = p + (newPos * bytes1); - memcpy(dest, src, bytes1); - } - } - } - - tfree(buf); - tfree(p); -} diff --git a/2.0/src/query/src/qFill.c b/2.0/src/query/src/qFill.c deleted file mode 100644 index 47ea63eeb0bd735ecba7811d767f121e4dd05415..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qFill.c +++ /dev/null @@ -1,523 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" - -#include "qAggMain.h" -#include "taosdef.h" -#include "tmsg.h" -#include "ttype.h" - -#include "qFill.h" -#include "qExtbuffer.h" -#include "queryLog.h" -#include "qExecutor.h" - -#define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC) -#define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1)))) - -static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { - for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) { - SFillColInfo* pCol = &pFillInfo->pFillCol[j]; - if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || TSDB_COL_IS_UD_COL(pCol->flag)) { - continue; - } - - char* val1 = elePtrAt(data[j], pCol->col.bytes, genRows); - - assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags); - SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - - assert (pTag->col.colId == pCol->col.colId); - assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type); - } -} - -static void setNullValueForRow(SFillInfo* pFillInfo, void** data, int32_t numOfCol, int32_t rowIndex) { - // the first are always the timestamp column, so start from the second column. - for (int32_t i = 1; i < numOfCol; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - - char* output = elePtrAt(data[i], pCol->col.bytes, rowIndex); - setNull(output, pCol->col.type, pCol->col.bytes); - } -} - -static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData, int64_t ts, bool outOfBound) { - char* prev = pFillInfo->prevValues; - char* next = pFillInfo->nextValues; - - SPoint point1, point2, point; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); - - // set the primary timestamp column value - int32_t index = pFillInfo->numOfCurrent; - char* val = elePtrAt(data[0], TSDB_KEYSIZE, index); - *(TSKEY*) val = pFillInfo->currentKey; - - // set the other values - if (pFillInfo->type == TSDB_FILL_PREV) { - char* p = FILL_IS_ASC_FILL(pFillInfo) ? prev : next; - - if (p != NULL) { - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)) { - continue; - } - - char* output = elePtrAt(data[i], pCol->col.bytes, index); - assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type); - } - } else { // no prev value yet, set the value for NULL - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); - } - } else if (pFillInfo->type == TSDB_FILL_NEXT) { - char* p = FILL_IS_ASC_FILL(pFillInfo)? next : prev; - - if (p != NULL) { - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)) { - continue; - } - - char* output = elePtrAt(data[i], pCol->col.bytes, index); - assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type); - } - } else { // no prev value yet, set the value for NULL - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); - } - } else if (pFillInfo->type == TSDB_FILL_LINEAR) { - // TODO : linear interpolation supports NULL value - if (prev != NULL && !outOfBound) { - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)) { - continue; - } - - int16_t type = pCol->col.type; - int16_t bytes = pCol->col.bytes; - - char *val1 = elePtrAt(data[i], pCol->col.bytes, index); - if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) { - setNull(val1, pCol->col.type, bytes); - continue; - } - - point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->col.offset}; - point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes}; - point = (SPoint){.key = pFillInfo->currentKey, .val = val1}; - taosGetLinearInterpolationVal(&point, type, &point1, &point2, type); - } - } else { - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); - } - } else { // fill the default value */ - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) { - continue; - } - - char* val1 = elePtrAt(data[i], pCol->col.bytes, index); - assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type); - } - } - - setTagsValue(pFillInfo, data, index); - pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision); - pFillInfo->numOfCurrent++; -} - -static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) { - if (*next != NULL) { - return; - } - - *next = calloc(1, pFillInfo->rowSize); - for (int i = 1; i < pFillInfo->numOfCols; i++) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - setNull(*next + pCol->col.offset, pCol->col.type, pCol->col.bytes); - } -} - -static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* buf) { - int32_t rowIndex = pFillInfo->index; - for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - memcpy(buf + pCol->col.offset, srcData[i] + rowIndex * pCol->col.bytes, pCol->col.bytes); - } -} - -static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputRows) { - pFillInfo->numOfCurrent = 0; - - char** srcData = pFillInfo->pData; - char** prev = &pFillInfo->prevValues; - char** next = &pFillInfo->nextValues; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); - - if (FILL_IS_ASC_FILL(pFillInfo)) { - assert(pFillInfo->currentKey >= pFillInfo->start); - } else { - assert(pFillInfo->currentKey <= pFillInfo->start); - } - - while (pFillInfo->numOfCurrent < outputRows) { - int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->index]; - - // set the next value for interpolation - if ((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || - (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) { - initBeforeAfterDataBuf(pFillInfo, next); - copyCurrentRowIntoBuf(pFillInfo, srcData, *next); - } - - if (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) && - pFillInfo->numOfCurrent < outputRows) { - - // fill the gap between two actual input rows - while (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || - (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) && - pFillInfo->numOfCurrent < outputRows) { - doFillOneRowResult(pFillInfo, data, srcData, ts, false); - } - - // output buffer is full, abort - if (pFillInfo->numOfCurrent == outputRows) { - pFillInfo->numOfTotal += pFillInfo->numOfCurrent; - return outputRows; - } - } else { - assert(pFillInfo->currentKey == ts); - initBeforeAfterDataBuf(pFillInfo, prev); - if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) { - initBeforeAfterDataBuf(pFillInfo, next); - ++pFillInfo->index; - copyCurrentRowIntoBuf(pFillInfo, srcData, *next); - --pFillInfo->index; - } - - // assign rows to dst buffer - for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) { - continue; - } - - char* output = elePtrAt(data[i], pCol->col.bytes, pFillInfo->numOfCurrent); - char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index); - - if (i == 0 || (pCol->functionId != TSDB_FUNC_COUNT && !isNull(src, pCol->col.type)) || - (pCol->functionId == TSDB_FUNC_COUNT && GET_INT64_VAL(src) != 0)) { - assignVal(output, src, pCol->col.bytes, pCol->col.type); - memcpy(*prev + pCol->col.offset, src, pCol->col.bytes); - } else { // i > 0 and data is null , do interpolation - if (pFillInfo->type == TSDB_FILL_PREV) { - assignVal(output, *prev + pCol->col.offset, pCol->col.bytes, pCol->col.type); - } else if (pFillInfo->type == TSDB_FILL_LINEAR) { - assignVal(output, src, pCol->col.bytes, pCol->col.type); - memcpy(*prev + pCol->col.offset, src, pCol->col.bytes); - } else if (pFillInfo->type == TSDB_FILL_NEXT) { - if (*next) { - assignVal(output, *next + pCol->col.offset, pCol->col.bytes, pCol->col.type); - } else { - setNull(output, pCol->col.type, pCol->col.bytes); - } - } else { - assignVal(output, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type); - } - } - } - - // set the tag value for final result - setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent); - - pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, - pFillInfo->interval.slidingUnit, pFillInfo->precision); - pFillInfo->index += 1; - pFillInfo->numOfCurrent += 1; - } - - if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) { - /* the raw data block is exhausted, next value does not exists */ - if (pFillInfo->index >= pFillInfo->numOfRows) { - tfree(*next); - } - - pFillInfo->numOfTotal += pFillInfo->numOfCurrent; - return pFillInfo->numOfCurrent; - } - } - - return pFillInfo->numOfCurrent; -} - -static int64_t appendFilledResult(SFillInfo* pFillInfo, void** output, int64_t resultCapacity) { - /* - * These data are generated according to fill strategy, since the current timestamp is out of the time window of - * real result set. Note that we need to keep the direct previous result rows, to generated the filled data. - */ - pFillInfo->numOfCurrent = 0; - while (pFillInfo->numOfCurrent < resultCapacity) { - doFillOneRowResult(pFillInfo, output, pFillInfo->pData, pFillInfo->start, true); - } - - pFillInfo->numOfTotal += pFillInfo->numOfCurrent; - - assert(pFillInfo->numOfCurrent == resultCapacity); - return resultCapacity; -} - -// there are no duplicated tags in the SFillTagColInfo list -static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t capacity) { - int32_t rowsize = 0; - int32_t numOfTags = 0; - - int32_t k = 0; - for (int32_t i = 0; i < numOfCols; ++i) { - SFillColInfo* pColInfo = &pFillInfo->pFillCol[i]; - pFillInfo->pData[i] = NULL; - - if (TSDB_COL_IS_TAG(pColInfo->flag) || pColInfo->col.type == TSDB_DATA_TYPE_BINARY) { - numOfTags += 1; - - bool exists = false; - int32_t index = -1; - for (int32_t j = 0; j < k; ++j) { - if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) { - exists = true; - index = j; - break; - } - } - - if (!exists) { - SSchema* pSchema = &pFillInfo->pTags[k].col; - pSchema->colId = pColInfo->col.colId; - pSchema->type = pColInfo->col.type; - pSchema->bytes = pColInfo->col.bytes; - - pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes); - pColInfo->tagIndex = k; - - k += 1; - } else { - pColInfo->tagIndex = index; - } - } - - rowsize += pColInfo->col.bytes; - } - - pFillInfo->numOfTags = numOfTags; - - assert(k <= pFillInfo->numOfTags); - return rowsize; -} - -static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { - if (pFillInfo->numOfRows == 0 || (pFillInfo->numOfRows > 0 && pFillInfo->index >= pFillInfo->numOfRows)) { - return 0; - } - - return pFillInfo->numOfRows - pFillInfo->index; -} - -SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, - int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - SFillColInfo* pCol, void* handle) { - if (fillType == TSDB_FILL_NONE) { - return NULL; - } - - SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo)); - taosResetFillInfo(pFillInfo, skey); - - pFillInfo->order = order; - pFillInfo->type = fillType; - pFillInfo->pFillCol = pCol; - pFillInfo->numOfTags = numOfTags; - pFillInfo->numOfCols = numOfCols; - pFillInfo->precision = precision; - pFillInfo->alloc = capacity; - pFillInfo->handle = handle; - - pFillInfo->interval.interval = slidingTime; - pFillInfo->interval.intervalUnit = slidingUnit; - pFillInfo->interval.sliding = slidingTime; - pFillInfo->interval.slidingUnit = slidingUnit; - - pFillInfo->pData = malloc(POINTER_BYTES * numOfCols); - -// if (numOfTags > 0) { - pFillInfo->pTags = calloc(numOfCols, sizeof(SFillTagColInfo)); - for (int32_t i = 0; i < numOfCols; ++i) { - pFillInfo->pTags[i].col.colId = -2; // TODO - } -// } - - pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc); - assert(pFillInfo->rowSize > 0); - - return pFillInfo; -} - -void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) { - pFillInfo->start = startTimestamp; - pFillInfo->currentKey = startTimestamp; - pFillInfo->end = startTimestamp; - pFillInfo->index = -1; - pFillInfo->numOfRows = 0; - pFillInfo->numOfCurrent = 0; - pFillInfo->numOfTotal = 0; -} - -void* taosDestroyFillInfo(SFillInfo* pFillInfo) { - if (pFillInfo == NULL) { - return NULL; - } - - tfree(pFillInfo->prevValues); - tfree(pFillInfo->nextValues); - - for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) { - tfree(pFillInfo->pTags[i].tagVal); - } - - tfree(pFillInfo->pTags); - - tfree(pFillInfo->pData); - tfree(pFillInfo->pFillCol); - - tfree(pFillInfo); - return NULL; -} - -void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) { - if (pFillInfo->type == TSDB_FILL_NONE) { - return; - } - - pFillInfo->end = endKey; - if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision); - } - - pFillInfo->index = 0; - pFillInfo->numOfRows = numOfRows; -} - -void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) { - for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - - SColumnInfoData* pColData = taosArrayGet(pInput->pDataBlock, i); - pFillInfo->pData[i] = pColData->pData; - - if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer - SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - assert (pTag->col.colId == pCol->col.colId); - memcpy(pTag->tagVal, pColData->pData, pCol->col.bytes); // TODO not memcpy?? - } - } -} - -bool taosFillHasMoreResults(SFillInfo* pFillInfo) { - int32_t remain = taosNumOfRemainRows(pFillInfo); - if (remain > 0) { - return true; - } - - if (pFillInfo->numOfTotal > 0 && (((pFillInfo->end > pFillInfo->start) && FILL_IS_ASC_FILL(pFillInfo)) || - (pFillInfo->end < pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo)))) { - return getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, 4096) > 0; - } - - return false; -} - -int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) { - int64_t* tsList = (int64_t*) pFillInfo->pData[0]; - - int32_t numOfRows = taosNumOfRemainRows(pFillInfo); - - TSKEY ekey1 = ekey; - if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision); - } - - int64_t numOfRes = -1; - if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set. - TSKEY lastKey = tsList[pFillInfo->numOfRows - 1]; - numOfRes = taosTimeCountInterval( - lastKey, - pFillInfo->currentKey, - pFillInfo->interval.sliding, - pFillInfo->interval.slidingUnit, - pFillInfo->precision); - numOfRes += 1; - assert(numOfRes >= numOfRows); - } else { // reach the end of data - if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || - (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { - return 0; - } - numOfRes = taosTimeCountInterval( - ekey1, - pFillInfo->currentKey, - pFillInfo->interval.sliding, - pFillInfo->interval.slidingUnit, - pFillInfo->precision); - numOfRes += 1; - } - - return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes; -} - -int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType) { - double v1 = -1, v2 = -1; - GET_TYPED_DATA(v1, double, inputType, point1->val); - GET_TYPED_DATA(v2, double, inputType, point2->val); - - double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key); - SET_TYPED_DATA(point->val, outputType, r); - - return TSDB_CODE_SUCCESS; -} - -int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity) { - int32_t remain = taosNumOfRemainRows(pFillInfo); - - int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity); - assert(numOfRes <= capacity); - - // no data existed for fill operation now, append result according to the fill strategy - if (remain == 0) { - appendFilledResult(pFillInfo, output, numOfRes); - } else { - fillResultImpl(pFillInfo, output, (int32_t) numOfRes); - assert(numOfRes == pFillInfo->numOfCurrent); - } - - qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%"PRId64"-%"PRId64", currentKey:%"PRId64", current:%d, total:%d, %p", - pFillInfo, pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey, pFillInfo->numOfCurrent, - pFillInfo->numOfTotal, pFillInfo->handle); - - return numOfRes; -} diff --git a/2.0/src/query/src/qFilter.c b/2.0/src/query/src/qFilter.c deleted file mode 100644 index a9bfe90d325865d1005ab407bfd497b7f850ef51..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qFilter.c +++ /dev/null @@ -1,3447 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "hash.h" -#include "os.h" -#include "queryLog.h" -#include "tcompare.h" -#include "tfilter.h" -#include "tscUtil.h" - -OptrStr gOptrStr[] = { - {TSDB_RELATION_INVALID, "invalid"}, - {TSDB_RELATION_LESS, "<"}, - {TSDB_RELATION_GREATER, ">"}, - {TSDB_RELATION_EQUAL, "="}, - {TSDB_RELATION_LESS_EQUAL, "<="}, - {TSDB_RELATION_GREATER_EQUAL, ">="}, - {TSDB_RELATION_NOT_EQUAL, "!="}, - {TSDB_RELATION_LIKE, "like"}, - {TSDB_RELATION_MATCH, "match"}, - {TSDB_RELATION_MATCH, "nmatch"}, - {TSDB_RELATION_ISNULL, "is null"}, - {TSDB_RELATION_NOTNULL, "not null"}, - {TSDB_RELATION_IN, "in"}, - {TSDB_RELATION_AND, "and"}, - {TSDB_RELATION_OR, "or"}, - {TSDB_RELATION_NOT, "not"} -}; - -static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) { - const SSchema *sch1 = desc1; - const SSchema *sch2 = desc2; - - return sch1->colId != sch2->colId; -} - -static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) { - const tVariant *val1 = desc1; - const tVariant *val2 = desc2; - - return tVariantCompare(val1, val2); -} - - -filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = { - NULL, - filterFieldColDescCompare, - filterFieldValDescCompare -}; - -bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) >= 0; -} -bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) > 0; -} -bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(minv, maxr) <= 0; -} -bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(minv, maxr) < 0; -} -bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) <= 0; -} -bool filterRangeCompee (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) > 0 && cfunc(minv, maxr) < 0; -} -bool filterRangeCompei (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) > 0 && cfunc(minv, maxr) <= 0; -} -bool filterRangeCompie (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) < 0; -} - -rangeCompFunc filterGetRangeCompFunc(char sflag, char eflag) { - if (FILTER_GET_FLAG(sflag, RANGE_FLG_NULL)) { - if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { - return filterRangeCompLe; - } - - return filterRangeCompLi; - } - - if (FILTER_GET_FLAG(eflag, RANGE_FLG_NULL)) { - if (FILTER_GET_FLAG(sflag, RANGE_FLG_EXCLUDE)) { - return filterRangeCompGe; - } - - return filterRangeCompGi; - } - - if (FILTER_GET_FLAG(sflag, RANGE_FLG_EXCLUDE)) { - if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { - return filterRangeCompee; - } - - return filterRangeCompei; - } - - if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { - return filterRangeCompie; - } - - return filterRangeCompii; -} - -rangeCompFunc gRangeCompare[] = {filterRangeCompee, filterRangeCompei, filterRangeCompie, filterRangeCompii, filterRangeCompGe, - filterRangeCompGi, filterRangeCompLe, filterRangeCompLi}; - - -int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { - if (optr2) { - assert(optr2 == TSDB_RELATION_LESS || optr2 == TSDB_RELATION_LESS_EQUAL); - - if (optr == TSDB_RELATION_GREATER) { - if (optr2 == TSDB_RELATION_LESS) { - return 0; - } - - return 1; - } - - if (optr2 == TSDB_RELATION_LESS) { - return 2; - } - - return 3; - } else { - switch (optr) { - case TSDB_RELATION_GREATER: - return 4; - case TSDB_RELATION_GREATER_EQUAL: - return 5; - case TSDB_RELATION_LESS: - return 6; - case TSDB_RELATION_LESS_EQUAL: - return 7; - default: - break; - } - } - - return -1; -} - -__compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, compareFloatVal, - compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp, - compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, - setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8, compareStrRegexCompMatch, compareStrRegexCompNMatch -}; - -int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { - int8_t comparFn = 0; - - if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - return 15; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - return 16; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_FLOAT: - return 17; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - return 18; - default: - assert(0); - } - } - - switch (type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: comparFn = 1; break; - case TSDB_DATA_TYPE_SMALLINT: comparFn = 2; break; - case TSDB_DATA_TYPE_INT: comparFn = 0; break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: comparFn = 3; break; - case TSDB_DATA_TYPE_FLOAT: comparFn = 4; break; - case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break; - case TSDB_DATA_TYPE_BINARY: { - if (optr == TSDB_RELATION_MATCH) { - comparFn = 19; - } else if (optr == TSDB_RELATION_NMATCH) { - comparFn = 20; - } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ - comparFn = 7; - } else if (optr == TSDB_RELATION_IN) { - comparFn = 8; - } else { /* normal relational comparFn */ - comparFn = 6; - } - - break; - } - - case TSDB_DATA_TYPE_NCHAR: { - if (optr == TSDB_RELATION_MATCH) { - comparFn = 19; - } else if (optr == TSDB_RELATION_NMATCH) { - comparFn = 20; - } else if (optr == TSDB_RELATION_LIKE) { - comparFn = 9; - } else if (optr == TSDB_RELATION_IN) { - comparFn = 8; - } else { - comparFn = 10; - } - break; - } - - case TSDB_DATA_TYPE_UTINYINT: comparFn = 11; break; - case TSDB_DATA_TYPE_USMALLINT: comparFn = 12;break; - case TSDB_DATA_TYPE_UINT: comparFn = 13;break; - case TSDB_DATA_TYPE_UBIGINT: comparFn = 14;break; - - default: - comparFn = 0; - break; - } - - return comparFn; -} - - -static FORCE_INLINE int32_t filterCompareGroupCtx(const void *pLeft, const void *pRight) { - SFilterGroupCtx *left = *((SFilterGroupCtx**)pLeft), *right = *((SFilterGroupCtx**)pRight); - if (left->colNum > right->colNum) return 1; - if (left->colNum < right->colNum) return -1; - return 0; -} - -int32_t filterInitUnitsFields(SFilterInfo *info) { - info->unitSize = FILTER_DEFAULT_UNIT_SIZE; - info->units = calloc(info->unitSize, sizeof(SFilterUnit)); - - info->fields[FLD_TYPE_COLUMN].num = 0; - info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE; - info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, COL_FIELD_SIZE); - info->fields[FLD_TYPE_VALUE].num = 0; - info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE; - info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); - - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilterRange* ra) { - SFilterRangeNode *r = NULL; - - if (ctx->rf) { - r = ctx->rf; - ctx->rf = ctx->rf->next; - r->prev = NULL; - r->next = NULL; - } else { - r = calloc(1, sizeof(SFilterRangeNode)); - } - - FILTER_COPY_RA(&r->ra, ra); - - return r; -} - -void* filterInitRangeCtx(int32_t type, int32_t options) { - if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - qError("not supported range type:%d", type); - return NULL; - } - - SFilterRangeCtx *ctx = calloc(1, sizeof(SFilterRangeCtx)); - - ctx->type = type; - ctx->options = options; - ctx->pCompareFunc = getComparFunc(type, 0); - - return ctx; -} - - -int32_t filterResetRangeCtx(SFilterRangeCtx *ctx) { - ctx->status = 0; - - if (ctx->rf == NULL) { - ctx->rf = ctx->rs; - ctx->rs = NULL; - return TSDB_CODE_SUCCESS; - } - - ctx->isnull = false; - ctx->notnull = false; - ctx->isrange = false; - - SFilterRangeNode *r = ctx->rf; - - while (r && r->next) { - r = r->next; - } - - r->next = ctx->rs; - ctx->rs = NULL; - return TSDB_CODE_SUCCESS; -} - -int32_t filterReuseRangeCtx(SFilterRangeCtx *ctx, int32_t type, int32_t options) { - filterResetRangeCtx(ctx); - - ctx->type = type; - ctx->options = options; - ctx->pCompareFunc = getComparFunc(type, 0); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterConvertRange(SFilterRangeCtx *cur, SFilterRange *ra, bool *notNull) { - if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { - int32_t sr = cur->pCompareFunc(&ra->s, getDataMin(cur->type)); - if (sr == 0) { - FILTER_SET_FLAG(ra->sflag, RANGE_FLG_NULL); - } - } - - if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { - int32_t er = cur->pCompareFunc(&ra->e, getDataMax(cur->type)); - if (er == 0) { - FILTER_SET_FLAG(ra->eflag, RANGE_FLG_NULL); - } - } - - - if (FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL) && FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { - *notNull = true; - } else { - *notNull = false; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t filterAddRangeOptr(void* h, uint8_t raOptr, int32_t optr, bool *empty, bool *all) { - SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - - if (optr == TSDB_RELATION_AND) { - SET_AND_OPTR(ctx, raOptr); - if (CHK_AND_OPTR(ctx) || (raOptr == FILTER_DUMMY_EMPTY_OPTR)) { - FILTER_SET_FLAG(ctx->status, MR_ST_EMPTY); - *empty = true; - } - } else { - SET_OR_OPTR(ctx, raOptr); - if (CHK_OR_OPTR(ctx)) { - FILTER_SET_FLAG(ctx->status, MR_ST_ALL); - *all = true; - } - } - - return TSDB_CODE_SUCCESS; -} - - - -int32_t filterAddRangeImpl(void* h, SFilterRange* ra, int32_t optr) { - SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - - if (ctx->rs == NULL) { - if ((FILTER_GET_FLAG(ctx->status, MR_ST_START) == 0) - || (FILTER_GET_FLAG(ctx->status, MR_ST_ALL) && (optr == TSDB_RELATION_AND)) - || ((!FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) && (optr == TSDB_RELATION_OR))) { - APPEND_RANGE(ctx, ctx->rs, ra); - FILTER_SET_FLAG(ctx->status, MR_ST_START); - } - - return TSDB_CODE_SUCCESS; - } - - SFilterRangeNode *r = ctx->rs; - SFilterRangeNode *rn = NULL; - int32_t cr = 0; - - if (optr == TSDB_RELATION_AND) { - while (r != NULL) { - cr = ctx->pCompareFunc(&r->ra.s, &ra->e); - if (FILTER_GREATER(cr, r->ra.sflag, ra->eflag)) { - FREE_FROM_RANGE(ctx, r); - break; - } - - cr = ctx->pCompareFunc(&ra->s, &r->ra.e); - if (FILTER_GREATER(cr, ra->sflag, r->ra.eflag)) { - rn = r->next; - FREE_RANGE(ctx, r); - r = rn; - continue; - } - - cr = ctx->pCompareFunc(&ra->s, &r->ra.s); - if (FILTER_GREATER(cr, ra->sflag, r->ra.sflag)) { - SIMPLE_COPY_VALUES((char *)&r->ra.s, &ra->s); - cr == 0 ? (r->ra.sflag |= ra->sflag) : (r->ra.sflag = ra->sflag); - } - - cr = ctx->pCompareFunc(&r->ra.e, &ra->e); - if (FILTER_GREATER(cr, r->ra.eflag, ra->eflag)) { - SIMPLE_COPY_VALUES((char *)&r->ra.e, &ra->e); - cr == 0 ? (r->ra.eflag |= ra->eflag) : (r->ra.eflag = ra->eflag); - break; - } - - r = r->next; - } - - return TSDB_CODE_SUCCESS; - } - - - //TSDB_RELATION_OR - - bool smerged = false; - bool emerged = false; - - while (r != NULL) { - cr = ctx->pCompareFunc(&r->ra.s, &ra->e); - if (FILTER_GREATER(cr, r->ra.sflag, ra->eflag)) { - if (emerged == false) { - INSERT_RANGE(ctx, r, ra); - } - - break; - } - - if (smerged == false) { - cr = ctx->pCompareFunc(&ra->s, &r->ra.e); - if (FILTER_GREATER(cr, ra->sflag, r->ra.eflag)) { - if (r->next) { - r= r->next; - continue; - } - - APPEND_RANGE(ctx, r, ra); - break; - } - - cr = ctx->pCompareFunc(&r->ra.s, &ra->s); - if (FILTER_GREATER(cr, r->ra.sflag, ra->sflag)) { - SIMPLE_COPY_VALUES((char *)&r->ra.s, &ra->s); - cr == 0 ? (r->ra.sflag &= ra->sflag) : (r->ra.sflag = ra->sflag); - } - - smerged = true; - } - - if (emerged == false) { - cr = ctx->pCompareFunc(&ra->e, &r->ra.e); - if (FILTER_GREATER(cr, ra->eflag, r->ra.eflag)) { - SIMPLE_COPY_VALUES((char *)&r->ra.e, &ra->e); - if (cr == 0) { - r->ra.eflag &= ra->eflag; - break; - } - - r->ra.eflag = ra->eflag; - emerged = true; - r = r->next; - continue; - } - - break; - } - - cr = ctx->pCompareFunc(&ra->e, &r->ra.e); - if (FILTER_GREATER(cr, ra->eflag, r->ra.eflag)) { - rn = r->next; - FREE_RANGE(ctx, r); - r = rn; - - continue; - } else { - SIMPLE_COPY_VALUES(&r->prev->ra.e, (char *)&r->ra.e); - cr == 0 ? (r->prev->ra.eflag &= r->ra.eflag) : (r->prev->ra.eflag = r->ra.eflag); - FREE_RANGE(ctx, r); - - break; - } - } - - if (ctx->rs && ctx->rs->next == NULL) { - bool notnull; - filterConvertRange(ctx, &ctx->rs->ra, ¬null); - if (notnull) { - bool all = false; - FREE_FROM_RANGE(ctx, ctx->rs); - filterAddRangeOptr(h, TSDB_RELATION_NOTNULL, optr, NULL, &all); - if (all) { - FILTER_SET_FLAG(ctx->status, MR_ST_ALL); - } - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t filterAddRange(void* h, SFilterRange* ra, int32_t optr) { - SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - - if (FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { - SIMPLE_COPY_VALUES(&ra->s, getDataMin(ctx->type)); - //FILTER_CLR_FLAG(ra->sflag, RA_NULL); - } - - if (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { - SIMPLE_COPY_VALUES(&ra->e, getDataMax(ctx->type)); - //FILTER_CLR_FLAG(ra->eflag, RA_NULL); - } - - return filterAddRangeImpl(h, ra, optr); -} - - -int32_t filterAddRangeCtx(void *dst, void *src, int32_t optr) { - SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; - SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; - - assert(optr == TSDB_RELATION_OR); - - if (sctx->rs == NULL) { - return TSDB_CODE_SUCCESS; - } - - SFilterRangeNode *r = sctx->rs; - - while (r) { - filterAddRange(dctx, &r->ra, optr); - r = r->next; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t filterCopyRangeCtx(void *dst, void *src) { - SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; - SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; - - dctx->status = sctx->status; - - dctx->isnull = sctx->isnull; - dctx->notnull = sctx->notnull; - dctx->isrange = sctx->isrange; - - SFilterRangeNode *r = sctx->rs; - SFilterRangeNode *dr = dctx->rs; - - while (r) { - APPEND_RANGE(dctx, dr, &r->ra); - if (dr == NULL) { - dr = dctx->rs; - } else { - dr = dr->next; - } - r = r->next; - } - - return TSDB_CODE_SUCCESS; -} - - - -int32_t filterFinishRange(void* h) { - SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - - if (FILTER_GET_FLAG(ctx->status, MR_ST_FIN)) { - return TSDB_CODE_SUCCESS; - } - - if (FILTER_GET_FLAG(ctx->options, FI_OPTION_TIMESTAMP)) { - SFilterRangeNode *r = ctx->rs; - SFilterRangeNode *rn = NULL; - - while (r && r->next) { - int64_t tmp = 1; - operateVal(&tmp, &r->ra.e, &tmp, TSDB_BINARY_OP_ADD, ctx->type); - if (ctx->pCompareFunc(&tmp, &r->next->ra.s) == 0) { - rn = r->next; - SIMPLE_COPY_VALUES((char *)&r->next->ra.s, (char *)&r->ra.s); - FREE_RANGE(ctx, r); - r = rn; - - continue; - } - - r = r->next; - } - } - - FILTER_SET_FLAG(ctx->status, MR_ST_FIN); - - return TSDB_CODE_SUCCESS; -} - -int32_t filterGetRangeNum(void* h, int32_t* num) { - filterFinishRange(h); - - SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - - *num = 0; - - SFilterRangeNode *r = ctx->rs; - - while (r) { - ++(*num); - r = r->next; - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterGetRangeRes(void* h, SFilterRange *ra) { - filterFinishRange(h); - - SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - uint32_t num = 0; - SFilterRangeNode* r = ctx->rs; - - while (r) { - FILTER_COPY_RA(ra, &r->ra); - - ++num; - r = r->next; - ++ra; - } - - if (num == 0) { - qError("no range result"); - return TSDB_CODE_QRY_APP_ERROR; - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterSourceRangeFromCtx(SFilterRangeCtx *ctx, void *sctx, int32_t optr, bool *empty, bool *all) { - SFilterRangeCtx *src = (SFilterRangeCtx *)sctx; - - if (src->isnull){ - filterAddRangeOptr(ctx, TSDB_RELATION_ISNULL, optr, empty, all); - if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { - *all = true; - } - } - - if (src->notnull) { - filterAddRangeOptr(ctx, TSDB_RELATION_NOTNULL, optr, empty, all); - if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { - *all = true; - } - } - - if (src->isrange) { - filterAddRangeOptr(ctx, 0, optr, empty, all); - - if (!(optr == TSDB_RELATION_OR && ctx->notnull)) { - filterAddRangeCtx(ctx, src, optr); - } - - if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { - *all = true; - } - } - - return TSDB_CODE_SUCCESS; -} - - - -int32_t filterFreeRangeCtx(void* h) { - if (h == NULL) { - return TSDB_CODE_SUCCESS; - } - - SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - SFilterRangeNode *r = ctx->rs; - SFilterRangeNode *rn = NULL; - - while (r) { - rn = r->next; - free(r); - r = rn; - } - - r = ctx->rf; - while (r) { - rn = r->next; - free(r); - r = rn; - } - - free(ctx); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterDetachCnfGroup(SFilterGroup *gp1, SFilterGroup *gp2, SArray* group) { - SFilterGroup gp = {0}; - - gp.unitNum = gp1->unitNum + gp2->unitNum; - gp.unitIdxs = calloc(gp.unitNum, sizeof(*gp.unitIdxs)); - memcpy(gp.unitIdxs, gp1->unitIdxs, gp1->unitNum * sizeof(*gp.unitIdxs)); - memcpy(gp.unitIdxs + gp1->unitNum, gp2->unitIdxs, gp2->unitNum * sizeof(*gp.unitIdxs)); - - gp.unitFlags = NULL; - - taosArrayPush(group, &gp); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { - int32_t leftSize = (int32_t)taosArrayGetSize(left); - int32_t rightSize = (int32_t)taosArrayGetSize(right); - - CHK_LRET(taosArrayGetSize(left) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); - CHK_LRET(taosArrayGetSize(right) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); - - for (int32_t l = 0; l < leftSize; ++l) { - SFilterGroup *gp1 = taosArrayGet(left, l); - - for (int32_t r = 0; r < rightSize; ++r) { - SFilterGroup *gp2 = taosArrayGet(right, r); - - filterDetachCnfGroup(gp1, gp2, group); - } - } - - - return TSDB_CODE_SUCCESS; -} - -int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) { - for (uint16_t i = 0; i < fields->num; ++i) { - if (0 == gDescCompare[type](fields->fields[i].desc, v)) { - return i; - } - } - - return -1; -} - - -int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) { - if (type == FLD_TYPE_VALUE) { - if (info->pctx.valHash == false) { - qError("value hash is empty"); - return -1; - } - - void *hv = taosHashGet(info->pctx.valHash, v, dataLen); - if (hv) { - return *(int32_t *)hv; - } - } - - return -1; -} - - -int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) { - int32_t idx = -1; - uint16_t *num; - - num = &info->fields[type].num; - - if (*num > 0) { - if (type == FLD_TYPE_COLUMN) { - idx = filterGetFiledByDesc(&info->fields[type], type, desc); - } else if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { - idx = filterGetFiledByData(info, type, *data, dataLen); - } - } - - if (idx < 0) { - idx = *num; - if (idx >= info->fields[type].size) { - info->fields[type].size += FILTER_DEFAULT_FIELD_SIZE; - info->fields[type].fields = realloc(info->fields[type].fields, info->fields[type].size * sizeof(SFilterField)); - } - - info->fields[type].fields[idx].flag = type; - info->fields[type].fields[idx].desc = desc; - info->fields[type].fields[idx].data = data ? *data : NULL; - - if (type == FLD_TYPE_COLUMN) { - FILTER_SET_FLAG(info->fields[type].fields[idx].flag, FLD_DATA_NO_FREE); - } - - ++(*num); - - if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { - if (info->pctx.valHash == NULL) { - info->pctx.valHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_VALUE_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); - } - - taosHashPut(info->pctx.valHash, *data, dataLen, &idx, sizeof(idx)); - } - } else { - if (freeIfExists) { - tfree(desc); - } - - if (data && freeIfExists) { - tfree(*data); - } - } - - fid->type = type; - fid->idx = idx; - - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilterField *field, SFilterFieldId *fid) { - filterAddField(info, field->desc, &field->data, FILTER_GET_TYPE(field->flag), fid, 0, false); - - FILTER_SET_FLAG(field->flag, FLD_DESC_NO_FREE); - FILTER_SET_FLAG(field->flag, FLD_DATA_NO_FREE); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) { - CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node"); - CHK_RET(node->nodeType != TSQL_NODE_COL && node->nodeType != TSQL_NODE_VALUE, TSDB_CODE_QRY_APP_ERROR); - - int32_t type; - void *v; - - if (node->nodeType == TSQL_NODE_COL) { - type = FLD_TYPE_COLUMN; - v = node->pSchema; - node->pSchema = NULL; - } else { - type = FLD_TYPE_VALUE; - v = node->pVal; - node->pVal = NULL; - } - - filterAddField(info, v, NULL, type, fid, 0, true); - - return TSDB_CODE_SUCCESS; -} - -int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint16_t *uidx) { - if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { - if (info->pctx.unitHash == NULL) { - info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false); - } else { - int64_t v = 0; - FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); - void *hu = taosHashGet(info->pctx.unitHash, &v, sizeof(v)); - if (hu) { - *uidx = *(uint16_t *)hu; - return TSDB_CODE_SUCCESS; - } - } - } - - if (info->unitNum >= info->unitSize) { - uint16_t psize = info->unitSize; - info->unitSize += FILTER_DEFAULT_UNIT_SIZE; - info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit)); - memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE); - } - - SFilterUnit *u = &info->units[info->unitNum]; - - u->compare.optr = optr; - u->left = *left; - if (right) { - u->right = *right; - } - - if (u->right.type == FLD_TYPE_VALUE) { - SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); - assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); - } else { - assert(optr == TSDB_RELATION_ISNULL || optr == TSDB_RELATION_NOTNULL || optr == FILTER_DUMMY_EMPTY_OPTR); - } - - SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u); - assert(FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN)); - - info->units[info->unitNum].compare.type = FILTER_GET_COL_FIELD_TYPE(col); - - *uidx = info->unitNum; - - if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { - int64_t v = 0; - FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); - taosHashPut(info->pctx.unitHash, &v, sizeof(v), uidx, sizeof(*uidx)); - } - - ++info->unitNum; - - return TSDB_CODE_SUCCESS; -} - - - -int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) { - if (group->unitNum >= group->unitSize) { - group->unitSize += FILTER_DEFAULT_UNIT_SIZE; - group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); - } - - group->unitIdxs[group->unitNum++] = unitIdx; - - return TSDB_CODE_SUCCESS; -} - -int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) { - SBufferReader br = tbufInitReader(buf, len, false); - uint32_t sType = tbufReadUint32(&br); - SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false); - int32_t code = 0; - - taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(tType)); - - int dummy = -1; - tVariant tmpVar = {0}; - size_t t = 0; - int32_t sz = tbufReadInt32(&br); - void *pvar = NULL; - int64_t val = 0; - int32_t bufLen = 0; - if (IS_NUMERIC_TYPE(sType)) { - bufLen = 60; // The maximum length of string that a number is converted to. - } else { - bufLen = 128; - } - - char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); - - for (int32_t i = 0; i < sz; i++) { - switch (sType) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { - *(uint8_t *)&val = (uint8_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { - *(uint16_t *)&val = (uint16_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - *(uint32_t *)&val = (uint32_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - *(uint64_t *)&val = (uint64_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - *(double *)&val = tbufReadDouble(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_FLOAT: { - *(float *)&val = (float)tbufReadDouble(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_BINARY: { - pvar = (char *)tbufReadBinary(&br, &t); - break; - } - case TSDB_DATA_TYPE_NCHAR: { - pvar = (char *)tbufReadBinary(&br, &t); - break; - } - default: - taosHashCleanup(pObj); - *q = NULL; - assert(0); - } - - tVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType); - - if (bufLen < t) { - tmp = realloc(tmp, t * TSDB_NCHAR_SIZE); - bufLen = (int32_t)t; - } - - bool converted = false; - char extInfo = 0; - - switch (tType) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { - if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { - if (converted) { - tVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - continue; - } - - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { - if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { - if (converted) { - tVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - continue; - } - - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { - if (converted) { - tVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - continue; - } - - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - if (tVariantDump(&tmpVar, (char *)&val, tType, false)) { - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - if (tVariantDump(&tmpVar, (char *)&val, tType, false)) { - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_FLOAT: { - if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { - if (converted) { - tVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - continue; - } - - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_BINARY: { - if (tVariantDump(&tmpVar, tmp, tType, true)) { - goto _return; - } - t = varDataLen(tmp); - pvar = varDataVal(tmp); - break; - } - case TSDB_DATA_TYPE_NCHAR: { - if (tVariantDump(&tmpVar, tmp, tType, true)) { - goto _return; - } - t = varDataLen(tmp); - pvar = varDataVal(tmp); - break; - } - default: - goto _return; - } - - taosHashPut(pObj, (char *)pvar, t, &dummy, sizeof(dummy)); - tVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - } - - *q = (void *)pObj; - pObj = NULL; - -_return: - tVariantDestroy(&tmpVar); - taosHashCleanup(pObj); - tfree(tmp); - - return code; -} - - - -int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) { - SFilterFieldId left = {0}, right = {0}; - - filterAddFieldFromNode(info, tree->_node.pLeft, &left); - - tVariant* var = tree->_node.pRight->pVal; - int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left)); - int32_t len = 0; - uint16_t uidx = 0; - - if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) { - void *data = NULL; - filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type); - CHK_LRET(data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); - - if (taosHashGetSize((SHashObj *)data) <= 0) { - filterAddUnit(info, FILTER_DUMMY_EMPTY_OPTR, &left, NULL, &uidx); - - SFilterGroup fgroup = {0}; - filterAddUnitToGroup(&fgroup, uidx); - - taosArrayPush(group, &fgroup); - taosHashCleanup(data); - - return TSDB_CODE_SUCCESS; - } - - void *p = taosHashIterate((SHashObj *)data, NULL); - while(p) { - void *key = taosHashGetDataKey((SHashObj *)data, p); - void *fdata = NULL; - - if (IS_VAR_DATA_TYPE(type)) { - len = (int32_t)taosHashGetDataKeyLen((SHashObj *)data, p); - fdata = malloc(len + VARSTR_HEADER_SIZE); - varDataLen(fdata) = len; - memcpy(varDataVal(fdata), key, len); - len += VARSTR_HEADER_SIZE; - } else { - fdata = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(fdata, key); - len = tDataTypes[type].bytes; - } - - filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true); - - filterAddUnit(info, TSDB_RELATION_EQUAL, &left, &right, &uidx); - - SFilterGroup fgroup = {0}; - filterAddUnitToGroup(&fgroup, uidx); - - taosArrayPush(group, &fgroup); - - p = taosHashIterate((SHashObj *)data, p); - } - - taosHashCleanup(data); - } else { - filterAddFieldFromNode(info, tree->_node.pRight, &right); - - filterAddUnit(info, tree->_node.optr, &left, &right, &uidx); - - SFilterGroup fgroup = {0}; - filterAddUnitToGroup(&fgroup, uidx); - - taosArrayPush(group, &fgroup); - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint16_t *uidx) { - SFilterFieldId left, right, *pright = &right; - int32_t type = FILTER_UNIT_DATA_TYPE(u); - uint16_t flag = FLD_DESC_NO_FREE; - - filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false); - SFilterField *t = FILTER_UNIT_LEFT_FIELD(src, u); - FILTER_SET_FLAG(t->flag, flag); - - if (u->right.type == FLD_TYPE_VALUE) { - void *data = FILTER_UNIT_VAL_DATA(src, u); - if (IS_VAR_DATA_TYPE(type)) { - if (FILTER_UNIT_OPTR(u) == TSDB_RELATION_IN) { - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, 0, false); - - t = FILTER_GET_FIELD(dst, right); - - FILTER_SET_FLAG(t->flag, FLD_DATA_IS_HASH); - } else { - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, varDataTLen(data), false); - } - } else { - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, false); - } - - flag = FLD_DATA_NO_FREE; - t = FILTER_UNIT_RIGHT_FIELD(src, u); - FILTER_SET_FLAG(t->flag, flag); - } else { - pright = NULL; - } - - return filterAddUnit(dst, FILTER_UNIT_OPTR(u), &left, pright, uidx); -} - -int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint16_t uidx) { - SFilterUnit *u = &info->units[uidx]; - - u->compare.optr2 = optr; - u->right2 = *right; - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint16_t cidx, SFilterGroup *g, int32_t optr, SArray *res) { - SFilterFieldId left, right, right2; - uint16_t uidx = 0; - - SFilterField *col = FILTER_GET_COL_FIELD(src, cidx); - - filterAddColFieldFromField(dst, col, &left); - - int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(dst, left)); - - if (optr == TSDB_RELATION_AND) { - if (ctx->isnull) { - assert(ctx->notnull == false && ctx->isrange == false); - filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); - filterAddUnitToGroup(g, uidx); - return TSDB_CODE_SUCCESS; - } - - if (ctx->notnull) { - assert(ctx->isnull == false && ctx->isrange == false); - filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); - filterAddUnitToGroup(g, uidx); - return TSDB_CODE_SUCCESS; - } - - if (!ctx->isrange) { - assert(ctx->isnull || ctx->notnull); - return TSDB_CODE_SUCCESS; - } - - assert(ctx->rs && ctx->rs->next == NULL); - - SFilterRange *ra = &ctx->rs->ra; - - assert(!((FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)))); - - if ((!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL))) { - __compar_fn_t func = getComparFunc(type, 0); - if (func(&ra->s, &ra->e) == 0) { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &ra->s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); - filterAddUnitToGroup(g, uidx); - return TSDB_CODE_SUCCESS; - } else { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &ra->s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - void *data2 = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data2, &ra->e); - filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); - - filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitRight(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); - filterAddUnitToGroup(g, uidx); - return TSDB_CODE_SUCCESS; - } - } - - if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &ra->s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitToGroup(g, uidx); - } - - if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &ra->e); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); - filterAddUnitToGroup(g, uidx); - } - - return TSDB_CODE_SUCCESS; - } - - // OR PROCESS - - SFilterGroup ng = {0}; - g = &ng; - - assert(ctx->isnull || ctx->notnull || ctx->isrange); - - if (ctx->isnull) { - filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); - filterAddUnitToGroup(g, uidx); - taosArrayPush(res, g); - } - - if (ctx->notnull) { - assert(!ctx->isrange); - memset(g, 0, sizeof(*g)); - - filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); - filterAddUnitToGroup(g, uidx); - taosArrayPush(res, g); - } - - if (!ctx->isrange) { - assert(ctx->isnull || ctx->notnull); - g->unitNum = 0; - return TSDB_CODE_SUCCESS; - } - - SFilterRangeNode *r = ctx->rs; - - while (r) { - memset(g, 0, sizeof(*g)); - - if ((!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) &&(!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL))) { - __compar_fn_t func = getComparFunc(type, 0); - if (func(&r->ra.s, &r->ra.e) == 0) { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &r->ra.s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); - filterAddUnitToGroup(g, uidx); - } else { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &r->ra.s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - void *data2 = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data2, &r->ra.e); - filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); - - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitRight(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); - filterAddUnitToGroup(g, uidx); - } - - taosArrayPush(res, g); - - r = r->next; - - continue; - } - - if (!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &r->ra.s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitToGroup(g, uidx); - } - - if (!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(data, &r->ra.e); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); - filterAddUnitToGroup(g, uidx); - } - - assert (g->unitNum > 0); - - taosArrayPush(res, g); - - r = r->next; - } - - g->unitNum = 0; - - return TSDB_CODE_SUCCESS; -} - - -static void filterFreeGroup(void *pItem) { - if (pItem == NULL) { - return; - } - - SFilterGroup* p = (SFilterGroup*) pItem; - tfree(p->unitIdxs); - tfree(p->unitFlags); -} - - -int32_t filterTreeToGroup(tExprNode* tree, SFilterInfo *info, SArray* group) { - int32_t code = TSDB_CODE_SUCCESS; - SArray* leftGroup = NULL; - SArray* rightGroup = NULL; - - if (tree->nodeType != TSQL_NODE_EXPR) { - qError("invalid nodeType:%d", tree->nodeType); - return TSDB_CODE_QRY_APP_ERROR; - } - - if (tree->_node.optr == TSDB_RELATION_AND) { - leftGroup = taosArrayInit(4, sizeof(SFilterGroup)); - rightGroup = taosArrayInit(4, sizeof(SFilterGroup)); - ERR_JRET(filterTreeToGroup(tree->_node.pLeft, info, leftGroup)); - ERR_JRET(filterTreeToGroup(tree->_node.pRight, info, rightGroup)); - - ERR_JRET(filterDetachCnfGroups(group, leftGroup, rightGroup)); - - taosArrayDestroyEx(leftGroup, filterFreeGroup); - taosArrayDestroyEx(rightGroup, filterFreeGroup); - - return TSDB_CODE_SUCCESS; - } - - if (tree->_node.optr == TSDB_RELATION_OR) { - ERR_RET(filterTreeToGroup(tree->_node.pLeft, info, group)); - ERR_RET(filterTreeToGroup(tree->_node.pRight, info, group)); - - return TSDB_CODE_SUCCESS; - } - - code = filterAddGroupUnitFromNode(info, tree, group); - - -_return: - - taosArrayDestroyEx(leftGroup, filterFreeGroup); - taosArrayDestroyEx(rightGroup, filterFreeGroup); - - return code; -} - -#if 0 -int32_t filterInitUnitFunc(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { - SFilterUnit* unit = &info->units[i]; - - info->cunits[i].func = getComparFunc(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); - } - - return TSDB_CODE_SUCCESS; -} -#endif - - -void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) { - if (qDebugFlag & DEBUG_DEBUG) { - CHK_LRETV(info == NULL, "%s - FilterInfo: EMPTY", msg); - - if (options == 0) { - qDebug("%s - FilterInfo:", msg); - qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { - SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SSchema *sch = field->desc; - qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name); - } - - qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num); - for (uint16_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { - SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i]; - if (field->desc) { - tVariant *var = field->desc; - if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { - qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); - } else { - qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO - } - } else if (field->data) { - qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO - } - } - - qDebug("UNIT Num:%u", info->unitNum); - for (uint16_t i = 0; i < info->unitNum; ++i) { - SFilterUnit *unit = &info->units[i]; - int32_t type = FILTER_UNIT_DATA_TYPE(unit); - int32_t len = 0; - int32_t tlen = 0; - char str[512] = {0}; - - SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); - SSchema *sch = left->desc; - len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str); - - if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { - SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); - char *data = right->data; - if (IS_VAR_DATA_TYPE(type)) { - tlen = varDataLen(data); - data += VARSTR_HEADER_SIZE; - } - converToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen); - } else { - strcat(str, "NULL"); - } - strcat(str, "]"); - - if (unit->compare.optr2) { - strcat(str, " && "); - sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str); - - if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { - SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit); - char *data = right->data; - if (IS_VAR_DATA_TYPE(type)) { - tlen = varDataLen(data); - data += VARSTR_HEADER_SIZE; - } - converToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen); - } else { - strcat(str, "NULL"); - } - strcat(str, "]"); - } - - qDebug("%s", str); //TODO - } - - qDebug("GROUP Num:%u", info->groupNum); - for (uint16_t i = 0; i < info->groupNum; ++i) { - SFilterGroup *group = &info->groups[i]; - qDebug("Group%d : unit num[%u]", i, group->unitNum); - - for (uint16_t u = 0; u < group->unitNum; ++u) { - qDebug("unit id:%u", group->unitIdxs[u]); - } - } - - return; - } - - if (options == 1) { - qDebug("%s - RANGE info:", msg); - - qDebug("RANGE Num:%u", info->colRangeNum); - for (uint16_t i = 0; i < info->colRangeNum; ++i) { - SFilterRangeCtx *ctx = info->colRange[i]; - qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange); - if (ctx->isrange) { - SFilterRangeNode *r = ctx->rs; - while (r) { - char str[256] = {0}; - int32_t tlen = 0; - if (FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { - strcat(str,"(NULL)"); - } else { - FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); - converToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen); - FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); - } - strcat(str, " - "); - if (FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { - strcat(str, "(NULL)"); - } else { - FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); - converToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen); - FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); - } - qDebug("range: %s", str); - - r = r->next; - } - } - } - - return; - } - - qDebug("%s - Block Filter info:", msg); - - if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) { - qDebug("Flag:%s", "ALL"); - return; - } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) { - qDebug("Flag:%s", "EMPTY"); - return; - } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE)){ - qDebug("Flag:%s", "ACTIVE"); - } - - qDebug("GroupNum:%d", info->blkGroupNum); - uint16_t *unitIdx = info->blkUnits; - for (uint16_t i = 0; i < info->blkGroupNum; ++i) { - qDebug("Group[%d] UnitNum: %d:", i, *unitIdx); - uint16_t unitNum = *(unitIdx++); - for (uint16_t m = 0; m < unitNum; ++m) { - qDebug("uidx[%d]", *(unitIdx++)); - } - } - } -} - -void filterFreeColInfo(void *data) { - SFilterColInfo* info = (SFilterColInfo *)data; - - if (info->info == NULL) { - return; - } - - if (info->type == RANGE_TYPE_VAR_HASH) { - //TODO - } else if (info->type == RANGE_TYPE_MR_CTX) { - filterFreeRangeCtx(info->info); - } else if (info->type == RANGE_TYPE_UNIT) { - taosArrayDestroy((SArray *)info->info); - } - - //NO NEED TO FREE UNIT - - info->type = 0; - info->info = NULL; -} - -void filterFreeColCtx(void *data) { - SFilterColCtx* ctx = (SFilterColCtx *)data; - - if (ctx->ctx) { - filterFreeRangeCtx(ctx->ctx); - } -} - - -void filterFreeGroupCtx(SFilterGroupCtx* gRes) { - if (gRes == NULL) { - return; - } - - tfree(gRes->colIdx); - - int16_t i = 0, j = 0; - - while (i < gRes->colNum) { - if (gRes->colInfo[j].info) { - filterFreeColInfo(&gRes->colInfo[j]); - ++i; - } - - ++j; - } - - tfree(gRes->colInfo); - tfree(gRes); -} - -void filterFreeField(SFilterField* field, int32_t type) { - if (field == NULL) { - return; - } - - if (!FILTER_GET_FLAG(field->flag, FLD_DESC_NO_FREE)) { - if (type == FLD_TYPE_VALUE) { - tVariantDestroy(field->desc); - } - - tfree(field->desc); - } - - if (!FILTER_GET_FLAG(field->flag, FLD_DATA_NO_FREE)) { - if (FILTER_GET_FLAG(field->flag, FLD_DATA_IS_HASH)) { - taosHashCleanup(field->data); - } else { - tfree(field->data); - } - } -} - -void filterFreePCtx(SFilterPCtx *pctx) { - taosHashCleanup(pctx->valHash); - taosHashCleanup(pctx->unitHash); -} - -void filterFreeInfo(SFilterInfo *info) { - CHK_RETV(info == NULL); - - tfree(info->cunits); - tfree(info->blkUnitRes); - tfree(info->blkUnits); - - for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { - for (uint16_t f = 0; f < info->fields[i].num; ++f) { - filterFreeField(&info->fields[i].fields[f], i); - } - - tfree(info->fields[i].fields); - } - - for (int32_t i = 0; i < info->groupNum; ++i) { - filterFreeGroup(&info->groups[i]); - } - - tfree(info->groups); - - tfree(info->units); - - tfree(info->unitRes); - - tfree(info->unitFlags); - - for (uint16_t i = 0; i < info->colRangeNum; ++i) { - filterFreeRangeCtx(info->colRange[i]); - } - - tfree(info->colRange); - - filterFreePCtx(&info->pctx); - - if (!FILTER_GET_FLAG(info->status, FI_STATUS_CLONED)) { - tfree(info); - } -} - -int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { - assert(extInfo > 0 || extInfo < 0); - - uint8_t optr = FILTER_UNIT_OPTR(unit); - switch (optr) { - case TSDB_RELATION_GREATER: - case TSDB_RELATION_GREATER_EQUAL: - unit->compare.optr = (extInfo > 0) ? FILTER_DUMMY_EMPTY_OPTR : TSDB_RELATION_NOTNULL; - break; - case TSDB_RELATION_LESS: - case TSDB_RELATION_LESS_EQUAL: - unit->compare.optr = (extInfo > 0) ? TSDB_RELATION_NOTNULL : FILTER_DUMMY_EMPTY_OPTR; - break; - case TSDB_RELATION_EQUAL: - unit->compare.optr = FILTER_DUMMY_EMPTY_OPTR; - break; - default: - assert(0); - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterInitValFieldData(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { - SFilterUnit* unit = &info->units[i]; - if (unit->right.type != FLD_TYPE_VALUE) { - assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR); - continue; - } - - SFilterField* right = FILTER_UNIT_RIGHT_FIELD(info, unit); - - assert(FILTER_GET_FLAG(right->flag, FLD_TYPE_VALUE)); - - uint32_t type = FILTER_UNIT_DATA_TYPE(unit); - SFilterField* fi = right; - - tVariant* var = fi->desc; - - if (var == NULL) { - assert(fi->data != NULL); - continue; - } - - if (unit->compare.optr == TSDB_RELATION_IN) { - filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type); - CHK_LRET(fi->data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); - - FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH); - - continue; - } - - if (type == TSDB_DATA_TYPE_BINARY) { - size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; - fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE); - } else if (type == TSDB_DATA_TYPE_NCHAR) { - size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; - fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); - } else { - if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE - fi->data = calloc(var->nLen, tDataTypes[type].bytes); - for (int32_t a = 0; a < var->nLen; ++a) { - int64_t *v = taosArrayGet(var->arr, a); - assignVal((char *)fi->data + a * tDataTypes[type].bytes, (char *)v, 0, type); - } - - continue; - } else { - fi->data = calloc(1, sizeof(int64_t)); - } - } - - bool converted = false; - char extInfo = 0; - if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) { - if (converted) { - filterHandleValueExtInfo(unit, extInfo); - - continue; - } - qError("dump value to type[%d] failed", type); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - return TSDB_CODE_SUCCESS; -} - - -bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) { - int32_t ret = func(left, right); - - switch (optr) { - case TSDB_RELATION_EQUAL: { - return ret == 0; - } - case TSDB_RELATION_NOT_EQUAL: { - return ret != 0; - } - case TSDB_RELATION_GREATER_EQUAL: { - return ret >= 0; - } - case TSDB_RELATION_GREATER: { - return ret > 0; - } - case TSDB_RELATION_LESS_EQUAL: { - return ret <= 0; - } - case TSDB_RELATION_LESS: { - return ret < 0; - } - case TSDB_RELATION_LIKE: { - return ret == 0; - } - case TSDB_RELATION_MATCH: { - return ret == 0; - } - case TSDB_RELATION_NMATCH: { - return ret == 0; - } - case TSDB_RELATION_IN: { - return ret == 1; - } - - default: - assert(false); - } - - return true; -} - - -int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *ctx, int32_t optr) { - int32_t type = FILTER_UNIT_DATA_TYPE(u); - uint8_t uoptr = FILTER_UNIT_OPTR(u); - void *val = FILTER_UNIT_VAL_DATA(info, u); - SFilterRange ra = {0}; - int64_t tmp = 0; - - switch (uoptr) { - case TSDB_RELATION_GREATER: - SIMPLE_COPY_VALUES(&ra.s, val); - FILTER_SET_FLAG(ra.sflag, RANGE_FLG_EXCLUDE); - FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); - break; - case TSDB_RELATION_GREATER_EQUAL: - SIMPLE_COPY_VALUES(&ra.s, val); - FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); - break; - case TSDB_RELATION_LESS: - SIMPLE_COPY_VALUES(&ra.e, val); - FILTER_SET_FLAG(ra.eflag, RANGE_FLG_EXCLUDE); - FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); - break; - case TSDB_RELATION_LESS_EQUAL: - SIMPLE_COPY_VALUES(&ra.e, val); - FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); - break; - case TSDB_RELATION_NOT_EQUAL: - assert(type == TSDB_DATA_TYPE_BOOL); - if (GET_INT8_VAL(val)) { - SIMPLE_COPY_VALUES(&ra.s, &tmp); - SIMPLE_COPY_VALUES(&ra.e, &tmp); - } else { - *(bool *)&tmp = true; - SIMPLE_COPY_VALUES(&ra.s, &tmp); - SIMPLE_COPY_VALUES(&ra.e, &tmp); - } - break; - case TSDB_RELATION_EQUAL: - SIMPLE_COPY_VALUES(&ra.s, val); - SIMPLE_COPY_VALUES(&ra.e, val); - break; - default: - assert(0); - } - - filterAddRange(ctx, &ra, optr); - - return TSDB_CODE_SUCCESS; -} - -int32_t filterCompareRangeCtx(SFilterRangeCtx *ctx1, SFilterRangeCtx *ctx2, bool *equal) { - CHK_JMP(ctx1->status != ctx2->status); - CHK_JMP(ctx1->isnull != ctx2->isnull); - CHK_JMP(ctx1->notnull != ctx2->notnull); - CHK_JMP(ctx1->isrange != ctx2->isrange); - - SFilterRangeNode *r1 = ctx1->rs; - SFilterRangeNode *r2 = ctx2->rs; - - while (r1 && r2) { - CHK_JMP(r1->ra.sflag != r2->ra.sflag); - CHK_JMP(r1->ra.eflag != r2->ra.eflag); - CHK_JMP(r1->ra.s != r2->ra.s); - CHK_JMP(r1->ra.e != r2->ra.e); - - r1 = r1->next; - r2 = r2->next; - } - - CHK_JMP(r1 != r2); - - *equal = true; - - return TSDB_CODE_SUCCESS; - -_return: - *equal = false; - return TSDB_CODE_SUCCESS; -} - - -int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colIdx, bool *empty) { - SArray* colArray = (SArray *)gRes->colInfo[colIdx].info; - int32_t size = (int32_t)taosArrayGetSize(colArray); - int32_t type = gRes->colInfo[colIdx].dataType; - SFilterRangeCtx* ctx = filterInitRangeCtx(type, 0); - - for (uint32_t i = 0; i < size; ++i) { - SFilterUnit* u = taosArrayGetP(colArray, i); - uint8_t optr = FILTER_UNIT_OPTR(u); - - filterAddRangeOptr(ctx, optr, TSDB_RELATION_AND, empty, NULL); - CHK_JMP(*empty); - - if (!FILTER_NO_MERGE_OPTR(optr)) { - filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND); - CHK_JMP(MR_EMPTY_RES(ctx)); - } - } - - taosArrayDestroy(colArray); - - FILTER_PUSH_CTX(gRes->colInfo[colIdx], ctx); - - return TSDB_CODE_SUCCESS; - -_return: - - *empty = true; - - filterFreeRangeCtx(ctx); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) { - bool empty = false; - uint16_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint16_t)); - uint16_t colIdxi = 0; - uint16_t gResIdx = 0; - - for (uint16_t i = 0; i < info->groupNum; ++i) { - SFilterGroup* g = info->groups + i; - - gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx)); - gRes[gResIdx]->colInfo = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(SFilterColInfo)); - colIdxi = 0; - empty = false; - - for (uint16_t j = 0; j < g->unitNum; ++j) { - SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j); - uint16_t cidx = FILTER_UNIT_COL_IDX(u); - - if (gRes[gResIdx]->colInfo[cidx].info == NULL) { - gRes[gResIdx]->colInfo[cidx].info = (SArray *)taosArrayInit(4, POINTER_BYTES); - colIdx[colIdxi++] = cidx; - ++gRes[gResIdx]->colNum; - } else { - if (!FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))) { - FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); - } - } - - FILTER_PUSH_UNIT(gRes[gResIdx]->colInfo[cidx], u); - } - - if (colIdxi > 1) { - qsort(colIdx, colIdxi, sizeof(uint16_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); - } - - for (uint16_t l = 0; l < colIdxi; ++l) { - int32_t type = gRes[gResIdx]->colInfo[colIdx[l]].dataType; - - if (FILTER_NO_MERGE_DATA_TYPE(type)) { - continue; - } - - filterMergeUnits(info, gRes[gResIdx], colIdx[l], &empty); - - if (empty) { - break; - } - } - - if (empty) { - FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); - filterFreeGroupCtx(gRes[gResIdx]); - gRes[gResIdx] = NULL; - - continue; - } - - gRes[gResIdx]->colNum = colIdxi; - FILTER_COPY_IDX(&gRes[gResIdx]->colIdx, colIdx, colIdxi); - ++gResIdx; - } - - tfree(colIdx); - - *gResNum = gResIdx; - - if (gResIdx == 0) { - FILTER_SET_FLAG(info->status, FI_STATUS_EMPTY); - } - - return TSDB_CODE_SUCCESS; -} - -void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) { - uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; - bool equal = false; - - for (; m < gRes1->colNum; ++m) { - idx1 = gRes1->colIdx[m]; - - equal = false; - - for (; n < gRes2->colNum; ++n) { - idx2 = gRes2->colIdx[n]; - if (idx1 < idx2) { - *conflict = true; - return; - } - - if (idx1 > idx2) { - continue; - } - - if (FILTER_NO_MERGE_DATA_TYPE(gRes1->colInfo[idx1].dataType)) { - *conflict = true; - return; - } - - ++n; - equal = true; - break; - } - - if (!equal) { - *conflict = true; - return; - } - } - - *conflict = false; - return; -} - - -int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint16_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) { - SFilterField *fi = FILTER_GET_COL_FIELD(info, cidx); - int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); - - if ((*ctx) == NULL) { - *ctx = filterInitRangeCtx(type, 0); - } else { - filterReuseRangeCtx(*ctx, type, 0); - } - - assert(gRes2->colInfo[cidx].type == RANGE_TYPE_MR_CTX); - assert(gRes1->colInfo[cidx].type == RANGE_TYPE_MR_CTX); - - filterCopyRangeCtx(*ctx, gRes2->colInfo[cidx].info); - filterSourceRangeFromCtx(*ctx, gRes1->colInfo[cidx].info, optr, empty, all); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilterGroupCtx** gRes2, bool *all) { - bool conflict = false; - - filterCheckColConflict(*gRes1, *gRes2, &conflict); - if (conflict) { - return TSDB_CODE_SUCCESS; - } - - FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); - - uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; - bool numEqual = (*gRes1)->colNum == (*gRes2)->colNum; - bool equal = false; - uint16_t equal1 = 0, equal2 = 0, merNum = 0; - SFilterRangeCtx *ctx = NULL; - SFilterColCtx colCtx = {0}; - SArray* colCtxs = taosArrayInit((*gRes2)->colNum, sizeof(SFilterColCtx)); - - for (; m < (*gRes1)->colNum; ++m) { - idx1 = (*gRes1)->colIdx[m]; - - for (; n < (*gRes2)->colNum; ++n) { - idx2 = (*gRes2)->colIdx[n]; - - if (idx1 > idx2) { - continue; - } - - assert(idx1 == idx2); - - ++merNum; - - filterMergeTwoGroupsImpl(info, &ctx, TSDB_RELATION_OR, idx1, *gRes1, *gRes2, NULL, all); - - CHK_JMP(*all); - - if (numEqual) { - if ((*gRes1)->colNum == 1) { - ++equal1; - colCtx.colIdx = idx1; - colCtx.ctx = ctx; - taosArrayPush(colCtxs, &colCtx); - break; - } else { - filterCompareRangeCtx(ctx, (*gRes1)->colInfo[idx1].info, &equal); - if (equal) { - ++equal1; - } - - filterCompareRangeCtx(ctx, (*gRes2)->colInfo[idx2].info, &equal); - if (equal) { - ++equal2; - } - - CHK_JMP(equal1 != merNum && equal2 != merNum); - colCtx.colIdx = idx1; - colCtx.ctx = ctx; - ctx = NULL; - taosArrayPush(colCtxs, &colCtx); - } - } else { - filterCompareRangeCtx(ctx, (*gRes1)->colInfo[idx1].info, &equal); - if (equal) { - ++equal1; - } - - CHK_JMP(equal1 != merNum); - colCtx.colIdx = idx1; - colCtx.ctx = ctx; - ctx = NULL; - taosArrayPush(colCtxs, &colCtx); - } - - ++n; - break; - } - } - - assert(merNum > 0); - - SFilterColInfo *colInfo = NULL; - assert (merNum == equal1 || merNum == equal2); - - filterFreeGroupCtx(*gRes2); - *gRes2 = NULL; - - assert(colCtxs && taosArrayGetSize(colCtxs) > 0); - - int32_t ctxSize = (int32_t)taosArrayGetSize(colCtxs); - SFilterColCtx *pctx = NULL; - - for (int32_t i = 0; i < ctxSize; ++i) { - pctx = taosArrayGet(colCtxs, i); - colInfo = &(*gRes1)->colInfo[pctx->colIdx]; - - filterFreeColInfo(colInfo); - FILTER_PUSH_CTX((*gRes1)->colInfo[pctx->colIdx], pctx->ctx); - } - - taosArrayDestroy(colCtxs); - - return TSDB_CODE_SUCCESS; - -_return: - - if (colCtxs) { - if (taosArrayGetSize(colCtxs) > 0) { - taosArrayDestroyEx(colCtxs, filterFreeColCtx); - } else { - taosArrayDestroy(colCtxs); - } - } - - filterFreeRangeCtx(ctx); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gResNum) { - if (*gResNum <= 1) { - return TSDB_CODE_SUCCESS; - } - - qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx); - - int32_t pEnd = 0, cStart = 0, cEnd = 0; - uint16_t pColNum = 0, cColNum = 0; - int32_t movedNum = 0; - bool all = false; - - cColNum = gRes[0]->colNum; - - for (int32_t i = 1; i <= *gResNum; ++i) { - if (i < (*gResNum) && gRes[i]->colNum == cColNum) { - continue; - } - - cEnd = i - 1; - - movedNum = 0; - if (pColNum > 0) { - for (int32_t m = 0; m <= pEnd; ++m) { - for (int32_t n = cStart; n <= cEnd; ++n) { - assert(m < n); - filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); - - CHK_JMP(all); - - if (gRes[n] == NULL) { - if (n < ((*gResNum) - 1)) { - memmove(&gRes[n], &gRes[n+1], (*gResNum-n-1) * POINTER_BYTES); - } - - --cEnd; - --(*gResNum); - ++movedNum; - --n; - } - } - } - } - - for (int32_t m = cStart; m < cEnd; ++m) { - for (int32_t n = m + 1; n <= cEnd; ++n) { - assert(m < n); - filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); - - CHK_JMP(all); - - if (gRes[n] == NULL) { - if (n < ((*gResNum) - 1)) { - memmove(&gRes[n], &gRes[n+1], (*gResNum-n-1) * POINTER_BYTES); - } - - --cEnd; - --(*gResNum); - ++movedNum; - --n; - } - } - } - - pColNum = cColNum; - pEnd = cEnd; - - i -= movedNum; - - if (i >= (*gResNum)) { - break; - } - - cStart = i; - cEnd = i; - cColNum = gRes[i]->colNum; - } - - return TSDB_CODE_SUCCESS; - -_return: - - FILTER_SET_FLAG(info->status, FI_STATUS_ALL); - - return TSDB_CODE_SUCCESS; -} - -int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { - size_t groupSize = taosArrayGetSize(group); - - info->groupNum = (uint16_t)groupSize; - - if (info->groupNum > 0) { - info->groups = calloc(info->groupNum, sizeof(*info->groups)); - } - - for (size_t i = 0; i < groupSize; ++i) { - SFilterGroup *pg = taosArrayGet(group, i); - pg->unitFlags = calloc(pg->unitNum, sizeof(*pg->unitFlags)); - info->groups[i] = *pg; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { - if (!FILTER_GET_FLAG(info->status, FI_STATUS_REWRITE)) { - qDebug("no need rewrite"); - return TSDB_CODE_SUCCESS; - } - - SFilterInfo oinfo = *info; - - FILTER_SET_FLAG(oinfo.status, FI_STATUS_CLONED); - - SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); - SFilterGroupCtx *res = NULL; - SFilterColInfo *colInfo = NULL; - int32_t optr = 0; - uint16_t uidx = 0; - - memset(info, 0, sizeof(*info)); - - info->colRangeNum = oinfo.colRangeNum; - info->colRange = oinfo.colRange; - oinfo.colRangeNum = 0; - oinfo.colRange = NULL; - - FILTER_SET_FLAG(info->options, FI_OPTION_NEED_UNIQE); - - filterInitUnitsFields(info); - - for (int32_t i = 0; i < gResNum; ++i) { - res = gRes[i]; - - optr = (res->colNum > 1) ? TSDB_RELATION_AND : TSDB_RELATION_OR; - - SFilterGroup ng = {0}; - - for (uint16_t m = 0; m < res->colNum; ++m) { - colInfo = &res->colInfo[res->colIdx[m]]; - if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { - assert(colInfo->type == RANGE_TYPE_UNIT); - int32_t usize = (int32_t)taosArrayGetSize((SArray *)colInfo->info); - - for (int32_t n = 0; n < usize; ++n) { - SFilterUnit* u = taosArrayGetP((SArray *)colInfo->info, n); - - filterAddUnitFromUnit(info, &oinfo, u, &uidx); - filterAddUnitToGroup(&ng, uidx); - } - - continue; - } - - assert(colInfo->type == RANGE_TYPE_MR_CTX); - - filterAddGroupUnitFromCtx(info, &oinfo, colInfo->info, res->colIdx[m], &ng, optr, group); - } - - if (ng.unitNum > 0) { - taosArrayPush(group, &ng); - } - } - - filterConvertGroupFromArray(info, group); - - taosArrayDestroy(group); - - filterFreeInfo(&oinfo); - - return TSDB_CODE_SUCCESS; -} - -int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { - uint16_t *idxs = NULL; - uint16_t colNum = 0; - SFilterGroupCtx *res = NULL; - uint16_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); - - for (int32_t i = 0; i < gResNum; ++i) { - for (uint16_t m = 0; m < gRes[i]->colNum; ++m) { - SFilterColInfo *colInfo = &gRes[i]->colInfo[gRes[i]->colIdx[m]]; - if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { - continue; - } - - ++idxNum[gRes[i]->colIdx[m]]; - } - } - - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { - if (idxNum[i] < gResNum) { - continue; - } - - assert(idxNum[i] == gResNum); - - if (idxs == NULL) { - idxs = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxs)); - } - - idxs[colNum++] = i; - } - - CHK_JMP(colNum <= 0); - - info->colRangeNum = colNum; - info->colRange = calloc(colNum, POINTER_BYTES); - - for (int32_t i = 0; i < gResNum; ++i) { - res = gRes[i]; - uint16_t n = 0; - - for (uint16_t m = 0; m < info->colRangeNum; ++m) { - for (; n < res->colNum; ++n) { - if (res->colIdx[n] < idxs[m]) { - continue; - } - - assert(res->colIdx[n] == idxs[m]); - - SFilterColInfo * colInfo = &res->colInfo[res->colIdx[n]]; - if (info->colRange[m] == NULL) { - info->colRange[m] = filterInitRangeCtx(colInfo->dataType, 0); - SFilterField* fi = FILTER_GET_COL_FIELD(info, res->colIdx[n]); - info->colRange[m]->colId = ((SSchema*)fi->desc)->colId; - } - - assert(colInfo->type == RANGE_TYPE_MR_CTX); - - bool all = false; - filterSourceRangeFromCtx(info->colRange[m], colInfo->info, TSDB_RELATION_OR, NULL, &all); - if (all) { - filterFreeRangeCtx(info->colRange[m]); - info->colRange[m] = NULL; - - if (m < (info->colRangeNum - 1)) { - memmove(&info->colRange[m], &info->colRange[m + 1], (info->colRangeNum - m - 1) * POINTER_BYTES); - memmove(&idxs[m], &idxs[m + 1], (info->colRangeNum - m - 1) * sizeof(*idxs)); - } - - --info->colRangeNum; - --m; - - CHK_JMP(info->colRangeNum <= 0); - } - - ++n; - break; - } - } - } - -_return: - tfree(idxNum); - tfree(idxs); - - return TSDB_CODE_SUCCESS; -} - -int32_t filterPostProcessRange(SFilterInfo *info) { - for (uint16_t i = 0; i < info->colRangeNum; ++i) { - SFilterRangeCtx* ctx = info->colRange[i]; - SFilterRangeNode *r = ctx->rs; - while (r) { - r->rc.func = filterGetRangeCompFunc(r->ra.sflag, r->ra.eflag); - r = r->next; - } - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterGenerateComInfo(SFilterInfo *info) { - uint16_t n = 0; - - info->cunits = malloc(info->unitNum * sizeof(*info->cunits)); - info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum); - info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); - - for (uint16_t i = 0; i < info->unitNum; ++i) { - SFilterUnit *unit = &info->units[i]; - - info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); - info->cunits[i].rfunc = filterGetRangeCompFuncFromOptrs(unit->compare.optr, unit->compare.optr2); - info->cunits[i].optr = FILTER_UNIT_OPTR(unit); - info->cunits[i].colData = NULL; - info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit); - - if (unit->right.type == FLD_TYPE_VALUE) { - info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); - } else { - info->cunits[i].valData = NULL; - } - if (unit->right2.type == FLD_TYPE_VALUE) { - info->cunits[i].valData2 = FILTER_GET_VAL_FIELD_DATA(FILTER_GET_FIELD(info, unit->right2)); - } else { - info->cunits[i].valData2 = info->cunits[i].valData; - } - - info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit); - info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit); - } - - uint16_t cgroupNum = info->groupNum + 1; - - for (uint16_t i = 0; i < info->groupNum; ++i) { - cgroupNum += info->groups[i].unitNum; - } - - info->cgroups = malloc(cgroupNum * sizeof(*info->cgroups)); - - for (uint16_t i = 0; i < info->groupNum; ++i) { - info->cgroups[n++] = info->groups[i].unitNum; - - for (uint16_t m = 0; m < info->groups[i].unitNum; ++m) { - info->cgroups[n++] = info->groups[i].unitIdxs[m]; - } - } - - info->cgroups[n] = 0; - - return TSDB_CODE_SUCCESS; -} - -int32_t filterUpdateComUnits(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { - SFilterUnit *unit = &info->units[i]; - - info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0); - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterRmUnitByRange(SFilterInfo *info, SDataStatis *pDataStatis, int32_t numOfCols, int32_t numOfRows) { - int32_t rmUnit = 0; - - memset(info->blkUnitRes, 0, sizeof(*info->blkUnitRes) * info->unitNum); - - for (int32_t k = 0; k < info->unitNum; ++k) { - int32_t index = -1; - SFilterComUnit *cunit = &info->cunits[k]; - - if (FILTER_NO_MERGE_DATA_TYPE(cunit->dataType)) { - continue; - } - - for(int32_t i = 0; i < numOfCols; ++i) { - if (pDataStatis[i].colId == cunit->colId) { - index = i; - break; - } - } - - if (index == -1) { - continue; - } - - if (pDataStatis[index].numOfNull <= 0) { - if (cunit->optr == TSDB_RELATION_ISNULL) { - info->blkUnitRes[k] = -1; - rmUnit = 1; - continue; - } - - if (cunit->optr == TSDB_RELATION_NOTNULL) { - info->blkUnitRes[k] = 1; - rmUnit = 1; - continue; - } - } else { - if (pDataStatis[index].numOfNull == numOfRows) { - if (cunit->optr == TSDB_RELATION_ISNULL) { - info->blkUnitRes[k] = 1; - rmUnit = 1; - continue; - } - - info->blkUnitRes[k] = -1; - rmUnit = 1; - continue; - } - } - - if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL - || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH - || cunit->optr == TSDB_RELATION_NOT_EQUAL) { - continue; - } - - SDataStatis* pDataBlockst = &pDataStatis[index]; - void *minVal, *maxVal; - - if (cunit->dataType == TSDB_DATA_TYPE_FLOAT) { - float minv = (float)(*(double *)(&pDataBlockst->min)); - float maxv = (float)(*(double *)(&pDataBlockst->max)); - - minVal = &minv; - maxVal = &maxv; - } else { - minVal = &pDataBlockst->min; - maxVal = &pDataBlockst->max; - } - - bool minRes = false, maxRes = false; - - if (cunit->rfunc >= 0) { - minRes = (*gRangeCompare[cunit->rfunc])(minVal, minVal, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); - maxRes = (*gRangeCompare[cunit->rfunc])(maxVal, maxVal, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); - - if (minRes && maxRes) { - info->blkUnitRes[k] = 1; - rmUnit = 1; - } else if ((!minRes) && (!maxRes)) { - minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS_EQUAL, minVal, cunit->valData); - maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER_EQUAL, maxVal, cunit->valData2); - - if (minRes && maxRes) { - continue; - } - - info->blkUnitRes[k] = -1; - rmUnit = 1; - } - } else { - minRes = filterDoCompare(gDataCompare[cunit->func], cunit->optr, minVal, cunit->valData); - maxRes = filterDoCompare(gDataCompare[cunit->func], cunit->optr, maxVal, cunit->valData); - - if (minRes && maxRes) { - info->blkUnitRes[k] = 1; - rmUnit = 1; - } else if ((!minRes) && (!maxRes)) { - if (cunit->optr == TSDB_RELATION_EQUAL) { - minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER, minVal, cunit->valData); - maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS, maxVal, cunit->valData); - if (minRes || maxRes) { - info->blkUnitRes[k] = -1; - rmUnit = 1; - } - - continue; - } - - info->blkUnitRes[k] = -1; - rmUnit = 1; - } - } - - } - - CHK_LRET(rmUnit == 0, TSDB_CODE_SUCCESS, "NO Block Filter APPLY"); - - info->blkGroupNum = info->groupNum; - - uint16_t *unitNum = info->blkUnits; - uint16_t *unitIdx = unitNum + 1; - int32_t all = 0, empty = 0; - - for (uint32_t g = 0; g < info->groupNum; ++g) { - SFilterGroup *group = &info->groups[g]; - *unitNum = group->unitNum; - all = 0; - empty = 0; - - for (uint32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; - if (info->blkUnitRes[uidx] == 1) { - --(*unitNum); - all = 1; - continue; - } else if (info->blkUnitRes[uidx] == -1) { - *unitNum = 0; - empty = 1; - break; - } - - *(unitIdx++) = uidx; - } - - if (*unitNum == 0) { - --info->blkGroupNum; - assert(empty || all); - - if (empty) { - FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY); - } else { - FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL); - goto _return; - } - - continue; - } - - unitNum = unitIdx; - ++unitIdx; - } - - if (info->blkGroupNum) { - FILTER_CLR_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY); - FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE); - } - -_return: - - filterDumpInfoToString(info, "Block Filter", 2); - - return TSDB_CODE_SUCCESS; -} - -bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - SFilterInfo *info = (SFilterInfo *)pinfo; - bool all = true; - uint16_t *unitIdx = NULL; - - *p = calloc(numOfRows, sizeof(int8_t)); - - for (int32_t i = 0; i < numOfRows; ++i) { - //FILTER_UNIT_CLR_F(info); - - unitIdx = info->blkUnits; - - for (uint32_t g = 0; g < info->blkGroupNum; ++g) { - uint16_t unitNum = *(unitIdx++); - for (uint32_t u = 0; u < unitNum; ++u) { - SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)]; - void *colData = (char *)cunit->colData + cunit->dataSize * i; - - //if (FILTER_UNIT_GET_F(info, uidx)) { - // p[i] = FILTER_UNIT_GET_R(info, uidx); - //} else { - uint8_t optr = cunit->optr; - - if (isNull(colData, cunit->dataType)) { - (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; - } else { - if (optr == TSDB_RELATION_NOTNULL) { - (*p)[i] = 1; - } else if (optr == TSDB_RELATION_ISNULL) { - (*p)[i] = 0; - } else if (cunit->rfunc >= 0) { - (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); - } else { - (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); - } - - //FILTER_UNIT_SET_R(info, uidx, p[i]); - //FILTER_UNIT_SET_F(info, uidx); - } - - if ((*p)[i] == 0) { - break; - } - } - - if ((*p)[i]) { - break; - } - - unitIdx += unitNum; - } - - if ((*p)[i] == 0) { - all = false; - } - } - - return all; -} - - - -int32_t filterExecuteBasedOnStatis(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols, bool* all) { - if (statis && numOfRows >= FILTER_RM_UNIT_MIN_ROWS) { - info->blkFlag = 0; - - filterRmUnitByRange(info, statis, numOfCols, numOfRows); - - if (info->blkFlag) { - if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) { - *all = true; - goto _return; - } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) { - *all = false; - goto _return; - } - - assert(info->unitNum > 1); - - *all = filterExecuteBasedOnStatisImpl(info, numOfRows, p, statis, numOfCols); - - goto _return; - } - } - - return 1; - -_return: - info->blkFlag = 0; - - return TSDB_CODE_SUCCESS; -} - - -static FORCE_INLINE bool filterExecuteImplAll(void *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - return true; -} -static FORCE_INLINE bool filterExecuteImplEmpty(void *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - return false; -} -static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - SFilterInfo *info = (SFilterInfo *)pinfo; - bool all = true; - - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { - return all; - } - - *p = calloc(numOfRows, sizeof(int8_t)); - - for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - (*p)[i] = isNull(colData, info->cunits[uidx].dataType); - if ((*p)[i] == 0) { - all = false; - } - } - - return all; -} -static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - SFilterInfo *info = (SFilterInfo *)pinfo; - bool all = true; - - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { - return all; - } - - *p = calloc(numOfRows, sizeof(int8_t)); - - for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - (*p)[i] = !isNull(colData, info->cunits[uidx].dataType); - if ((*p)[i] == 0) { - all = false; - } - } - - return all; -} - -bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - SFilterInfo *info = (SFilterInfo *)pinfo; - bool all = true; - uint16_t dataSize = info->cunits[0].dataSize; - char *colData = (char *)info->cunits[0].colData; - rangeCompFunc rfunc = gRangeCompare[info->cunits[0].rfunc]; - void *valData = info->cunits[0].valData; - void *valData2 = info->cunits[0].valData2; - __compar_fn_t func = gDataCompare[info->cunits[0].func]; - - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { - return all; - } - - *p = calloc(numOfRows, sizeof(int8_t)); - - for (int32_t i = 0; i < numOfRows; ++i) { - if (isNull(colData, info->cunits[0].dataType)) { - all = false; - colData += dataSize; - continue; - } - - (*p)[i] = (*rfunc)(colData, colData, valData, valData2, func); - - if ((*p)[i] == 0) { - all = false; - } - - colData += dataSize; - } - - return all; -} - -bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - SFilterInfo *info = (SFilterInfo *)pinfo; - bool all = true; - - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { - return all; - } - - *p = calloc(numOfRows, sizeof(int8_t)); - - for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - if (isNull(colData, info->cunits[uidx].dataType)) { - all = false; - continue; - } - - (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); - - if ((*p)[i] == 0) { - all = false; - } - } - - return all; -} - - -bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - SFilterInfo *info = (SFilterInfo *)pinfo; - bool all = true; - - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { - return all; - } - - *p = calloc(numOfRows, sizeof(int8_t)); - - for (int32_t i = 0; i < numOfRows; ++i) { - //FILTER_UNIT_CLR_F(info); - - for (uint32_t g = 0; g < info->groupNum; ++g) { - SFilterGroup *group = &info->groups[g]; - for (uint32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; - SFilterComUnit *cunit = &info->cunits[uidx]; - void *colData = (char *)cunit->colData + cunit->dataSize * i; - - //if (FILTER_UNIT_GET_F(info, uidx)) { - // p[i] = FILTER_UNIT_GET_R(info, uidx); - //} else { - uint8_t optr = cunit->optr; - - if (isNull(colData, cunit->dataType)) { - (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; - } else { - if (optr == TSDB_RELATION_NOTNULL) { - (*p)[i] = 1; - } else if (optr == TSDB_RELATION_ISNULL) { - (*p)[i] = 0; - } else if (cunit->rfunc >= 0) { - (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); - } else { - (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); - } - - //FILTER_UNIT_SET_R(info, uidx, p[i]); - //FILTER_UNIT_SET_F(info, uidx); - } - - if ((*p)[i] == 0) { - break; - } - } - - if ((*p)[i]) { - break; - } - } - - if ((*p)[i] == 0) { - all = false; - } - } - - return all; -} - - -FORCE_INLINE bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { - return (*info->func)(info, numOfRows, p, statis, numOfCols); -} - -int32_t filterSetExecFunc(SFilterInfo *info) { - if (FILTER_ALL_RES(info)) { - info->func = filterExecuteImplAll; - return TSDB_CODE_SUCCESS; - } - - if (FILTER_EMPTY_RES(info)) { - info->func = filterExecuteImplEmpty; - return TSDB_CODE_SUCCESS; - } - - if (info->unitNum > 1) { - info->func = filterExecuteImpl; - return TSDB_CODE_SUCCESS; - } - - if (info->units[0].compare.optr == TSDB_RELATION_ISNULL) { - info->func = filterExecuteImplIsNull; - return TSDB_CODE_SUCCESS; - } - - if (info->units[0].compare.optr == TSDB_RELATION_NOTNULL) { - info->func = filterExecuteImplNotNull; - return TSDB_CODE_SUCCESS; - } - - if (info->cunits[0].rfunc >= 0) { - info->func = filterExecuteImplRange; - return TSDB_CODE_SUCCESS; - } - - info->func = filterExecuteImplMisc; - return TSDB_CODE_SUCCESS; -} - - - -int32_t filterPreprocess(SFilterInfo *info) { - SFilterGroupCtx** gRes = calloc(info->groupNum, sizeof(SFilterGroupCtx *)); - int32_t gResNum = 0; - - filterMergeGroupUnits(info, gRes, &gResNum); - - filterMergeGroups(info, gRes, &gResNum); - - if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) { - qInfo("Final - FilterInfo: [ALL]"); - goto _return; - } - - - if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { - qInfo("Final - FilterInfo: [EMPTY]"); - goto _return; - } - - filterGenerateColRange(info, gRes, gResNum); - - filterDumpInfoToString(info, "Final", 1); - - filterPostProcessRange(info); - - filterRewrite(info, gRes, gResNum); - - filterGenerateComInfo(info); - -_return: - - filterSetExecFunc(info); - - for (int32_t i = 0; i < gResNum; ++i) { - filterFreeGroupCtx(gRes[i]); - } - - tfree(gRes); - - return TSDB_CODE_SUCCESS; -} - -int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock) { - CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); - CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); - - if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) { - return TSDB_CODE_SUCCESS; - } - - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { - SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SSchema* sch = fi->desc; - - for (int32_t j = 0; j < numOfCols; ++j) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j); - if (sch->colId == pColInfo->info.colId) { - fi->data = pColInfo->pData; - - break; - } - } - } - - filterUpdateComUnits(info); - - return TSDB_CODE_SUCCESS; -} - - -int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options) { - int32_t code = TSDB_CODE_SUCCESS; - SFilterInfo *info = NULL; - - CHK_LRET(tree == NULL || pinfo == NULL, TSDB_CODE_QRY_APP_ERROR, "invalid param"); - - if (*pinfo == NULL) { - *pinfo = calloc(1, sizeof(SFilterInfo)); - } - - info = *pinfo; - - info->options = options; - - SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); - - filterInitUnitsFields(info); - - code = filterTreeToGroup(tree, info, group); - - ERR_JRET(code); - - filterConvertGroupFromArray(info, group); - - ERR_JRET(filterInitValFieldData(info)); - - if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) { - filterDumpInfoToString(info, "Before preprocess", 0); - - ERR_JRET(filterPreprocess(info)); - - CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL)); - - if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { - taosArrayDestroy(group); - return code; - } - - //ERR_JRET(filterInitUnitFunc(info)); - } - - info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes)); - info->unitFlags = malloc(info->unitNum * sizeof(*info->unitFlags)); - - filterDumpInfoToString(info, "Final", 0); - - taosArrayDestroy(group); - - return code; - -_return: - qInfo("No filter, code:%d", code); - - taosArrayDestroy(group); - - filterFreeInfo(*pinfo); - - *pinfo = NULL; - - return code; -} - - - - -bool filterRangeExecute(SFilterInfo *info, SDataStatis *pDataStatis, int32_t numOfCols, int32_t numOfRows) { - if (FILTER_EMPTY_RES(info)) { - return false; - } - - if (FILTER_ALL_RES(info)) { - return true; - } - - bool ret = true; - void *minVal, *maxVal; - - for (int32_t k = 0; k < info->colRangeNum; ++k) { - int32_t index = -1; - SFilterRangeCtx *ctx = info->colRange[k]; - for(int32_t i = 0; i < numOfCols; ++i) { - if (pDataStatis[i].colId == ctx->colId) { - index = i; - break; - } - } - - // no statistics data, load the true data block - if (index == -1) { - break; - } - - // not support pre-filter operation on binary/nchar data type - if (FILTER_NO_MERGE_DATA_TYPE(ctx->type)) { - break; - } - - if ((pDataStatis[index].numOfNull <= 0) && (ctx->isnull && !ctx->notnull && !ctx->isrange)) { - ret = false; - break; - } - - // all data in current column are NULL, no need to check its boundary value - if (pDataStatis[index].numOfNull == numOfRows) { - - // if isNULL query exists, load the null data column - if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { - ret = false; - break; - } - - continue; - } - - SDataStatis* pDataBlockst = &pDataStatis[index]; - - SFilterRangeNode *r = ctx->rs; - - if (ctx->type == TSDB_DATA_TYPE_FLOAT) { - float minv = (float)(*(double *)(&pDataBlockst->min)); - float maxv = (float)(*(double *)(&pDataBlockst->max)); - - minVal = &minv; - maxVal = &maxv; - } else { - minVal = &pDataBlockst->min; - maxVal = &pDataBlockst->max; - } - - while (r) { - ret = r->rc.func(minVal, maxVal, &r->rc.s, &r->rc.e, ctx->pCompareFunc); - if (ret) { - break; - } - r = r->next; - } - - CHK_RET(!ret, ret); - } - - return ret; -} - - - -int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { - SFilterRange ra = {0}; - SFilterRangeCtx *prev = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); - SFilterRangeCtx *tmpc = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); - SFilterRangeCtx *cur = NULL; - int32_t num = 0; - int32_t optr = 0; - int32_t code = 0; - bool empty = false, all = false; - - for (int32_t i = 0; i < info->groupNum; ++i) { - SFilterGroup *group = &info->groups[i]; - if (group->unitNum > 1) { - cur = tmpc; - optr = TSDB_RELATION_AND; - } else { - cur = prev; - optr = TSDB_RELATION_OR; - } - - for (int32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; - SFilterUnit *unit = &info->units[uidx]; - - uint8_t raOptr = FILTER_UNIT_OPTR(unit); - - filterAddRangeOptr(cur, raOptr, TSDB_RELATION_AND, &empty, NULL); - CHK_JMP(empty); - - if (FILTER_NO_MERGE_OPTR(raOptr)) { - continue; - } - - SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); - void *s = FILTER_GET_VAL_FIELD_DATA(right); - void *e = FILTER_GET_VAL_FIELD_DATA(right) + tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; - - SIMPLE_COPY_VALUES(&ra.s, s); - SIMPLE_COPY_VALUES(&ra.e, e); - - filterAddRange(cur, &ra, optr); - } - - if (cur->notnull) { - prev->notnull = true; - break; - } - - if (group->unitNum > 1) { - filterSourceRangeFromCtx(prev, cur, TSDB_RELATION_OR, &empty, &all); - filterResetRangeCtx(cur); - if (all) { - break; - } - } - } - - if (prev->notnull) { - *win = TSWINDOW_INITIALIZER; - } else { - filterGetRangeNum(prev, &num); - if (num > 1) { - qError("only one time range accepted, num:%d", num); - ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION); - } - - CHK_JMP(num < 1); - - SFilterRange tra; - filterGetRangeRes(prev, &tra); - win->skey = tra.s; - win->ekey = tra.e; - } - - filterFreeRangeCtx(prev); - filterFreeRangeCtx(tmpc); - - qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); - return TSDB_CODE_SUCCESS; - -_return: - - *win = TSWINDOW_DESC_INITIALIZER; - - filterFreeRangeCtx(prev); - filterFreeRangeCtx(tmpc); - - qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); - - return code; -} - - -int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar) { - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { - SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; - int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); - if (type == TSDB_DATA_TYPE_NCHAR) { - SFilterField nfi = {0}; - nfi.desc = fi->desc; - int32_t bytes = FILTER_GET_COL_FIELD_SIZE(fi); - nfi.data = malloc(rows * bytes); - int32_t bufSize = bytes - VARSTR_HEADER_SIZE; - for (int32_t j = 0; j < rows; ++j) { - char *src = FILTER_GET_COL_FIELD_DATA(fi, j); - char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j); - int32_t len = 0; - taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); - varDataLen(dst) = len; - } - - fi->data = nfi.data; - - *gotNchar = true; - } - } - - if (*gotNchar) { - filterUpdateComUnits(info); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t filterFreeNcharColumns(SFilterInfo* info) { - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { - SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; - int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); - if (type == TSDB_DATA_TYPE_NCHAR) { - tfree(fi->data); - } - } - - return TSDB_CODE_SUCCESS; -} - - - - - diff --git a/2.0/src/query/src/qFilterfunc.c b/2.0/src/query/src/qFilterfunc.c deleted file mode 100644 index 6ad7a6ef63edc7cd420c3a7829397b26c294b858..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qFilterfunc.c +++ /dev/null @@ -1,465 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" - -#include "qExecutor.h" -#include "tmsg.h" -#include "tcompare.h" -#include "ttype.h" - -#define FLT_COMPAR_TOL_FACTOR 4 -#define FLT_EQUAL(_x, _y) (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON)) -#define FLT_GREATER(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) > (_y))) -#define FLT_LESS(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) < (_y))) -#define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y))) -#define FLT_LESSEQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) < (_y))) - -bool lessOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { - SColumnFilterInfo* pFilterInfo = &pFilter->filterInfo; - - switch(type) { - case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_INT: return (*(int32_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)minval < pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_FLOAT: return FLT_LESS(*(float*)minval, pFilter->filterInfo.upperBndd); - case TSDB_DATA_TYPE_DOUBLE: return (*(double *)minval < pFilterInfo->upperBndd); - default: - return false; - } -} - -bool greaterOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_INT: return (*(int32_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)maxval > pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_FLOAT: return FLT_GREATER(*(float *)maxval, pFilterInfo->lowerBndd); - case TSDB_DATA_TYPE_DOUBLE: return (*(double *)maxval > pFilterInfo->lowerBndd); - default: - return false; - } -} - -bool lessEqualOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { - SColumnFilterInfo* pFilterInfo = &pFilter->filterInfo; - - switch(type) { - case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_INT: return (*(int32_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)minval <= pFilterInfo->upperBndi); - case TSDB_DATA_TYPE_FLOAT: return FLT_LESSEQUAL(*(float*)minval, pFilterInfo->upperBndd); - case TSDB_DATA_TYPE_DOUBLE: { - if ((fabs(*(double*)minval) - pFilterInfo->upperBndd) <= 2 * DBL_EPSILON) { - return true; - } - - return (*(double *)minval <= pFilterInfo->upperBndd); - } - default: - return false; - } -} - -bool greaterEqualOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_INT: return (*(int32_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)maxval >= pFilterInfo->lowerBndi); - case TSDB_DATA_TYPE_FLOAT: return FLT_GREATEREQUAL(*(float*)maxval, pFilterInfo->lowerBndd); - case TSDB_DATA_TYPE_DOUBLE: { - if (fabs(*(double *)maxval - pFilterInfo->lowerBndd) <= 2 * DBL_EPSILON) { - return true; - } - - return (*(double *)maxval - pFilterInfo->lowerBndd > (2 * DBL_EPSILON)); - } - default: - return false; - } -} - -//////////////////////////////////////////////////////////////////////// -bool equalOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t minv = -1, maxv = -1; - GET_TYPED_DATA(minv, int64_t, type, minval); - GET_TYPED_DATA(maxv, int64_t, type, maxval); - - if (minv == maxv) { - return minv == pFilterInfo->lowerBndi; - } else { - assert(minv < maxv); - return minv <= pFilterInfo->lowerBndi && pFilterInfo->lowerBndi <= maxv; - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - uint64_t minv = 0, maxv = 0; - GET_TYPED_DATA(minv, uint64_t, type, minval); - GET_TYPED_DATA(maxv, uint64_t, type, maxval); - - if (minv == maxv) { - return minv == pFilterInfo->lowerBndi; - } else { - assert(minv < maxv); - return minv <= pFilterInfo->lowerBndi && pFilterInfo->lowerBndi <= maxv; - } - } else if (IS_FLOAT_TYPE(type)) { - double minv = -1, maxv = -1; - GET_TYPED_DATA(minv, double, type, minval); - GET_TYPED_DATA(maxv, double, type, maxval); - - if (minv == maxv) { - return FLT_EQUAL(minv, pFilterInfo->lowerBndd); - } else { // range filter - assert(minv < maxv); - return minv <= pFilterInfo->lowerBndd && pFilterInfo->lowerBndd <= maxv; - } - } else if (type == TSDB_DATA_TYPE_BINARY) { - // query condition string is greater than the max length of string, not qualified data - if (pFilterInfo->len != varDataLen(minval)) { - return false; - } - - return strncmp((char *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval)) == 0; - } else if (type == TSDB_DATA_TYPE_NCHAR) { - // query condition string is greater than the max length of string, not qualified data - if (pFilterInfo->len != varDataLen(minval)) { - return false; - } - - return wcsncmp((wchar_t *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval) / TSDB_NCHAR_SIZE) == 0; - } else { - return false; - } -} - -//////////////////////////////////////////////////////////////// -bool likeOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - if (type == TSDB_DATA_TYPE_BINARY) { - SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - return patternMatch((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval), &info) == TSDB_PATTERN_MATCH; - } else if (type == TSDB_DATA_TYPE_NCHAR) { - SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - return WCSPatternMatch((wchar_t*)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH; - } else { - return false; - } -} - -//////////////////////////////////////////////////////////////// -/** - * If minval equals to maxval, it may serve as the one element filter, - * or all elements of an array are identical during pref-filter stage. - * Otherwise, it must be pre-filter of array list of elements. - * - * During pre-filter stage, if there is one element that locates in [minval, maxval], - * the filter function will return true. - */ -// TODO not equal need to refactor -bool notEqualOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t minv = -1, maxv = -1; - GET_TYPED_DATA(minv, int64_t, type, minval); - GET_TYPED_DATA(maxv, int64_t, type, maxval); - - if (minv == maxv) { - return minv != pFilterInfo->lowerBndi; - } - return true; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - uint64_t minv = 0, maxv = 0; - GET_TYPED_DATA(minv, uint64_t, type, minval); - GET_TYPED_DATA(maxv, uint64_t, type, maxval); - - if (minv == maxv) { - return minv != pFilterInfo->lowerBndi; - } - return true; - } else if (IS_FLOAT_TYPE(type)) { - double minv = -1, maxv = -1; - GET_TYPED_DATA(minv, double, type, minval); - GET_TYPED_DATA(maxv, double, type, maxval); - - if (minv == maxv) { - return !FLT_EQUAL(minv, pFilterInfo->lowerBndd); - } - return true; - } else if (type == TSDB_DATA_TYPE_BINARY) { - if (pFilterInfo->len != varDataLen(minval)) { - return true; - } - return strncmp((char *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval)) != 0; - } else if (type == TSDB_DATA_TYPE_NCHAR) { - if (pFilterInfo->len != varDataLen(minval)) { - return true; - } - return wcsncmp((wchar_t *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0; - } else { - return false; - } -} - -//////////////////////////////////////////////////////////////// -// dummy filter, not used -bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { - return true; -} - -bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { - return true; -} -bool inOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) { - if (type == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t minv = -1, maxv = -1; - GET_TYPED_DATA(minv, int64_t, type, minval); - GET_TYPED_DATA(maxv, int64_t, type, maxval); - if (minv == maxv) { - return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&minv, sizeof(minv)); - } - return false; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - uint64_t minv = 0, maxv = 0; - GET_TYPED_DATA(minv, uint64_t, type, minval); - GET_TYPED_DATA(maxv, uint64_t, type, maxval); - if (minv == maxv) { - return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&minv, sizeof(minv)); - } - return false; - }else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_FLOAT) { - double v; - GET_TYPED_DATA(v, double, type, minval); - return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&v, sizeof(v)); - } else if (type == TSDB_DATA_TYPE_BINARY) { - return NULL != taosHashGet((SHashObj *)pFilter->q, varDataVal(minval), varDataLen(minval)); - } else if (type == TSDB_DATA_TYPE_NCHAR){ - return NULL != taosHashGet((SHashObj *)pFilter->q, varDataVal(minval), varDataLen(minval)); - } - - return false; -} - -/////////////////////////////////////////////////////////////////////////////// -bool rangeFilter_ii(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - return ((*(int8_t *)minval <= pFilterInfo->upperBndi) && (*(int8_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UTINYINT: - return ((*(uint8_t *)minval <= pFilterInfo->upperBndi) && (*(uint8_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_SMALLINT: - return ((*(int16_t *)minval <= pFilterInfo->upperBndi) && (*(int16_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_USMALLINT: - return ((*(uint16_t *)minval <= pFilterInfo->upperBndi) && (*(uint16_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_INT: - return ((*(int32_t *)minval <= pFilterInfo->upperBndi) && (*(int32_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UINT: - return ((*(uint32_t *)minval <= pFilterInfo->upperBndi) && (*(uint32_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - return ((*(int64_t *)minval <= pFilterInfo->upperBndi) && (*(int64_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UBIGINT: - return ((*(uint64_t *)minval <= pFilterInfo->upperBndi) && (*(uint64_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_FLOAT: - return FLT_LESSEQUAL(*(float *)minval, pFilterInfo->upperBndd) && - FLT_GREATEREQUAL(*(float *)maxval, pFilterInfo->lowerBndd); - case TSDB_DATA_TYPE_DOUBLE: - return (*(double *)minval <= pFilterInfo->upperBndd && *(double *)maxval >= pFilterInfo->lowerBndd); - default: - return false; - } -} - -bool rangeFilter_ee(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - return ((*(int8_t *)minval < pFilterInfo->upperBndi) && (*(int8_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UTINYINT: - return ((*(uint8_t *)minval < pFilterInfo->upperBndi) && (*(uint8_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_SMALLINT: - return ((*(int16_t *)minval < pFilterInfo->upperBndi) && (*(int16_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_USMALLINT: - return ((*(uint16_t *)minval < pFilterInfo->upperBndi) && (*(uint16_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_INT: - return ((*(int32_t *)minval < pFilterInfo->upperBndi) && (*(int32_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UINT: - return ((*(uint32_t *)minval < pFilterInfo->upperBndi) && (*(uint32_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - return ((*(int64_t *)minval < pFilterInfo->upperBndi) && (*(int64_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UBIGINT: - return ((*(uint64_t *)minval < pFilterInfo->upperBndi) && (*(uint64_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_FLOAT: - return ((*(float *)minval < pFilterInfo->upperBndd) && (*(float *)maxval > pFilterInfo->lowerBndd)); - case TSDB_DATA_TYPE_DOUBLE: - return ((*(double *)minval < pFilterInfo->upperBndd) && (*(double *)maxval > pFilterInfo->lowerBndd)); - default: - return false; - } -} - -bool rangeFilter_ie(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - return ((*(int8_t *)minval < pFilterInfo->upperBndi) && (*(int8_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UTINYINT: - return ((*(uint8_t *)minval < pFilterInfo->upperBndi) && (*(uint8_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_SMALLINT: - return ((*(int16_t *)minval < pFilterInfo->upperBndi) && (*(int16_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_USMALLINT: - return ((*(uint16_t *)minval < pFilterInfo->upperBndi) && (*(uint16_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_INT: - return ((*(int32_t *)minval < pFilterInfo->upperBndi) && (*(int32_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UINT: - return ((*(uint32_t *)minval < pFilterInfo->upperBndi) && (*(uint32_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - return ((*(int64_t *)minval < pFilterInfo->upperBndi) && (*(int64_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UBIGINT: - return ((*(uint64_t *)minval < pFilterInfo->upperBndi) && (*(uint64_t *)maxval >= pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_FLOAT: - return ((*(float *)minval < pFilterInfo->upperBndd) && (*(float *)maxval >= pFilterInfo->lowerBndd)); - case TSDB_DATA_TYPE_DOUBLE: - return ((*(double *)minval < pFilterInfo->upperBndd) && (*(double *)maxval >= pFilterInfo->lowerBndd)); - default: - return false; - } -} - -bool rangeFilter_ei(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { - SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - return ((*(int8_t *)minval <= pFilterInfo->upperBndi) && (*(int8_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UTINYINT: - return ((*(uint8_t *)minval <= pFilterInfo->upperBndi) && (*(uint8_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_SMALLINT: - return ((*(int16_t *)minval <= pFilterInfo->upperBndi) && (*(int16_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_USMALLINT: - return ((*(uint16_t *)minval <= pFilterInfo->upperBndi) && (*(uint16_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_INT: - return ((*(int32_t *)minval <= pFilterInfo->upperBndi) && (*(int32_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UINT: - return ((*(uint32_t *)minval <= pFilterInfo->upperBndi) && (*(uint32_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - return ((*(int64_t *)minval <= pFilterInfo->upperBndi) && (*(int64_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_UBIGINT: - return ((*(uint64_t *)minval <= pFilterInfo->upperBndi) && (*(uint64_t *)maxval > pFilterInfo->lowerBndi)); - case TSDB_DATA_TYPE_FLOAT: - return FLT_GREATER(*(float *)maxval, pFilterInfo->lowerBndd) && - FLT_LESSEQUAL(*(float *)minval, pFilterInfo->upperBndd); - case TSDB_DATA_TYPE_DOUBLE: - return ((*(double *)minval <= pFilterInfo->upperBndd) && (*(double *)maxval > pFilterInfo->lowerBndd)); - default: - return false; - } -} - -//////////////////////////////////////////////////////////////////////////// -bool (*filterOperators[])(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) = { - NULL, - lessOperator, - greaterOperator, - equalOperator, - lessEqualOperator, - greaterEqualOperator, - notEqualOperator, - likeOperator, - isNullOperator, - notNullOperator, - inOperator, -}; - -bool (*rangeFilterOperators[])(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) = { - NULL, - rangeFilter_ee, - rangeFilter_ie, - rangeFilter_ei, - rangeFilter_ii, - -}; - -__filter_func_t getFilterOperator(int32_t lowerOptr, int32_t upperOptr) { - __filter_func_t funcFp = NULL; - - if ((lowerOptr == TSDB_RELATION_GREATER_EQUAL || lowerOptr == TSDB_RELATION_GREATER) && - (upperOptr == TSDB_RELATION_LESS_EQUAL || upperOptr == TSDB_RELATION_LESS)) { - if (lowerOptr == TSDB_RELATION_GREATER_EQUAL) { - if (upperOptr == TSDB_RELATION_LESS_EQUAL) { - funcFp = rangeFilterOperators[4]; - } else { - funcFp = rangeFilterOperators[2]; - } - } else { - if (upperOptr == TSDB_RELATION_LESS_EQUAL) { - funcFp = rangeFilterOperators[3]; - } else { - funcFp = rangeFilterOperators[1]; - } - } - } else { // set callback filter function - if (lowerOptr != TSDB_RELATION_INVALID) { - funcFp = filterOperators[lowerOptr]; - - // invalid filter condition: %d", pQInfo, type - if (upperOptr != TSDB_RELATION_INVALID) { - return NULL; - } - } else { - funcFp = filterOperators[upperOptr]; - } - } - - return funcFp; -} diff --git a/2.0/src/query/src/qHistogram.c b/2.0/src/query/src/qHistogram.c deleted file mode 100644 index b406c0f70370117701466dd58504d7344f45a8b1..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qHistogram.c +++ /dev/null @@ -1,667 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "os.h" - -#include "qHistogram.h" -#include "taosdef.h" -#include "tmsg.h" -#include "tlosertree.h" - -/** - * - * implement the histogram and percentile_approx based on the paper: - * Yael Ben-Haim, Elad Tom-Tov. A Streaming Parallel Decision Tree Algorithm, - * The Journal of Machine Learning Research.Volume 11, 3/1/2010 pp.849-872 - * https://dl.acm.org/citation.cfm?id=1756034 - * - * @data 2018-12-14 - * @version 0.1 - * - */ - -// SHeapEntry* tHeapCreate(int32_t numOfEntries) { -// SHeapEntry* pEntry = calloc(1, sizeof(SHeapEntry)*(numOfEntries + 1)); -// return pEntry; -//} -// -// int32_t tHeapPut(SHeapEntry* pEntry, int32_t maxSize, int32_t num, void* -// pData, double v) { -// pEntry[num].val = v; -// pEntry[num].pData = pData; -// -// return num; -//} -// -////min heap -// void tHeapAdjust(SHeapEntry* pEntry, int32_t index, int32_t len) { -// SHeapEntry* ptr = NULL; -// -// int32_t end = len - 1; -// -// SHeapEntry p1 = pEntry[index]; -// int32_t next = index; -// -// for(int32_t i=index; i<=(end-1)/2; ) { -// int32_t lc = (i<<1) + 1; -// int32_t rc = (i+1) << 1; -// -// ptr = &pEntry[lc]; -// next = lc; -// -// if (rc < len && (pEntry[lc].val > pEntry[rc].val)) { -// ptr = &pEntry[rc]; -// next = rc; -// } -// -// if (p1.val < ptr->val) { -// next = i; -// break; -// } -// pEntry[i] = *ptr; -// tSkipListNode* pnode = (tSkipListNode*) pEntry[i].pData; -// if(pnode != NULL) { -// ((SHistBin*) pnode->pData)->index = i; -// } -// -// i = next; -// } -// -// pEntry[next] = p1; -// -// tSkipListNode* pnode = (tSkipListNode*) p1.pData; -// if (pnode != NULL) { -// ((SHistBin*) pnode->pData)->index = next; -// } -//} -// -// void tHeapSort(SHeapEntry* pEntry, int32_t len) { -// int32_t last = len/2 - 1; -// -// for(int32_t i=last; i >= 0; --i) { -// tHeapAdjust(pEntry, i, len); -// } -//} - -// typedef struct SInsertSupporter { -// int32_t numOfEntries; -// tSkipList* pSkipList; -// SLoserTreeInfo* pTree; -//} SInsertSupporter; -// -// int32_t compare(const void* pleft, const void* pright, void* param) { -// SLoserTreeNode* left = (SLoserTreeNode*) pleft; -// SLoserTreeNode* right = (SLoserTreeNode *)pright; -// -// SInsertSupporter* pss = (SInsertSupporter*) param; -// -// tSkipListNode* pLeftNode = (tSkipListNode*) left->pData; -// tSkipListNode* pRightNode = (tSkipListNode*) right->pData; -// -// SHistBin* pLeftBin = (SHistBin*)pLeftNode->pData; -// SHistBin* pRightBin = (SHistBin*)pRightNode->pData; -// -// if (pLeftBin->delta == pRightBin->delta) { -// return 0; -// } else { -// return ((pLeftBin->delta < pRightBin->delta)? -1:1); -// } -//} - -static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val); - -SHistogramInfo* tHistogramCreate(int32_t numOfEntries) { - /* need one redundant slot */ - SHistogramInfo* pHisto = malloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1)); - -#if !defined(USE_ARRAYLIST) - pHisto->pList = SSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - SInsertSupporter* pss = malloc(sizeof(SInsertSupporter)); - pss->numOfEntries = pHisto->maxEntries; - pss->pSkipList = pHisto->pList; - - int32_t ret = tLoserTreeCreate1(&pHisto->pLoserTree, numOfEntries, pss, compare); - pss->pTree = pHisto->pLoserTree; -#endif - - return tHistogramCreateFrom(pHisto, numOfEntries); -} - -SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) { - memset(pBuf, 0, sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfBins + 1)); - - SHistogramInfo* pHisto = (SHistogramInfo*)pBuf; - pHisto->elems = (SHistBin*)((char*)pBuf + sizeof(SHistogramInfo)); - for(int32_t i = 0; i < numOfBins; ++i) { - pHisto->elems[i].val = -DBL_MAX; - } - - pHisto->maxEntries = numOfBins; - - pHisto->min = DBL_MAX; - pHisto->max = -DBL_MAX; - - return pBuf; -} - -int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { - if (*pHisto == NULL) { - *pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN); - } - -#if defined(USE_ARRAYLIST) - int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); - assert(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL); - - if ((*pHisto)->elems[idx].val == val && idx >= 0) { - (*pHisto)->elems[idx].num += 1; - - if ((*pHisto)->numOfEntries == 0) { - (*pHisto)->numOfEntries += 1; - } - } else { /* insert a new slot */ - if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) { - if (idx > 0) { - assert((*pHisto)->elems[idx - 1].val <= val); - } else { - assert((*pHisto)->elems[idx].val > val); - } - } else if ((*pHisto)->numOfElems > 0) { - assert((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val); - } - - histogramCreateBin(*pHisto, idx, val); - } -#else - tSkipListKey key = tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &val, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize); - SHistBin* entry = calloc(1, sizeof(SHistBin)); - entry->val = val; - - tSkipListNode* pResNode = SSkipListPut((*pHisto)->pList, entry, &key, 0); - - SHistBin* pEntry1 = (SHistBin*)pResNode->pData; - pEntry1->index = -1; - - tSkipListNode* pLast = NULL; - - if (pEntry1->num == 0) { /* it is a new node */ - (*pHisto)->numOfEntries += 1; - pEntry1->num += 1; - - /* number of entries reaches the upper limitation */ - if (pResNode->pForward[0] != NULL) { - /* we need to update the last updated slot in loser tree*/ - pEntry1->delta = ((SHistBin*)pResNode->pForward[0]->pData)->val - val; - - if ((*pHisto)->ordered) { - int32_t lastIndex = (*pHisto)->maxIndex; - SLoserTreeInfo* pTree = (*pHisto)->pLoserTree; - - (*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].pData = pResNode; - pEntry1->index = (*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].index; - - // update the loser tree - if ((*pHisto)->ordered) { - tLoserTreeAdjust(pTree, pEntry1->index + pTree->numOfEntries); - } - - tSkipListKey kx = - tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &(*pHisto)->max, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize); - pLast = tSkipListGetOne((*pHisto)->pList, &kx); - } - } else { - /* this node located at the last position of the skiplist, we do not - * update the loser-tree */ - pEntry1->delta = DBL_MAX; - pLast = pResNode; - } - - if (pResNode->pBackward[0] != &(*pHisto)->pList->pHead) { - SHistBin* pPrevEntry = (SHistBin*)pResNode->pBackward[0]->pData; - pPrevEntry->delta = val - pPrevEntry->val; - - SLoserTreeInfo* pTree = (*pHisto)->pLoserTree; - if ((*pHisto)->ordered) { - tLoserTreeAdjust(pTree, pPrevEntry->index + pTree->numOfEntries); - tLoserTreeDisplay(pTree); - } - } - - if ((*pHisto)->numOfEntries >= (*pHisto)->maxEntries + 1) { - // set the right value for loser-tree - assert((*pHisto)->pLoserTree != NULL); - if (!(*pHisto)->ordered) { - SSkipListPrint((*pHisto)->pList, 1); - - SLoserTreeInfo* pTree = (*pHisto)->pLoserTree; - tSkipListNode* pHead = (*pHisto)->pList->pHead.pForward[0]; - - tSkipListNode* p1 = pHead; - - printf("\n"); - while (p1 != NULL) { - printf("%f\t", ((SHistBin*)(p1->pData))->delta); - p1 = p1->pForward[0]; - } - printf("\n"); - - /* last one in skiplist is ignored */ - for (int32_t i = pTree->numOfEntries; i < pTree->totalEntries; ++i) { - pTree->pNode[i].pData = pHead; - pTree->pNode[i].index = i - pTree->numOfEntries; - SHistBin* pBin = (SHistBin*)pHead->pData; - pBin->index = pTree->pNode[i].index; - - pHead = pHead->pForward[0]; - } - - pLast = pHead; - - for (int32_t i = 0; i < pTree->numOfEntries; ++i) { - pTree->pNode[i].index = -1; - } - - tLoserTreeDisplay(pTree); - - for (int32_t i = pTree->totalEntries - 1; i >= pTree->numOfEntries; i--) { - tLoserTreeAdjust(pTree, i); - } - - tLoserTreeDisplay(pTree); - (*pHisto)->ordered = true; - } - - printf("delta is:%lf\n", pEntry1->delta); - - SSkipListPrint((*pHisto)->pList, 1); - - /* the chosen node */ - tSkipListNode* pNode = (*pHisto)->pLoserTree->pNode[0].pData; - SHistBin* pEntry = (SHistBin*)pNode->pData; - - tSkipListNode* pNext = pNode->pForward[0]; - SHistBin* pNextEntry = (SHistBin*)pNext->pData; - assert(pNextEntry->val - pEntry->val == pEntry->delta); - - double newVal = (pEntry->val * pEntry->num + pNextEntry->val * pNextEntry->num) / (pEntry->num + pNextEntry->num); - pEntry->val = newVal; - pNode->key.dKey = newVal; - pEntry->num = pEntry->num + pNextEntry->num; - - // update delta value in current node - pEntry->delta = (pNextEntry->delta + pNextEntry->val) - pEntry->val; - - // reset delta value in the previous node - SHistBin* pPrevEntry = (SHistBin*)pNode->pBackward[0]->pData; - if (pPrevEntry) { - pPrevEntry->delta = pEntry->val - pPrevEntry->val; - } - - SLoserTreeInfo* pTree = (*pHisto)->pLoserTree; - if (pNextEntry->index != -1) { - (*pHisto)->maxIndex = pNextEntry->index; - - // set the last element in skiplist, of which delta is FLT_MAX; - pTree->pNode[pNextEntry->index + pTree->numOfEntries].pData = pLast; - ((SHistBin*)pLast->pData)->index = pNextEntry->index; - int32_t f = pTree->pNode[pNextEntry->index + pTree->numOfEntries].index; - printf("disappear index is:%d\n", f); - } - - tLoserTreeAdjust(pTree, pEntry->index + pTree->numOfEntries); - // remove the next node in skiplist - tSkipListRemoveNode((*pHisto)->pList, pNext); - SSkipListPrint((*pHisto)->pList, 1); - - tLoserTreeDisplay((*pHisto)->pLoserTree); - } else { // add to heap - if (pResNode->pForward[0] != NULL) { - pEntry1->delta = ((SHistBin*)pResNode->pForward[0]->pData)->val - val; - } else { - pEntry1->delta = DBL_MAX; - } - - if (pResNode->pBackward[0] != &(*pHisto)->pList->pHead) { - SHistBin* pPrevEntry = (SHistBin*)pResNode->pBackward[0]->pData; - pEntry1->delta = val - pPrevEntry->val; - } - - printf("delta is:%9lf\n", pEntry1->delta); - } - - } else { - SHistBin* pEntry = (SHistBin*)pResNode->pData; - assert(pEntry->val == val); - pEntry->num += 1; - } - -#endif - if (val > (*pHisto)->max) { - (*pHisto)->max = val; - } - - if (val < (*pHisto)->min) { - (*pHisto)->min = val; - } - - (*pHisto)->numOfElems += 1; - return 0; -} - -int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) { - int32_t end = len - 1; - int32_t start = 0; - - while (start <= end) { - int32_t mid = (end - start) / 2 + start; - if (pEntry[mid].val == val) { - return mid; - } - - if (pEntry[mid].val < val) { - start = mid + 1; - } else { - end = mid - 1; - } - } - - int32_t ret = start > end ? start : end; - if (ret < 0) { - return 0; - } else { - return ret; - } -} - -static void histogramMergeImpl(SHistBin* pHistBin, int32_t* size) { -#if defined(USE_ARRAYLIST) - int32_t oldSize = *size; - - double delta = DBL_MAX; - int32_t index = -1; - for (int32_t i = 1; i < oldSize; ++i) { - double d = pHistBin[i].val - pHistBin[i - 1].val; - if (d < delta) { - delta = d; - index = i - 1; - } - } - - SHistBin* s1 = &pHistBin[index]; - SHistBin* s2 = &pHistBin[index + 1]; - - double newVal = (s1->val * s1->num + s2->val * s2->num) / (s1->num + s2->num); - s1->val = newVal; - s1->num = s1->num + s2->num; - - memmove(&pHistBin[index + 1], &pHistBin[index + 2], (oldSize - index - 2) * sizeof(SHistBin)); - (*size) -= 1; -#endif -} - -/* optimize this procedure */ -int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) { -#if defined(USE_ARRAYLIST) - int32_t remain = pHisto->numOfEntries - index; - if (remain > 0) { - memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain); - } - - assert(index >= 0 && index <= pHisto->maxEntries); - - pHisto->elems[index].num = 1; - pHisto->elems[index].val = val; - pHisto->numOfEntries += 1; - - /* we need to merge the slot */ - if (pHisto->numOfEntries == pHisto->maxEntries + 1) { - histogramMergeImpl(pHisto->elems, &pHisto->numOfEntries); - - pHisto->elems[pHisto->maxEntries].val = 0; - pHisto->elems[pHisto->maxEntries].num = 0; - } -#endif - assert(pHisto->numOfEntries <= pHisto->maxEntries); - return 0; -} - -void tHistogramDestroy(SHistogramInfo** pHisto) { - if (*pHisto == NULL) { - return; - } - - free(*pHisto); - *pHisto = NULL; -} - -void tHistogramPrint(SHistogramInfo* pHisto) { - printf("total entries: %d, elements: %"PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems); -#if defined(USE_ARRAYLIST) - for (int32_t i = 0; i < pHisto->numOfEntries; ++i) { - printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num); - } -#else - tSkipListNode* pNode = pHisto->pList->pHead.pForward[0]; - - for (int32_t i = 0; i < pHisto->numOfEntries; ++i) { - SHistBin* pEntry = (SHistBin*)pNode->pData; - printf("%d: (%f, %" PRId64 ")\n", i + 1, pEntry->val, pEntry->num); - pNode = pNode->pForward[0]; - } -#endif -} - -/** - * Estimated number of points in the interval (−inf,b]. - * @param pHisto - * @param v - */ -int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { -#if defined(USE_ARRAYLIST) - int32_t slotIdx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, v); - if (pHisto->elems[slotIdx].val != v) { - slotIdx -= 1; - - if (slotIdx < 0) { - slotIdx = 0; - assert(v <= pHisto->elems[slotIdx].val); - } else { - assert(v >= pHisto->elems[slotIdx].val); - - if (slotIdx + 1 < pHisto->numOfEntries) { - assert(v < pHisto->elems[slotIdx + 1].val); - } - } - } - - double m1 = (double)pHisto->elems[slotIdx].num; - double v1 = pHisto->elems[slotIdx].val; - - double m2 = (double)pHisto->elems[slotIdx + 1].num; - double v2 = pHisto->elems[slotIdx + 1].val; - - double estNum = m1 + (m2 - m1) * (v - v1) / (v2 - v1); - double s1 = (m1 + estNum) * (v - v1) / (2 * (v2 - v1)); - - for (int32_t i = 0; i < slotIdx; ++i) { - s1 += pHisto->elems[i].num; - } - - s1 = s1 + m1 / 2; - - return (int64_t)s1; -#endif -} - -double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { -#if defined(USE_ARRAYLIST) - double* pVal = malloc(num * sizeof(double)); - - for (int32_t i = 0; i < num; ++i) { - double numOfElem = (ratio[i] / 100) * pHisto->numOfElems; - - if (numOfElem == 0) { - pVal[i] = pHisto->min; - continue; - } else if (numOfElem <= pHisto->elems[0].num) { - pVal[i] = pHisto->elems[0].val; - continue; - } else if (numOfElem == pHisto->numOfElems) { - pVal[i] = pHisto->max; - continue; - } - - int32_t j = 0; - int64_t total = 0; - - while (j < pHisto->numOfEntries) { - total += pHisto->elems[j].num; - if (total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem) { - break; - } - - j += 1; - } - - assert(total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem); - - double delta = numOfElem - total; - if (fabs(delta) < FLT_EPSILON) { - pVal[i] = pHisto->elems[j].val; - } - - double start = (double)pHisto->elems[j].num; - double range = pHisto->elems[j + 1].num - start; - - if (range == 0) { - pVal[i] = (pHisto->elems[j + 1].val - pHisto->elems[j].val) * delta / start + pHisto->elems[j].val; - } else { - double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range); - pVal[i] = pHisto->elems[j].val + (pHisto->elems[j + 1].val - pHisto->elems[j].val) * factor; - } - } -#else - double* pVal = malloc(num * sizeof(double)); - - for (int32_t i = 0; i < num; ++i) { - double numOfElem = ratio[i] * pHisto->numOfElems; - - tSkipListNode* pFirst = pHisto->pList->pHead.pForward[0]; - SHistBin* pEntry = (SHistBin*)pFirst->pData; - if (numOfElem == 0) { - pVal[i] = pHisto->min; - printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); - continue; - } else if (numOfElem <= pEntry->num) { - pVal[i] = pEntry->val; - printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); - continue; - } else if (numOfElem == pHisto->numOfElems) { - pVal[i] = pHisto->max; - printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); - continue; - } - - int32_t j = 0; - int64_t total = 0; - SHistBin* pPrev = pEntry; - - while (j < pHisto->numOfEntries) { - if (total <= numOfElem && total + pEntry->num > numOfElem) { - break; - } - - total += pEntry->num; - pPrev = pEntry; - - pFirst = pFirst->pForward[0]; - pEntry = (SHistBin*)pFirst->pData; - - j += 1; - } - - assert(total <= numOfElem && total + pEntry->num > numOfElem); - - double delta = numOfElem - total; - if (fabs(delta) < FLT_EPSILON) { - // printf("i/numofSlot: %f, v:%f, %f\n", - // (double)i/numOfSlots, numOfElem, pHisto->elems[j].val); - pVal[i] = pPrev->val; - } - - double start = pPrev->num; - double range = pEntry->num - start; - - if (range == 0) { - pVal[i] = (pEntry->val - pPrev->val) * delta / start + pPrev->val; - } else { - double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range); - pVal[i] = pPrev->val + (pEntry->val - pPrev->val) * factor; - } - // printf("i/numofSlot: %f, v:%f, %f\n", (double)i/numOfSlots, - // numOfElem, val); - } -#endif - return pVal; -} - -SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries) { - SHistogramInfo* pResHistogram = tHistogramCreate(numOfEntries); - - // error in histogram info - if (pHisto1->numOfEntries > MAX_HISTOGRAM_BIN || pHisto2->numOfEntries > MAX_HISTOGRAM_BIN) { - return pResHistogram; - } - - SHistBin* pHistoBins = calloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries)); - int32_t i = 0, j = 0, k = 0; - - while (i < pHisto1->numOfEntries && j < pHisto2->numOfEntries) { - if (pHisto1->elems[i].val < pHisto2->elems[j].val) { - pHistoBins[k++] = pHisto1->elems[i++]; - } else if (pHisto1->elems[i].val > pHisto2->elems[j].val) { - pHistoBins[k++] = pHisto2->elems[j++]; - } else { - pHistoBins[k] = pHisto1->elems[i++]; - pHistoBins[k++].num += pHisto2->elems[j++].num; - } - } - - if (i < pHisto1->numOfEntries) { - int32_t remain = pHisto1->numOfEntries - i; - memcpy(&pHistoBins[k], &pHisto1->elems[i], sizeof(SHistBin) * remain); - k += remain; - } - - if (j < pHisto2->numOfEntries) { - int32_t remain = pHisto2->numOfEntries - j; - memcpy(&pHistoBins[k], &pHisto2->elems[j], sizeof(SHistBin) * remain); - k += remain; - } - - /* update other information */ - pResHistogram->numOfElems = pHisto1->numOfElems + pHisto2->numOfElems; - pResHistogram->min = (pHisto1->min < pHisto2->min) ? pHisto1->min : pHisto2->min; - pResHistogram->max = (pHisto1->max > pHisto2->max) ? pHisto1->max : pHisto2->max; - - while (k > numOfEntries) { - histogramMergeImpl(pHistoBins, &k); - } - - pResHistogram->numOfEntries = k; - memcpy(pResHistogram->elems, pHistoBins, sizeof(SHistBin) * k); - - free(pHistoBins); - return pResHistogram; -} diff --git a/2.0/src/query/src/qPercentile.c b/2.0/src/query/src/qPercentile.c deleted file mode 100644 index e9022db503f005ae6713e66e47bbde440bb4aaf7..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qPercentile.c +++ /dev/null @@ -1,535 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "os.h" - -#include "qPercentile.h" -#include "qResultbuf.h" -#include "queryLog.h" -#include "taosdef.h" -#include "tcompare.h" -#include "ttype.h" - -#define DEFAULT_NUM_OF_SLOT 1024 - -int32_t getGroupId(int32_t numOfSlots, int32_t slotIndex, int32_t times) { - return (times * numOfSlots) + slotIndex; -} - -static tFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) { - tFilePage *buffer = (tFilePage *)calloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(tFilePage)); - - int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times); - SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId); - - int32_t offset = 0; - for(int32_t i = 0; i < list->size; ++i) { - SPageInfo* pgInfo = *(SPageInfo**) taosArrayGet(list, i); - - tFilePage* pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId); - memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes)); - - offset += (int32_t)(pg->num * pMemBucket->bytes); - } - - qsort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn); - return buffer; -} - -static void resetBoundingBox(MinMaxEntry* range, int32_t type) { - if (IS_SIGNED_NUMERIC_TYPE(type)) { - range->i64MaxVal = INT64_MIN; - range->i64MinVal = INT64_MAX; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - range->u64MaxVal = 0; - range->u64MinVal = UINT64_MAX; - } else { - range->dMaxVal = -DBL_MAX; - range->dMinVal = DBL_MAX; - } -} - -static int32_t setBoundingBox(MinMaxEntry* range, int16_t type, double minval, double maxval) { - if (minval > maxval) { - return -1; - } - - if (IS_SIGNED_NUMERIC_TYPE(type)) { - range->i64MinVal = (int64_t) minval; - range->i64MaxVal = (int64_t) maxval; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)){ - range->u64MinVal = (uint64_t) minval; - range->u64MaxVal = (uint64_t) maxval; - } else { - range->dMinVal = minval; - range->dMaxVal = maxval; - } - - return 0; -} - -static void resetPosInfo(SSlotInfo* pInfo) { - pInfo->size = 0; - pInfo->pageId = -1; - pInfo->data = NULL; -} - -double findOnlyResult(tMemBucket *pMemBucket) { - assert(pMemBucket->total == 1); - - for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) { - tMemBucketSlot *pSlot = &pMemBucket->pSlots[i]; - if (pSlot->info.size == 0) { - continue; - } - - int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times); - SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId); - assert(list->size == 1); - - SPageInfo* pgInfo = (SPageInfo*) taosArrayGetP(list, 0); - tFilePage* pPage = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId); - assert(pPage->num == 1); - - double v = 0; - GET_TYPED_DATA(v, double, pMemBucket->type, pPage->data); - return v; - } - - return 0; -} - -int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pBucket->type, value); - - int32_t index = -1; - - if (v > pBucket->range.i64MaxVal || v < pBucket->range.i64MinVal) { - return index; - } - - // divide the value range into 1024 buckets - uint64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal; - if (span < pBucket->numOfSlots) { - int64_t delta = v - pBucket->range.i64MinVal; - index = (delta % pBucket->numOfSlots); - } else { - double slotSpan = (double)span / pBucket->numOfSlots; - index = (int32_t)((v - pBucket->range.i64MinVal) / slotSpan); - if (v == pBucket->range.i64MaxVal) { - index -= 1; - } - } - - assert(index >= 0 && index < pBucket->numOfSlots); - return index; -} - -int32_t tBucketUintHash(tMemBucket *pBucket, const void *value) { - int64_t v = 0; - GET_TYPED_DATA(v, uint64_t, pBucket->type, value); - - int32_t index = -1; - - if (v > pBucket->range.u64MaxVal || v < pBucket->range.u64MinVal) { - return index; - } - - // divide the value range into 1024 buckets - uint64_t span = pBucket->range.u64MaxVal - pBucket->range.u64MinVal; - if (span < pBucket->numOfSlots) { - int64_t delta = v - pBucket->range.u64MinVal; - index = (int32_t) (delta % pBucket->numOfSlots); - } else { - double slotSpan = (double)span / pBucket->numOfSlots; - index = (int32_t)((v - pBucket->range.u64MinVal) / slotSpan); - if (v == pBucket->range.u64MaxVal) { - index -= 1; - } - } - - assert(index >= 0 && index < pBucket->numOfSlots); - return index; -} - -int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) { - double v = 0; - if (pBucket->type == TSDB_DATA_TYPE_FLOAT) { - v = GET_FLOAT_VAL(value); - } else { - v = GET_DOUBLE_VAL(value); - } - - int32_t index = -1; - - if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal) { - return index; - } - - // divide a range of [dMinVal, dMaxVal] into 1024 buckets - double span = pBucket->range.dMaxVal - pBucket->range.dMinVal; - if (span < pBucket->numOfSlots) { - int32_t delta = (int32_t)(v - pBucket->range.dMinVal); - index = (delta % pBucket->numOfSlots); - } else { - double slotSpan = span / pBucket->numOfSlots; - index = (int32_t)((v - pBucket->range.dMinVal) / slotSpan); - if (v == pBucket->range.dMaxVal) { - index -= 1; - } - } - - assert(index >= 0 && index < pBucket->numOfSlots); - return index; -} - -static __perc_hash_func_t getHashFunc(int32_t type) { - if (IS_SIGNED_NUMERIC_TYPE(type)) { - return tBucketIntHash; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - return tBucketUintHash; - } else { - return tBucketDoubleHash; - } -} - -static void resetSlotInfo(tMemBucket* pBucket) { - for (int32_t i = 0; i < pBucket->numOfSlots; ++i) { - tMemBucketSlot* pSlot = &pBucket->pSlots[i]; - - resetBoundingBox(&pSlot->range, pBucket->type); - resetPosInfo(&pSlot->info); - } -} - -tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) { - tMemBucket *pBucket = (tMemBucket *)calloc(1, sizeof(tMemBucket)); - if (pBucket == NULL) { - return NULL; - } - - pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT; - pBucket->bufPageSize = DEFAULT_PAGE_SIZE * 4; // 4k per page - - pBucket->type = dataType; - pBucket->bytes = nElemSize; - pBucket->total = 0; - pBucket->times = 1; - - pBucket->maxCapacity = 200000; - - if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) { - qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval); - free(pBucket); - return NULL; - } - - pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(tFilePage))/pBucket->bytes; - pBucket->comparFn = getKeyComparFunc(pBucket->type, TSDB_ORDER_ASC); - - pBucket->hashFunc = getHashFunc(pBucket->type); - if (pBucket->hashFunc == NULL) { - qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type); - free(pBucket); - return NULL; - } - - pBucket->pSlots = (tMemBucketSlot *)calloc(pBucket->numOfSlots, sizeof(tMemBucketSlot)); - if (pBucket->pSlots == NULL) { - free(pBucket); - return NULL; - } - - resetSlotInfo(pBucket); - - int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1); - if (ret != TSDB_CODE_SUCCESS) { - tMemBucketDestroy(pBucket); - return NULL; - } - - qDebug("MemBucket:%p, elem size:%d", pBucket, pBucket->bytes); - return pBucket; -} - -void tMemBucketDestroy(tMemBucket *pBucket) { - if (pBucket == NULL) { - return; - } - - destroyResultBuf(pBucket->pBuffer); - tfree(pBucket->pSlots); - tfree(pBucket); -} - -void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) { - if (IS_SIGNED_NUMERIC_TYPE(dataType)) { - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, dataType, data); - - if (r->i64MinVal > v) { - r->i64MinVal = v; - } - - if (r->i64MaxVal < v) { - r->i64MaxVal = v; - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) { - uint64_t v = 0; - GET_TYPED_DATA(v, uint64_t, dataType, data); - - if (r->i64MinVal > v) { - r->i64MinVal = v; - } - - if (r->i64MaxVal < v) { - r->i64MaxVal = v; - } - } else if (IS_FLOAT_TYPE(dataType)) { - double v = 0; - GET_TYPED_DATA(v, double, dataType, data); - - if (r->dMinVal > v) { - r->dMinVal = v; - } - - if (r->dMaxVal < v) { - r->dMaxVal = v; - } - } else { - assert(0); - } -} - -/* - * in memory bucket, we only accept data array list - */ -int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { - assert(pBucket != NULL && data != NULL && size > 0); - - int32_t count = 0; - int32_t bytes = pBucket->bytes; - for (int32_t i = 0; i < size; ++i) { - char *d = (char *) data + i * bytes; - - int32_t index = (pBucket->hashFunc)(pBucket, d); - if (index < 0) { - continue; - } - - count += 1; - - tMemBucketSlot *pSlot = &pBucket->pSlots[index]; - tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type); - - // ensure available memory pages to allocate - int32_t groupId = getGroupId(pBucket->numOfSlots, index, pBucket->times); - int32_t pageId = -1; - - if (pSlot->info.data == NULL || pSlot->info.data->num >= pBucket->elemPerPage) { - if (pSlot->info.data != NULL) { - assert(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0); - - // keep the pointer in memory - releaseResBufPage(pBucket->pBuffer, pSlot->info.data); - pSlot->info.data = NULL; - } - - pSlot->info.data = getNewDataBuf(pBucket->pBuffer, groupId, &pageId); - pSlot->info.pageId = pageId; - } - - memcpy(pSlot->info.data->data + pSlot->info.data->num * pBucket->bytes, d, pBucket->bytes); - - pSlot->info.data->num += 1; - pSlot->info.size += 1; - } - - pBucket->total += count; - return 0; -} - -//////////////////////////////////////////////////////////////////////////////////////////// -/* - * - * now, we need to find the minimum value of the next slot for - * interpolating the percentile value - * j is the last slot of current segment, we need to get the first - * slot of the next segment. - */ -static MinMaxEntry getMinMaxEntryOfNextSlotWithData(tMemBucket *pMemBucket, int32_t slotIdx) { - int32_t j = slotIdx + 1; - while (j < pMemBucket->numOfSlots && (pMemBucket->pSlots[j].info.size == 0)) { - ++j; - } - - assert(j < pMemBucket->numOfSlots); - return pMemBucket->pSlots[j].range; -} - -static bool isIdenticalData(tMemBucket *pMemBucket, int32_t index); - -static double getIdenticalDataVal(tMemBucket* pMemBucket, int32_t slotIndex) { - assert(isIdenticalData(pMemBucket, slotIndex)); - - tMemBucketSlot *pSlot = &pMemBucket->pSlots[slotIndex]; - - double finalResult = 0.0; - if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { - finalResult = (double) pSlot->range.i64MinVal; - } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { - finalResult = (double) pSlot->range.u64MinVal; - } else { - finalResult = (double) pSlot->range.dMinVal; - } - - return finalResult; -} - -double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) { - int32_t num = 0; - - for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) { - tMemBucketSlot *pSlot = &pMemBucket->pSlots[i]; - if (pSlot->info.size == 0) { - continue; - } - - // required value in current slot - if (num < (count + 1) && num + pSlot->info.size >= (count + 1)) { - if (pSlot->info.size + num == (count + 1)) { - /* - * now, we need to find the minimum value of the next slot for interpolating the percentile value - * j is the last slot of current segment, we need to get the first slot of the next segment. - */ - MinMaxEntry next = getMinMaxEntryOfNextSlotWithData(pMemBucket, i); - - double maxOfThisSlot = 0; - double minOfNextSlot = 0; - if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { - maxOfThisSlot = (double) pSlot->range.i64MaxVal; - minOfNextSlot = (double) next.i64MinVal; - } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { - maxOfThisSlot = (double) pSlot->range.u64MaxVal; - minOfNextSlot = (double) next.u64MinVal; - } else { - maxOfThisSlot = (double) pSlot->range.dMaxVal; - minOfNextSlot = (double) next.dMinVal; - } - - assert(minOfNextSlot > maxOfThisSlot); - - double val = (1 - fraction) * maxOfThisSlot + fraction * minOfNextSlot; - return val; - } - - if (pSlot->info.size <= pMemBucket->maxCapacity) { - // data in buffer and file are merged together to be processed. - tFilePage *buffer = loadDataFromFilePage(pMemBucket, i); - int32_t currentIdx = count - num; - - char *thisVal = buffer->data + pMemBucket->bytes * currentIdx; - char *nextVal = thisVal + pMemBucket->bytes; - - double td = 1.0, nd = 1.0; - GET_TYPED_DATA(td, double, pMemBucket->type, thisVal); - GET_TYPED_DATA(nd, double, pMemBucket->type, nextVal); - - double val = (1 - fraction) * td + fraction * nd; - tfree(buffer); - - return val; - } else { // incur a second round bucket split - if (isIdenticalData(pMemBucket, i)) { - return getIdenticalDataVal(pMemBucket, i); - } - - // try next round - pMemBucket->times += 1; - qDebug("MemBucket:%p, start next round data bucketing, time:%d", pMemBucket, pMemBucket->times); - - pMemBucket->range = pSlot->range; - pMemBucket->total = 0; - - resetSlotInfo(pMemBucket); - - int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1); - SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId); - assert(list->size > 0); - - for (int32_t f = 0; f < list->size; ++f) { - SPageInfo *pgInfo = *(SPageInfo **)taosArrayGet(list, f); - tFilePage *pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId); - - tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num); - releaseResBufPageInfo(pMemBucket->pBuffer, pgInfo); - } - - return getPercentileImpl(pMemBucket, count - num, fraction); - } - } else { - num += pSlot->info.size; - } - } - - return 0; -} - -double getPercentile(tMemBucket *pMemBucket, double percent) { - if (pMemBucket->total == 0) { - return 0.0; - } - - // if only one elements exists, return it - if (pMemBucket->total == 1) { - return findOnlyResult(pMemBucket); - } - - percent = fabs(percent); - - // find the min/max value, no need to scan all data in bucket - if (fabs(percent - 100.0) < DBL_EPSILON || (percent < DBL_EPSILON)) { - MinMaxEntry* pRange = &pMemBucket->range; - - if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { - double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal); - return v; - } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { - double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->u64MaxVal : pRange->u64MinVal); - return v; - } else { - return fabs(percent - 100) < DBL_EPSILON? pRange->dMaxVal:pRange->dMinVal; - } - } - - double percentVal = (percent * (pMemBucket->total - 1)) / ((double)100.0); - - // do put data by using buckets - int32_t orderIdx = (int32_t)percentVal; - return getPercentileImpl(pMemBucket, orderIdx, percentVal - orderIdx); -} - -/* - * check if data in one slot are all identical only need to compare with the bounding box - */ -bool isIdenticalData(tMemBucket *pMemBucket, int32_t index) { - tMemBucketSlot *pSeg = &pMemBucket->pSlots[index]; - - if (IS_FLOAT_TYPE(pMemBucket->type)) { - return fabs(pSeg->range.dMaxVal - pSeg->range.dMinVal) < DBL_EPSILON; - } else { - return pSeg->range.i64MinVal == pSeg->range.i64MaxVal; - } -} diff --git a/2.0/src/query/src/qPlan.c b/2.0/src/query/src/qPlan.c deleted file mode 100644 index 61ad0a04987310a9f331dc3c99ddaa9b55a82552..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qPlan.c +++ /dev/null @@ -1,714 +0,0 @@ -#include "os.h" -#include "qTableMeta.h" -#include "qPlan.h" -#include "qExecutor.h" -#include "qUtil.h" -#include "texpr.h" -#include "tscUtil.h" -#include "tsclient.h" - -#define QNODE_TAGSCAN 1 -#define QNODE_TABLESCAN 2 -#define QNODE_PROJECT 3 -#define QNODE_AGGREGATE 4 -#define QNODE_GROUPBY 5 -#define QNODE_LIMIT 6 -#define QNODE_JOIN 7 -#define QNODE_DISTINCT 8 -#define QNODE_SORT 9 -#define QNODE_UNIONALL 10 -#define QNODE_TIMEWINDOW 11 -#define QNODE_SESSIONWINDOW 12 -#define QNODE_FILL 13 - -typedef struct SFillEssInfo { - int32_t fillType; // fill type - int64_t *val; // fill value -} SFillEssInfo; - -typedef struct SJoinCond { - bool tagExists; // denote if tag condition exists or not - SColumn *tagCond[2]; - SColumn *colCond[2]; -} SJoinCond; - -static SQueryNode* createQueryNode(int32_t type, const char* name, SQueryNode** prev, int32_t numOfPrev, - SExprInfo** pExpr, int32_t numOfOutput, SQueryTableInfo* pTableInfo, - void* pExtInfo) { - SQueryNode* pNode = calloc(1, sizeof(SQueryNode)); - - pNode->info.type = type; - pNode->info.name = strdup(name); - - if (pTableInfo->id.uid != 0 && pTableInfo->tableName) { // it is a true table - pNode->tableInfo.id = pTableInfo->id; - pNode->tableInfo.tableName = strdup(pTableInfo->tableName); - } - - pNode->numOfOutput = numOfOutput; - pNode->pExpr = calloc(numOfOutput, sizeof(SExprInfo)); - for(int32_t i = 0; i < numOfOutput; ++i) { - tscExprAssign(&pNode->pExpr[i], pExpr[i]); - } - - pNode->pPrevNodes = taosArrayInit(4, POINTER_BYTES); - for(int32_t i = 0; i < numOfPrev; ++i) { - taosArrayPush(pNode->pPrevNodes, &prev[i]); - } - - switch(type) { - case QNODE_TABLESCAN: { - STimeWindow* window = calloc(1, sizeof(STimeWindow)); - memcpy(window, pExtInfo, sizeof(STimeWindow)); - pNode->pExtInfo = window; - break; - } - - case QNODE_TIMEWINDOW: { - SInterval* pInterval = calloc(1, sizeof(SInterval)); - pNode->pExtInfo = pInterval; - memcpy(pInterval, pExtInfo, sizeof(SInterval)); - break; - } - - case QNODE_GROUPBY: { - SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo; - SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr)); - - pGroupbyExpr->tableIndex = p->tableIndex; - pGroupbyExpr->orderType = p->orderType; - pGroupbyExpr->orderIndex = p->orderIndex; - pGroupbyExpr->numOfGroupCols = p->numOfGroupCols; - pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo); - pNode->pExtInfo = pGroupbyExpr; - break; - } - - case QNODE_FILL: { // todo !! - pNode->pExtInfo = pExtInfo; - break; - } - - case QNODE_LIMIT: { - pNode->pExtInfo = calloc(1, sizeof(SLimitVal)); - memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimitVal)); - break; - } - } - return pNode; -} - -static SQueryNode* doAddTableColumnNode(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, - SArray* pExprs, SArray* tableCols) { - if (pQueryInfo->onlyTagQuery) { - int32_t num = (int32_t) taosArrayGetSize(pExprs); - SQueryNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info, NULL); - - if (pQueryInfo->distinct) { - pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, info, NULL); - } - - return pNode; - } - - STimeWindow* window = &pQueryInfo->window; - SQueryNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info, window); - - if (pQueryInfo->projectionQuery) { - int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs); - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, info, NULL); - } else { - // table source column projection, generate the projection expr - int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols); - SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES); - SSchema* pSchema = pTableMetaInfo->pTableMeta->schema; - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(tableCols, i); - - SColumnIndex index = {.tableIndex = 0, .columnIndex = pCol->columnIndex}; - STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SExprInfo* p = tscExprCreate(pTableMetaInfo1, TSDB_FUNC_PRJ, &index, pCol->info.type, pCol->info.bytes, - pCol->info.colId, 0, TSDB_COL_NORMAL); - strncpy(p->base.aliasName, pSchema[pCol->columnIndex].name, tListLen(p->base.aliasName)); - - pExpr[i] = p; - } - - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, info, NULL); - for (int32_t i = 0; i < numOfCols; ++i) { - destroyQueryFuncExpr(pExpr[i], 1); - } - tfree(pExpr); - } - - return pNode; -} - -static SQueryNode* doCreateQueryPlanForOneTableImpl(SQueryInfo* pQueryInfo, SQueryNode* pNode, SQueryTableInfo* info, - SArray* pExprs) { - // check for aggregation - if (pQueryInfo->interval.interval > 0) { - int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs); - - pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, pExprs->pData, numOfOutput, info, - &pQueryInfo->interval); - if (pQueryInfo->groupbyExpr.numOfGroupCols != 0) { - pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, pExprs->pData, numOfOutput, info, &pQueryInfo->groupbyExpr); - } - } else if (pQueryInfo->groupbyColumn) { - int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs); - pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, pExprs->pData, numOfOutput, info, - &pQueryInfo->groupbyExpr); - } else if (pQueryInfo->sessionWindow.gap > 0) { - pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, NULL, 0, info, NULL); - } else if (pQueryInfo->simpleAgg) { - int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs); - pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, pExprs->pData, numOfOutput, info, NULL); - } - - if (pQueryInfo->havingFieldNum > 0 || pQueryInfo->arithmeticOnAgg) { - int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1); - pNode = - createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, info, NULL); - } - - if (pQueryInfo->fillType != TSDB_FILL_NONE) { - SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo)); - pInfo->fillType = pQueryInfo->fillType; - pInfo->val = calloc(pNode->numOfOutput, sizeof(int64_t)); - memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfOutput); - - pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, NULL, 0, info, pInfo); - } - - if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) { - pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, info, &pQueryInfo->limit); - } - - return pNode; -} - -static SQueryNode* doCreateQueryPlanForSingleTable(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, - SArray* tableCols) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - - SQueryTableInfo info = {.tableName = strdup(name), .id = pTableMetaInfo->pTableMeta->id,}; - - // handle the only tag query - SQueryNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols); - if (pQueryInfo->onlyTagQuery) { - tfree(info.tableName); - return pNode; - } - - SQueryNode* pNode1 = doCreateQueryPlanForOneTableImpl(pQueryInfo, pNode, &info, pExprs); - tfree(info.tableName); - return pNode1; -} - -SArray* createQueryPlanImpl(SQueryInfo* pQueryInfo) { - SArray* upstream = NULL; - - if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause - upstream = taosArrayInit(4, POINTER_BYTES); - - size_t size = taosArrayGetSize(pQueryInfo->pUpstream); - for(int32_t i = 0; i < size; ++i) { - SQueryInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i); - SArray* p = createQueryPlanImpl(pq); - taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p)); - } - } - - if (pQueryInfo->numOfTables > 1) { // it is a join query - // 1. separate the select clause according to table - taosArrayDestroy(upstream); - upstream = taosArrayInit(5, POINTER_BYTES); - - for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i]; - uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; - - SArray* exprList = taosArrayInit(4, POINTER_BYTES); - if (tscExprCopy(exprList, pQueryInfo->exprList, uid, true) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscExprDestroy(exprList); - exit(-1); - } - - // 2. create the query execution node - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - SQueryTableInfo info = {.tableName = strdup(name), .id = pTableMetaInfo->pTableMeta->id,}; - - // 3. get the required table column list - SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn)); - tscColumnListCopy(tableColumnList, pQueryInfo->colList, uid); - - // 4. add the projection query node - SQueryNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList); - tscColumnListDestroy(tableColumnList); - tscExprDestroy(exprList); - taosArrayPush(upstream, &pNode); - } - - // 3. add the join node here - SQueryTableInfo info = {0}; - int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList); - SQueryNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables, - pQueryInfo->exprList->pData, num, &info, NULL); - - // 4. add the aggregation or projection execution node - pNode = doCreateQueryPlanForOneTableImpl(pQueryInfo, pNode, &info, pQueryInfo->exprList); - upstream = taosArrayInit(5, POINTER_BYTES); - taosArrayPush(upstream, &pNode); - } else { // only one table, normal query process - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - SQueryNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList, pQueryInfo->colList); - upstream = taosArrayInit(5, POINTER_BYTES); - taosArrayPush(upstream, &pNode); - } - - return upstream; -} - -SQueryNode* qCreateQueryPlan(SQueryInfo* pQueryInfo) { - SArray* upstream = createQueryPlanImpl(pQueryInfo); - assert(taosArrayGetSize(upstream) == 1); - - SQueryNode* p = taosArrayGetP(upstream, 0); - taosArrayDestroy(upstream); - - return p; -} - -static void doDestroyQueryNode(SQueryNode* pQueryNode) { - tfree(pQueryNode->pExtInfo); - tfree(pQueryNode->pSchema); - tfree(pQueryNode->info.name); - - tfree(pQueryNode->tableInfo.tableName); - - pQueryNode->pExpr = destroyQueryFuncExpr(pQueryNode->pExpr, pQueryNode->numOfOutput); - - if (pQueryNode->pPrevNodes != NULL) { - int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pPrevNodes); - for(int32_t i = 0; i < size; ++i) { - SQueryNode* p = taosArrayGetP(pQueryNode->pPrevNodes, i); - doDestroyQueryNode(p); - } - - taosArrayDestroy(pQueryNode->pPrevNodes); - } - - tfree(pQueryNode); -} - -void* qDestroyQueryPlan(SQueryNode* pQueryNode) { - if (pQueryNode == NULL) { - return NULL; - } - - doDestroyQueryNode(pQueryNode); - return NULL; -} - -bool hasAliasName(SExprInfo* pExpr) { - assert(pExpr != NULL); - return strncmp(pExpr->base.token, pExpr->base.aliasName, tListLen(pExpr->base.aliasName)) != 0; -} - -static int32_t doPrintPlan(char* buf, SQueryNode* pQueryNode, int32_t level, int32_t totalLen) { - if (level > 0) { - sprintf(buf + totalLen, "%*c", level, ' '); - totalLen += level; - } - - int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name); - int32_t len = len1 + totalLen; - - switch(pQueryNode->info.type) { - case QNODE_TABLESCAN: { - STimeWindow* win = (STimeWindow*)pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64 "\n", - pQueryNode->tableInfo.tableName, pQueryNode->tableInfo.id.uid, win->skey, win->ekey); - len += len1; - break; - } - - case QNODE_PROJECT: { - len1 = sprintf(buf + len, "cols: "); - len += len1; - - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { - SSqlExpr* p = &pQueryNode->pExpr[i].base; - len1 = sprintf(buf + len, "[%s #%d]", p->aliasName, p->resColId); - len += len1; - - if (i < pQueryNode->numOfOutput - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } - - len1 = sprintf(buf + len, ")"); - len += len1; - - //todo print filter info - len1 = sprintf(buf + len, " filters:(nil)\n"); - len += len1; - break; - } - - case QNODE_AGGREGATE: { - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { - SSqlExpr* pExpr = &pQueryNode->pExpr[i].base; - if (hasAliasName(&pQueryNode->pExpr[i])) { - len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->aliasName); - } else { - len1 = sprintf(buf + len,"[%s]", pExpr->token); - } - - len += len1; - if (i < pQueryNode->numOfOutput - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } - - len1 = sprintf(buf + len, ")\n"); - len += len1; - break; - } - - case QNODE_TIMEWINDOW: { - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { - SSqlExpr* pExpr = &pQueryNode->pExpr[i].base; - if (hasAliasName(&pQueryNode->pExpr[i])) { - len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->aliasName); - } else { - len1 = sprintf(buf + len,"[%s]", pExpr->token); - } - - len += len1; - if (i < pQueryNode->numOfOutput - 1) { - len1 = sprintf(buf + len,", "); - len += len1; - } - } - - len1 = sprintf(buf + len,") "); - len += len1; - - SInterval* pInterval = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "\n", - pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding, TSDB_TIME_PRECISION_MILLI_STR, - pInterval->offset); - len += len1; - - break; - } - - case QNODE_GROUPBY: { // todo hide the invisible column - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { - SSqlExpr* pExpr = &pQueryNode->pExpr[i].base; - - if (hasAliasName(&pQueryNode->pExpr[i])) { - len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->aliasName); - } else { - len1 = sprintf(buf + len,"[%s]", pExpr->token); - } - - len += len1; - if (i < pQueryNode->numOfOutput - 1) { - len1 = sprintf(buf + len,", "); - len += len1; - } - } - - SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo; - SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, 0); - - len1 = sprintf(buf + len,") groupby_col: [%s #%d]\n", pIndex->name, pIndex->colId); - len += len1; - - break; - } - - case QNODE_FILL: { - SFillEssInfo* pEssInfo = pQueryNode->pExtInfo; - len1 = sprintf(buf + len,"%d", pEssInfo->fillType); - len += len1; - - if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) { - len1 = sprintf(buf + len,", val:"); - len += len1; - - // todo get the correct fill data type - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { - len1 = sprintf(buf + len,"%"PRId64, pEssInfo->val[i]); - len += len1; - - if (i < pQueryNode->numOfOutput - 1) { - len1 = sprintf(buf + len,", "); - len += len1; - } - } - } - - len1 = sprintf(buf + len,")\n"); - len += len1; - break; - } - - case QNODE_LIMIT: { - SLimitVal* pVal = pQueryNode->pExtInfo; - len1 = sprintf(buf + len,"limit: %"PRId64", offset: %"PRId64")\n", pVal->limit, pVal->offset); - len += len1; - break; - } - - case QNODE_DISTINCT: - case QNODE_TAGSCAN: { - len1 = sprintf(buf + len,"cols: "); - len += len1; - - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { - SSqlExpr* p = &pQueryNode->pExpr[i].base; - len1 = sprintf(buf + len,"[%s #%d]", p->aliasName, p->resColId); - len += len1; - - if (i < pQueryNode->numOfOutput - 1) { - len1 = sprintf(buf + len,", "); - len += len1; - } - } - - len1 = sprintf(buf + len,")\n"); - len += len1; - - break; - } - - case QNODE_JOIN: { - // print join condition - len1 = sprintf(buf + len, ")\n"); - len += len1; - break; - } - } - - return len; -} - -int32_t queryPlanToStringImpl(char* buf, SQueryNode* pQueryNode, int32_t level, int32_t totalLen) { - int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen); - - for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pPrevNodes); ++i) { - SQueryNode* p1 = taosArrayGetP(pQueryNode->pPrevNodes, i); - int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len); - len = len1; - } - - return len; -} - -char* queryPlanToString(SQueryNode* pQueryNode) { - assert(pQueryNode); - - char* buf = calloc(1, 4096); - - int32_t len = sprintf(buf, "===== logic plan =====\n"); - queryPlanToStringImpl(buf, pQueryNode, 0, len); - return buf; -} - -SQueryNode* queryPlanFromString() { - return NULL; -} - -SArray* createTableScanPlan(SQueryAttr* pQueryAttr) { - SArray* plan = taosArrayInit(4, sizeof(int32_t)); - - int32_t op = 0; - if (onlyQueryTags(pQueryAttr)) { -// op = OP_TagScan; - } else { - if (pQueryAttr->queryBlockDist) { - op = OP_TableBlockInfoScan; - } else if (pQueryAttr->tsCompQuery || pQueryAttr->pointInterpQuery || pQueryAttr->diffQuery) { - op = OP_TableSeqScan; - } else if (pQueryAttr->needReverseScan) { - op = OP_DataBlocksOptScan; - } else { - op = OP_TableScan; - } - - taosArrayPush(plan, &op); - } - - return plan; -} - -SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { - SArray* plan = taosArrayInit(4, sizeof(int32_t)); - int32_t op = 0; - - if (onlyQueryTags(pQueryAttr)) { // do nothing for tags query - op = OP_TagScan; - taosArrayPush(plan, &op); - - if (pQueryAttr->distinct) { - op = OP_Distinct; - taosArrayPush(plan, &op); - } - } else if (pQueryAttr->interval.interval > 0) { - if (pQueryAttr->stableQuery) { - if (pQueryAttr->pointInterpQuery) { - op = OP_AllMultiTableTimeInterval; - } else { - op = OP_MultiTableTimeInterval; - } - taosArrayPush(plan, &op); - } else { - if (pQueryAttr->pointInterpQuery) { - op = OP_AllTimeWindow; - } else { - op = OP_TimeWindow; - } - taosArrayPush(plan, &op); - - if (pQueryAttr->pExpr2 != NULL) { - op = OP_Project; - taosArrayPush(plan, &op); - } - - if (pQueryAttr->fillType != TSDB_FILL_NONE) { - op = OP_Fill; - taosArrayPush(plan, &op); - } - } - - } else if (pQueryAttr->groupbyColumn) { - op = OP_Groupby; - taosArrayPush(plan, &op); - - if (!pQueryAttr->stableQuery && pQueryAttr->havingNum > 0) { - op = OP_Filter; - taosArrayPush(plan, &op); - } - - if (pQueryAttr->pExpr2 != NULL) { - op = OP_Project; - taosArrayPush(plan, &op); - } - } else if (pQueryAttr->sw.gap > 0) { - op = OP_SessionWindow; - taosArrayPush(plan, &op); - - if (pQueryAttr->pExpr2 != NULL) { - op = OP_Project; - taosArrayPush(plan, &op); - } - } else if (pQueryAttr->stateWindow) { - op = OP_StateWindow; - taosArrayPush(plan, &op); - - if (pQueryAttr->pExpr2 != NULL) { - op = OP_Project; - taosArrayPush(plan, &op); - } - } else if (pQueryAttr->simpleAgg) { - if (pQueryAttr->stableQuery && !pQueryAttr->tsCompQuery && !pQueryAttr->diffQuery) { - op = OP_MultiTableAggregate; - } else { - op = OP_Aggregate; - } - - taosArrayPush(plan, &op); - - if (!pQueryAttr->stableQuery && pQueryAttr->havingNum > 0) { - op = OP_Filter; - taosArrayPush(plan, &op); - } - - if (pQueryAttr->pExpr2 != NULL && !pQueryAttr->stableQuery) { - op = OP_Project; - taosArrayPush(plan, &op); - } - } else { // diff/add/multiply/subtract/division - if (pQueryAttr->numOfFilterCols > 0 && pQueryAttr->createFilterOperator && pQueryAttr->vgId == 0) { // todo refactor - op = OP_Filter; - taosArrayPush(plan, &op); - } else { - op = OP_Project; - taosArrayPush(plan, &op); - if (pQueryAttr->distinct) { - op = OP_Distinct; - taosArrayPush(plan, &op); - } - } - - // outer query order by support - int32_t orderColId = pQueryAttr->order.orderColId; - if (pQueryAttr->vgId == 0 && orderColId != PRIMARYKEY_TIMESTAMP_COL_INDEX && orderColId != INT32_MIN) { - op = OP_Order; - taosArrayPush(plan, &op); - } - } - - if (pQueryAttr->limit.limit > 0 || pQueryAttr->limit.offset > 0) { - op = OP_Limit; - taosArrayPush(plan, &op); - } - - return plan; -} - -SArray* createGlobalMergePlan(SQueryAttr* pQueryAttr) { - SArray* plan = taosArrayInit(4, sizeof(int32_t)); - - if (!pQueryAttr->stableQuery) { - return plan; - } - - int32_t op = OP_MultiwayMergeSort; - taosArrayPush(plan, &op); - - if (pQueryAttr->distinct) { - op = OP_Distinct; - taosArrayPush(plan, &op); - } - - if (pQueryAttr->simpleAgg || (pQueryAttr->interval.interval > 0 || pQueryAttr->sw.gap > 0)) { - op = OP_GlobalAggregate; - taosArrayPush(plan, &op); - - if (pQueryAttr->havingNum > 0) { - op = OP_Filter; - taosArrayPush(plan, &op); - } - - if (pQueryAttr->pExpr2 != NULL) { - op = OP_Project; - taosArrayPush(plan, &op); - } - } - - // fill operator - if (pQueryAttr->fillType != TSDB_FILL_NONE && pQueryAttr->interval.interval > 0) { - op = OP_Fill; - taosArrayPush(plan, &op); - } - - // limit/offset operator - if (pQueryAttr->limit.limit > 0 || pQueryAttr->limit.offset > 0 || - pQueryAttr->slimit.limit > 0 || pQueryAttr->slimit.offset > 0) { - op = OP_SLimit; - taosArrayPush(plan, &op); - } - - return plan; -} diff --git a/2.0/src/query/src/qResultbuf.c b/2.0/src/query/src/qResultbuf.c deleted file mode 100644 index 3724362e2991bac86ca15d299b651f8b11668fdc..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qResultbuf.c +++ /dev/null @@ -1,453 +0,0 @@ -#include "qResultbuf.h" -#include "hash.h" -#include "qExtbuffer.h" -#include "queryLog.h" -#include "stddef.h" -#include "taoserror.h" -#include "tcompression.h" - -#define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES) -#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages) - -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId) { - *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf)); - - SDiskbasedResultBuf* pResBuf = *pResultBuf; - if (pResBuf == NULL) { - return TSDB_CODE_COM_OUT_OF_MEMORY; - } - - pResBuf->pageSize = pagesize; - pResBuf->numOfPages = 0; // all pages are in buffer in the first place - pResBuf->totalBufSize = 0; - pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit. - pResBuf->allocateId = -1; - pResBuf->comp = true; - pResBuf->file = NULL; - pResBuf->qId = qId; - pResBuf->fileSize = 0; - - // at least more than 2 pages must be in memory - assert(inMemBufSize >= pagesize * 2); - - pResBuf->lruList = tdListNew(POINTER_BYTES); - - // init id hash table - pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES - pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - - char path[PATH_MAX] = {0}; - taosGetTmpfilePath("qbuf", path); - pResBuf->path = strdup(path); - - pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); - - qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize, - pResBuf->inMemPages, pResBuf->path); - - return TSDB_CODE_SUCCESS; -} - -static int32_t createDiskFile(SDiskbasedResultBuf* pResultBuf) { - pResultBuf->file = fopen(pResultBuf->path, "wb+"); - if (pResultBuf->file == NULL) { - qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - return TSDB_CODE_SUCCESS; -} - -static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing - if (!pResultBuf->comp) { - *dst = srcSize; - return data; - } - - *dst = tsCompressString(data, srcSize, 1, pResultBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0); - - memcpy(data, pResultBuf->assistBuf, *dst); - return data; -} - -static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing - if (!pResultBuf->comp) { - *dst = srcSize; - return data; - } - - *dst = tsDecompressString(data, srcSize, 1, pResultBuf->assistBuf, pResultBuf->pageSize, ONE_STAGE_COMP, NULL, 0); - if (*dst > 0) { - memcpy(data, pResultBuf->assistBuf, *dst); - } - return data; -} - -static int32_t allocatePositionInFile(SDiskbasedResultBuf* pResultBuf, size_t size) { - if (pResultBuf->pFree == NULL) { - return pResultBuf->nextPos; - } else { - int32_t offset = -1; - - size_t num = taosArrayGetSize(pResultBuf->pFree); - for(int32_t i = 0; i < num; ++i) { - SFreeListItem* pi = taosArrayGet(pResultBuf->pFree, i); - if (pi->len >= size) { - offset = pi->offset; - pi->offset += (int32_t)size; - pi->len -= (int32_t)size; - - return offset; - } - } - - // no available recycle space, allocate new area in file - return pResultBuf->nextPos; - } -} - -static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { - assert(!pg->used && pg->pData != NULL); - - int32_t size = -1; - char* t = doCompressData(GET_DATA_PAYLOAD(pg), pResultBuf->pageSize, &size, pResultBuf); - - // this page is flushed to disk for the first time - if (pg->info.offset == -1) { - pg->info.offset = allocatePositionInFile(pResultBuf, size); - pResultBuf->nextPos += size; - - int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - assert(ret == 0); - - ret = (int32_t) fwrite(t, 1, size, pResultBuf->file); - assert(ret == size); - - if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { - pResultBuf->fileSize = pg->info.offset + pg->info.length; - } - } else { - // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing - if (pg->info.length < size) { - // 1. add current space to free list - taosArrayPush(pResultBuf->pFree, &pg->info); - - // 2. allocate new position, and update the info - pg->info.offset = allocatePositionInFile(pResultBuf, size); - pResultBuf->nextPos += size; - } - - //3. write to disk. - int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - if (ret != 0) { // todo handle the error case - - } - - ret = (int32_t)fwrite(t, size, 1, pResultBuf->file); - if (ret != size) { // todo handle the error case - - } - - if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { - pResultBuf->fileSize = pg->info.offset + pg->info.length; - } - } - - char* ret = pg->pData; - memset(ret, 0, pResultBuf->pageSize); - - pg->pData = NULL; - pg->info.length = size; - - pResultBuf->statis.flushBytes += pg->info.length; - - return ret; -} - -static char* flushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { - int32_t ret = TSDB_CODE_SUCCESS; - assert(((int64_t) pResultBuf->numOfPages * pResultBuf->pageSize) == pResultBuf->totalBufSize && pResultBuf->numOfPages >= pResultBuf->inMemPages); - - if (pResultBuf->file == NULL) { - if ((ret = createDiskFile(pResultBuf)) != TSDB_CODE_SUCCESS) { - terrno = ret; - return NULL; - } - } - - return doFlushPageToDisk(pResultBuf, pg); -} - -// load file block data in disk -static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { - int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - ret = (int32_t)fread(GET_DATA_PAYLOAD(pg), 1, pg->info.length, pResultBuf->file); - if (ret != pg->info.length) { - terrno = errno; - return NULL; - } - - pResultBuf->statis.loadBytes += pg->info.length; - - int32_t fullSize = 0; - doDecompressData(GET_DATA_PAYLOAD(pg), pg->info.length, &fullSize, pResultBuf); - - return (char*)GET_DATA_PAYLOAD(pg); -} - -static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL); - - SArray* pa = taosArrayInit(1, POINTER_BYTES); - int32_t ret = taosHashPut(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES); - assert(ret == 0); - - return pa; -} - -static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { - SIDList list = NULL; - - char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t)); - if (p == NULL) { // it is a new group id - list = addNewGroup(pResultBuf, groupId); - } else { - list = (SIDList) (*p); - } - - pResultBuf->numOfPages += 1; - - SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL}; - - ppi->pageId = pageId; - ppi->pData = NULL; - ppi->info = PAGE_INFO_INITIALIZER; - ppi->used = true; - ppi->pn = NULL; - - return *(SPageInfo**) taosArrayPush(list, &ppi); -} - -static SListNode* getEldestUnrefedPage(SDiskbasedResultBuf* pResultBuf) { - SListIter iter = {0}; - tdListInitIter(pResultBuf->lruList, &iter, TD_LIST_BACKWARD); - - SListNode* pn = NULL; - while((pn = tdListNext(&iter)) != NULL) { - assert(pn != NULL); - - SPageInfo* pageInfo = *(SPageInfo**) pn->data; - assert(pageInfo->pageId >= 0 && pageInfo->pn == pn); - - if (!pageInfo->used) { - break; - } - } - - return pn; -} - -static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) { - char* bufPage = NULL; - SListNode* pn = getEldestUnrefedPage(pResultBuf); - - // all pages are referenced by user, try to allocate new space - if (pn == NULL) { - int32_t prev = pResultBuf->inMemPages; - - // increase by 50% of previous mem pages - pResultBuf->inMemPages = (int32_t)(pResultBuf->inMemPages * 1.5f); - - qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev, - pResultBuf->inMemPages, pResultBuf->pageSize); - } else { - pResultBuf->statis.flushPages += 1; - tdListPopNode(pResultBuf->lruList, pn); - - SPageInfo* d = *(SPageInfo**) pn->data; - assert(d->pn == pn); - - d->pn = NULL; - tfree(pn); - - bufPage = flushPageToDisk(pResultBuf, d); - } - - return bufPage; -} - -static void lruListPushFront(SList *pList, SPageInfo* pi) { - tdListPrepend(pList, &pi); - SListNode* front = tdListGetHead(pList); - pi->pn = front; -} - -static void lruListMoveToFront(SList *pList, SPageInfo* pi) { - tdListPopNode(pList, pi->pn); - tdListPrependNode(pList, pi->pn); -} - -static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { - return pageSize + POINTER_BYTES + 2 + sizeof(tFilePage); -} - -tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { - pResultBuf->statis.getPages += 1; - - char* availablePage = NULL; - if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) { - availablePage = evicOneDataPage(pResultBuf); - } - - // register new id in this group - *pageId = (++pResultBuf->allocateId); - - // register page id info - SPageInfo* pi = registerPage(pResultBuf, groupId, *pageId); - - // add to LRU list - assert(listNEles(pResultBuf->lruList) < pResultBuf->inMemPages && pResultBuf->inMemPages > 0); - - lruListPushFront(pResultBuf->lruList, pi); - - // add to hash map - taosHashPut(pResultBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES); - - // allocate buf - if (availablePage == NULL) { - pi->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize)); // add extract bytes in case of zipped buffer increased. - } else { - pi->pData = availablePage; - } - - pResultBuf->totalBufSize += pResultBuf->pageSize; - - ((void**)pi->pData)[0] = pi; - pi->used = true; - - return (void *)(GET_DATA_PAYLOAD(pi)); -} - -tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { - assert(pResultBuf != NULL && id >= 0); - pResultBuf->statis.getPages += 1; - - SPageInfo** pi = taosHashGet(pResultBuf->all, &id, sizeof(int32_t)); - assert(pi != NULL && *pi != NULL); - - if ((*pi)->pData != NULL) { // it is in memory - // no need to update the LRU list if only one page exists - if (pResultBuf->numOfPages == 1) { - (*pi)->used = true; - return (void *)(GET_DATA_PAYLOAD(*pi)); - } - - SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data); - assert(*pInfo == *pi); - - lruListMoveToFront(pResultBuf->lruList, (*pi)); - (*pi)->used = true; - - return (void *)(GET_DATA_PAYLOAD(*pi)); - - } else { // not in memory - assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->info.length >= 0 && (*pi)->info.offset >= 0); - - char* availablePage = NULL; - if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) { - availablePage = evicOneDataPage(pResultBuf); - } - - if (availablePage == NULL) { - (*pi)->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize)); - } else { - (*pi)->pData = availablePage; - } - - ((void**)((*pi)->pData))[0] = (*pi); - - lruListPushFront(pResultBuf->lruList, *pi); - (*pi)->used = true; - - loadPageFromDisk(pResultBuf, *pi); - return (void *)(GET_DATA_PAYLOAD(*pi)); - } -} - -void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page) { - assert(pResultBuf != NULL && page != NULL); - char* p = (char*) page - POINTER_BYTES; - - SPageInfo* ppi = ((SPageInfo**) p)[0]; - releaseResBufPageInfo(pResultBuf, ppi); -} - -void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) { - assert(pi->pData != NULL && pi->used); - - pi->used = false; - pResultBuf->statis.releasePages += 1; -} - -size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); } - -size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return (size_t)pResultBuf->totalBufSize; } - -SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - assert(pResultBuf != NULL); - - char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t)); - if (p == NULL) { // it is a new group id - return pResultBuf->emptyDummyIdList; - } else { - return (SArray*) (*p); - } -} - -void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { - if (pResultBuf == NULL) { - return; - } - - if (pResultBuf->file != NULL) { - qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f Kb", - pResultBuf->qId, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0, - pResultBuf->fileSize/1024.0); - - fclose(pResultBuf->file); - } else { - qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, no file created", pResultBuf->qId, - pResultBuf->totalBufSize/1024.0); - } - - unlink(pResultBuf->path); - tfree(pResultBuf->path); - - SArray** p = taosHashIterate(pResultBuf->groupSet, NULL); - while(p) { - size_t n = taosArrayGetSize(*p); - for(int32_t i = 0; i < n; ++i) { - SPageInfo* pi = taosArrayGetP(*p, i); - tfree(pi->pData); - tfree(pi); - } - - taosArrayDestroy(*p); - p = taosHashIterate(pResultBuf->groupSet, p); - } - - tdListFree(pResultBuf->lruList); - taosArrayDestroy(pResultBuf->emptyDummyIdList); - taosHashCleanup(pResultBuf->groupSet); - taosHashCleanup(pResultBuf->all); - - tfree(pResultBuf->assistBuf); - tfree(pResultBuf); -} - -SPageInfo* getLastPageInfo(SIDList pList) { - size_t size = taosArrayGetSize(pList); - return (SPageInfo*) taosArrayGetP(pList, size - 1); -} - diff --git a/2.0/src/query/src/qSqlParser.c b/2.0/src/query/src/qSqlParser.c deleted file mode 100644 index b4ad30b291957b00d6337d7b16c592d5af1cb0ff..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qSqlParser.c +++ /dev/null @@ -1,1209 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "qSqlparser.h" -#include "os.h" -#include "taosdef.h" -#include "tmsg.h" -#include "tmsgtype.h" -#include "tstrbuild.h" -#include "ttoken.h" -#include "ttokendef.h" -#include "tutil.h" - -SSqlInfo qSqlParse(const char *pStr) { - void *pParser = ParseAlloc(malloc); - - SSqlInfo sqlInfo = {0}; - sqlInfo.valid = true; - sqlInfo.funcs = taosArrayInit(4, sizeof(SStrToken)); - - int32_t i = 0; - while (1) { - SStrToken t0 = {0}; - - if (pStr[i] == 0) { - Parse(pParser, 0, t0, &sqlInfo); - goto abort_parse; - } - - t0.n = tGetToken((char *)&pStr[i], &t0.type); - t0.z = (char *)(pStr + i); - i += t0.n; - - switch (t0.type) { - case TK_SPACE: - case TK_COMMENT: { - break; - } - case TK_SEMI: { - Parse(pParser, 0, t0, &sqlInfo); - goto abort_parse; - } - - case TK_QUESTION: - case TK_ILLEGAL: { - snprintf(sqlInfo.msg, tListLen(sqlInfo.msg), "unrecognized token: \"%s\"", t0.z); - sqlInfo.valid = false; - goto abort_parse; - } - - case TK_HEX: - case TK_OCT: - case TK_BIN:{ - snprintf(sqlInfo.msg, tListLen(sqlInfo.msg), "unsupported token: \"%s\"", t0.z); - sqlInfo.valid = false; - goto abort_parse; - } - - default: - Parse(pParser, t0.type, t0, &sqlInfo); - if (sqlInfo.valid == false) { - goto abort_parse; - } - } - } - -abort_parse: - ParseFree(pParser, free); - return sqlInfo; -} - -SArray *tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SStrToken *pDistinct, SStrToken *pToken) { - if (pList == NULL) { - pList = taosArrayInit(4, sizeof(tSqlExprItem)); - } - - if (pNode || pToken) { - struct tSqlExprItem item = {0}; - - item.pNode = pNode; - item.distinct = (pDistinct != NULL); - - if (pToken) { // set the as clause - item.aliasName = malloc(pToken->n + 1); - strncpy(item.aliasName, pToken->z, pToken->n); - item.aliasName[pToken->n] = 0; - - strdequote(item.aliasName); - } - - taosArrayPush(pList, &item); - } - - return pList; -} - -static void freeExprElem(void* item) { - tSqlExprItem* exprItem = item; - - tfree(exprItem->aliasName); - tSqlExprDestroy(exprItem->pNode); -} - -void tSqlExprListDestroy(SArray *pList) { - if (pList == NULL) { - return; - } - - taosArrayDestroyEx(pList, freeExprElem); -} - - -SArray *tStrTokenAppend(SArray *pList, SStrToken *pToken) { - if (pList == NULL) { - pList = taosArrayInit(4, sizeof(tVariantListItem)); - } - - if (pToken) { - taosArrayPush(pList, pToken); - } - - return pList; -} - -tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType) { - tSqlExpr *pSqlExpr = calloc(1, sizeof(tSqlExpr)); - - if (pToken != NULL) { - pSqlExpr->exprToken = *pToken; - } - - if (optrType == TK_NULL) { - if (pToken){ - pToken->type = TSDB_DATA_TYPE_NULL; - tVariantCreate(&pSqlExpr->value, pToken); - } - pSqlExpr->tokenId = optrType; - pSqlExpr->type = SQL_NODE_VALUE; - } else if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { - if (pToken) { - toTSDBType(pToken->type); - tVariantCreate(&pSqlExpr->value, pToken); - } - pSqlExpr->tokenId = optrType; - pSqlExpr->type = SQL_NODE_VALUE; - } else if (optrType == TK_NOW) { - // use nanosecond by default TODO set value after getting database precision - pSqlExpr->value.i64 = taosGetTimestamp(TSDB_TIME_PRECISION_NANO); - pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT; - pSqlExpr->tokenId = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond - pSqlExpr->type = SQL_NODE_VALUE; - pSqlExpr->flags |= 1 << EXPR_FLAG_NS_TIMESTAMP; - } else if (optrType == TK_VARIABLE) { - // use nanosecond by default - // TODO set value after getting database precision - if (pToken) { - char unit = 0; - int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->value.i64, &unit, TSDB_TIME_PRECISION_NANO); - if (ret != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - } - - pSqlExpr->flags |= 1 << EXPR_FLAG_NS_TIMESTAMP; - pSqlExpr->flags |= 1 << EXPR_FLAG_TIMESTAMP_VAR; - pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT; - pSqlExpr->tokenId = TK_TIMESTAMP; - pSqlExpr->type = SQL_NODE_VALUE; - } else { - // Here it must be the column name (tk_id) if it is not a number or string. - assert(optrType == TK_ID || optrType == TK_ALL); - if (pToken != NULL) { - pSqlExpr->columnName = *pToken; - } - - pSqlExpr->tokenId = optrType; - pSqlExpr->type = SQL_NODE_TABLE_COLUMN; - } - - return pSqlExpr; -} - -/* - * pList is the parameters for function with id(optType) - * function name is denoted by pFunctionToken - */ -tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) { - if (pFuncToken == NULL) { - return NULL; - } - - tSqlExpr *pExpr = calloc(1, sizeof(tSqlExpr)); - pExpr->tokenId = optType; - pExpr->type = SQL_NODE_SQLFUNCTION; - pExpr->Expr.paramList = pParam; - - int32_t len = (int32_t)((endToken->z + endToken->n) - pFuncToken->z); - pExpr->Expr.operand = (*pFuncToken); - - pExpr->exprToken.n = len; - pExpr->exprToken.z = pFuncToken->z; - pExpr->exprToken.type = pFuncToken->type; - - return pExpr; -} - -/* - * create binary expression in this procedure - * if the expr is arithmetic, calculate the result and set it to tSqlExpr Object - */ -tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) { - tSqlExpr *pExpr = calloc(1, sizeof(tSqlExpr)); - - pExpr->type = SQL_NODE_EXPR; - if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) { - char* endPos = pRight->exprToken.z + pRight->exprToken.n; - pExpr->exprToken.z = pLeft->exprToken.z; - pExpr->exprToken.n = (uint32_t)(endPos - pExpr->exprToken.z); - pExpr->exprToken.type = pLeft->exprToken.type; - } - - if ((pLeft != NULL && pRight != NULL) && - (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM)) { - /* - * if a exprToken is noted as the TK_TIMESTAMP, the time precision is microsecond - * Otherwise, the time precision is adaptive, determined by the time precision from databases. - */ - if ((pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_INTEGER) || - (pLeft->tokenId == TK_TIMESTAMP && pRight->tokenId == TK_TIMESTAMP)) { - pExpr->value.nType = TSDB_DATA_TYPE_BIGINT; - pExpr->tokenId = pLeft->tokenId; - pExpr->type = SQL_NODE_VALUE; - pExpr->flags = pLeft->flags | pRight->flags; - - if ((pLeft->flags & (1 << EXPR_FLAG_TIMESTAMP_VAR)) && (pRight->flags & (1 << EXPR_FLAG_TIMESTAMP_VAR))) { - pExpr->flags |= 1 << EXPR_FLAG_TS_ERROR; - } else { - pExpr->flags &= ~(1 << EXPR_FLAG_TIMESTAMP_VAR); - pExpr->flags &= ~(1 << EXPR_FLAG_TS_ERROR); - } - - switch (optrType) { - case TK_PLUS: { - pExpr->value.i64 = pLeft->value.i64 + pRight->value.i64; - break; - } - case TK_MINUS: { - pExpr->value.i64 = pLeft->value.i64 - pRight->value.i64; - break; - } - case TK_STAR: { - pExpr->value.i64 = pLeft->value.i64 * pRight->value.i64; - break; - } - case TK_DIVIDE: { - pExpr->tokenId = TK_FLOAT; - pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE; - pExpr->value.dKey = (double)pLeft->value.i64 / pRight->value.i64; - break; - } - case TK_REM: { - pExpr->value.i64 = pLeft->value.i64 % pRight->value.i64; - break; - } - } - - tSqlExprDestroy(pLeft); - tSqlExprDestroy(pRight); - } else if ((pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_INTEGER) || - (pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_FLOAT) || - (pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_FLOAT)) { - pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE; - pExpr->tokenId = TK_FLOAT; - pExpr->type = SQL_NODE_VALUE; - - double left = (pLeft->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->value.dKey : pLeft->value.i64; - double right = (pRight->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->value.dKey : pRight->value.i64; - - switch (optrType) { - case TK_PLUS: { - pExpr->value.dKey = left + right; - break; - } - case TK_MINUS: { - pExpr->value.dKey = left - right; - break; - } - case TK_STAR: { - pExpr->value.dKey = left * right; - break; - } - case TK_DIVIDE: { - pExpr->value.dKey = left / right; - break; - } - case TK_REM: { - pExpr->value.dKey = left - ((int64_t)(left / right)) * right; - break; - } - } - - tSqlExprDestroy(pLeft); - tSqlExprDestroy(pRight); - - } else { - pExpr->tokenId = optrType; - pExpr->pLeft = pLeft; - pExpr->pRight = pRight; - } - } else if (optrType == TK_IN) { - pExpr->tokenId = optrType; - pExpr->pLeft = pLeft; - - tSqlExpr *pRSub = calloc(1, sizeof(tSqlExpr)); - pRSub->tokenId = TK_SET; // TODO refactor ..... - pRSub->Expr.paramList = (SArray *)pRight; - - pExpr->pRight = pRSub; - } else { - pExpr->tokenId = optrType; - pExpr->pLeft = pLeft; - - if (pLeft != NULL && pRight == NULL) { - pRight = calloc(1, sizeof(tSqlExpr)); - } - - pExpr->pRight = pRight; - } - - return pExpr; -} - -static FORCE_INLINE int32_t tStrTokenCompare(SStrToken* left, SStrToken* right) { - return (left->type == right->type && left->n == right->n && strncasecmp(left->z, right->z, left->n) == 0) ? 0 : 1; -} - -// this function is not used for temporary -int32_t tSqlExprCompare(tSqlExpr *left, tSqlExpr *right) { - if ((left == NULL && right) || (left && right == NULL) || (left == NULL && right == NULL)) { - return 1; - } - - if (left->type != right->type) { - return 1; - } - - if (left->tokenId != right->tokenId) { - return 1; - } - - if (left->functionId != right->functionId) { - return 1; - } - - if ((left->pLeft && right->pLeft == NULL) - || (left->pLeft == NULL && right->pLeft) - || (left->pRight && right->pRight == NULL) - || (left->pRight == NULL && right->pRight) - || (left->Expr.paramList && right->Expr.paramList == NULL) - || (left->Expr.paramList == NULL && right->Expr.paramList)) { - return 1; - } - - if (tVariantCompare(&left->value, &right->value)) { - return 1; - } - - if (tStrTokenCompare(&left->columnName, &right->columnName)) { - return 1; - } - - if (right->Expr.paramList && left->Expr.paramList) { - size_t size = taosArrayGetSize(right->Expr.paramList); - if (left->Expr.paramList && taosArrayGetSize(left->Expr.paramList) != size) { - return 1; - } - - for (int32_t i = 0; i < size; i++) { - tSqlExprItem* pLeftElem = taosArrayGet(left->Expr.paramList, i); - tSqlExpr* pSubLeft = pLeftElem->pNode; - tSqlExprItem* pRightElem = taosArrayGet(right->Expr.paramList, i); - tSqlExpr* pSubRight = pRightElem->pNode; - - if (tSqlExprCompare(pSubLeft, pSubRight)) { - return 1; - } - } - } - - if (left->pLeft && tSqlExprCompare(left->pLeft, right->pLeft)) { - return 1; - } - - if (left->pRight && tSqlExprCompare(left->pRight, right->pRight)) { - return 1; - } - - return 0; -} - -tSqlExpr *tSqlExprClone(tSqlExpr *pSrc) { - tSqlExpr *pExpr = calloc(1, sizeof(tSqlExpr)); - - memcpy(pExpr, pSrc, sizeof(*pSrc)); - - if (pSrc->pLeft) { - pExpr->pLeft = tSqlExprClone(pSrc->pLeft); - } - - if (pSrc->pRight) { - pExpr->pRight = tSqlExprClone(pSrc->pRight); - } - - memset(&pExpr->value, 0, sizeof(pExpr->value)); - tVariantAssign(&pExpr->value, &pSrc->value); - - //we don't clone paramList now because clone is only used for between/and - assert(pSrc->Expr.paramList == NULL); - return pExpr; -} - -void tSqlExprCompact(tSqlExpr** pExpr) { - if (*pExpr == NULL || tSqlExprIsParentOfLeaf(*pExpr)) { - return; - } - - if ((*pExpr)->pLeft) { - tSqlExprCompact(&(*pExpr)->pLeft); - } - - if ((*pExpr)->pRight) { - tSqlExprCompact(&(*pExpr)->pRight); - } - - if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight == NULL && ((*pExpr)->tokenId == TK_OR || (*pExpr)->tokenId == TK_AND)) { - tSqlExprDestroy(*pExpr); - *pExpr = NULL; - } else if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight != NULL) { - tSqlExpr* tmpPtr = (*pExpr)->pRight; - (*pExpr)->pRight = NULL; - - tSqlExprDestroy(*pExpr); - (*pExpr) = tmpPtr; - } else if ((*pExpr)->pRight == NULL && (*pExpr)->pLeft != NULL) { - tSqlExpr* tmpPtr = (*pExpr)->pLeft; - (*pExpr)->pLeft = NULL; - - tSqlExprDestroy(*pExpr); - (*pExpr) = tmpPtr; - } -} - -bool tSqlExprIsLeaf(tSqlExpr* pExpr) { - return (pExpr->pRight == NULL && pExpr->pLeft == NULL) && - (pExpr->tokenId == 0 || - (pExpr->tokenId == TK_ID) || - (pExpr->tokenId >= TK_BOOL && pExpr->tokenId <= TK_NCHAR) || - (pExpr->tokenId == TK_NULL) || - (pExpr->tokenId == TK_SET)); -} - -bool tSqlExprIsParentOfLeaf(tSqlExpr* pExpr) { - return (pExpr->pLeft != NULL && pExpr->pRight != NULL) && - (tSqlExprIsLeaf(pExpr->pLeft) && tSqlExprIsLeaf(pExpr->pRight)); -} - -static void doDestroySqlExprNode(tSqlExpr *pExpr) { - if (pExpr == NULL) { - return; - } - - tVariantDestroy(&pExpr->value); - - tSqlExprListDestroy(pExpr->Expr.paramList); - free(pExpr); -} - -void tSqlExprDestroy(tSqlExpr *pExpr) { - if (pExpr == NULL) { - return; - } - - tSqlExprDestroy(pExpr->pLeft); - pExpr->pLeft = NULL; - tSqlExprDestroy(pExpr->pRight); - pExpr->pRight = NULL; - - doDestroySqlExprNode(pExpr); -} - -SArray *tVariantListAppendToken(SArray *pList, SStrToken *pToken, uint8_t order) { - if (pList == NULL) { - pList = taosArrayInit(4, sizeof(tVariantListItem)); - } - - if (pToken) { - tVariantListItem item; - tVariantCreate(&item.pVar, pToken); - item.sortOrder = order; - - taosArrayPush(pList, &item); - } - - return pList; -} - -SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder) { - if (pList == NULL) { - pList = taosArrayInit(4, sizeof(tVariantListItem)); - } - - if (pVar == NULL) { - return pList; - } - - /* - * Here we do not employ the assign function, since we need the pz attribute of structure - * , which is the point to char string, to free it! - * - * Otherwise, the original pointer may be lost, which causes memory leak. - */ - tVariantListItem item; - item.pVar = *pVar; - item.sortOrder = sortOrder; - - taosArrayPush(pList, &item); - return pList; -} - -SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index) { - if (pList == NULL || pVar == NULL || index >= taosArrayGetSize(pList)) { - return tVariantListAppend(NULL, pVar, sortOrder); - } - - tVariantListItem item; - - item.pVar = *pVar; - item.sortOrder = sortOrder; - - taosArrayInsert(pList, index, &item); - return pList; -} - -SRelationInfo *setTableNameList(SRelationInfo* pRelationInfo, SStrToken *pName, SStrToken* pAlias) { - if (pRelationInfo == NULL) { - pRelationInfo = calloc(1, sizeof(SRelationInfo)); - pRelationInfo->list = taosArrayInit(4, sizeof(SRelElement)); - } - - pRelationInfo->type = SQL_FROM_NODE_TABLES; - SRelElement p = {.tableName = *pName}; - if (pAlias != NULL) { - p.aliasName = *pAlias; - } else { - TPARSER_SET_NONE_TOKEN(p.aliasName); - } - - taosArrayPush(pRelationInfo->list, &p); - return pRelationInfo; -} - -void* destroyRelationInfo(SRelationInfo* pRelationInfo) { - if (pRelationInfo == NULL) { - return NULL; - } - - if (pRelationInfo->type == SQL_FROM_NODE_TABLES) { - taosArrayDestroy(pRelationInfo->list); - } else { - size_t size = taosArrayGetSize(pRelationInfo->list); - for(int32_t i = 0; i < size; ++i) { - SArray* pa = taosArrayGetP(pRelationInfo->list, i); - destroyAllSqlNode(pa); - } - taosArrayDestroy(pRelationInfo->list); - } - - tfree(pRelationInfo); - return NULL; -} - -SRelationInfo* addSubqueryElem(SRelationInfo* pRelationInfo, SArray* pSub, SStrToken* pAlias) { - if (pRelationInfo == NULL) { - pRelationInfo = calloc(1, sizeof(SRelationInfo)); - pRelationInfo->list = taosArrayInit(4, sizeof(SRelElement)); - } - - pRelationInfo->type = SQL_FROM_NODE_SUBQUERY; - - SRelElement p = {.pSubquery = pSub}; - if (pAlias != NULL) { - p.aliasName = *pAlias; - } else { - TPARSER_SET_NONE_TOKEN(p.aliasName); - } - - taosArrayPush(pRelationInfo->list, &p); - return pRelationInfo; -} - -void tSetDbName(SStrToken *pCpxName, SStrToken *pDb) { - pCpxName->type = pDb->type; - pCpxName->z = pDb->z; - pCpxName->n = pDb->n; -} - -void tSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { - int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); - - // column name is too long, set the it to be invalid. - if ((int32_t) pName->n >= maxLen) { - pName->n = -1; - } else { - strncpy(pField->name, pName->z, pName->n); - pField->name[pName->n] = 0; - } - - pField->type = pType->type; - if(!isValidDataType(pField->type)){ - pField->bytes = 0; - } else { - pField->bytes = pType->bytes; - } -} - -static int32_t tryParseNameTwoParts(SStrToken *type) { - int32_t t = -1; - - char* str = strndup(type->z, type->n); - if (str == NULL) { - return t; - } - - char* p = strtok(str, " "); - if (p == NULL) { - tfree(str); - return t; - } else { - char* unsign = strtok(NULL, " "); - if (unsign == NULL) { - tfree(str); - return t; - } - - if (strncasecmp(unsign, "UNSIGNED", 8) == 0) { - for(int32_t j = TSDB_DATA_TYPE_TINYINT; j <= TSDB_DATA_TYPE_BIGINT; ++j) { - if (strcasecmp(p, tDataTypes[j].name) == 0) { - t = j; - break; - } - } - - tfree(str); - - if (t == -1) { - return -1; - } - - switch(t) { - case TSDB_DATA_TYPE_TINYINT: return TSDB_DATA_TYPE_UTINYINT; - case TSDB_DATA_TYPE_SMALLINT: return TSDB_DATA_TYPE_USMALLINT; - case TSDB_DATA_TYPE_INT: return TSDB_DATA_TYPE_UINT; - case TSDB_DATA_TYPE_BIGINT: return TSDB_DATA_TYPE_UBIGINT; - default: - return -1; - } - - } else { - tfree(str); - return -1; - } - } -} - -void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) { - // set the field type invalid - pField->type = -1; - pField->name[0] = 0; - - int32_t i = 0; - while (i < tListLen(tDataTypes)) { - if ((type->n == tDataTypes[i].nameLen) && - (strncasecmp(type->z, tDataTypes[i].name, tDataTypes[i].nameLen) == 0)) { - break; - } - - i += 1; - } - - // no qualified data type found, try unsigned data type - if (i == tListLen(tDataTypes)) { - i = tryParseNameTwoParts(type); - if (i == -1) { - return; - } - } - - pField->type = i; - pField->bytes = tDataTypes[i].bytes; - - if (i == TSDB_DATA_TYPE_NCHAR) { - /* - * for nchar, the TOKENTYPE is the number of character, so the length is the - * number of bytes in UCS-4 format, which is 4 times larger than the number of characters - */ - if (type->type == 0) { - pField->bytes = 0; - } else { - int32_t bytes = -(int32_t)(type->type); - if (bytes > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - // overflowed. set bytes to -1 so that error can be reported - bytes = -1; - } else { - bytes = bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - } - pField->bytes = (int16_t)bytes; - } - } else if (i == TSDB_DATA_TYPE_BINARY) { - /* for binary, the TOKENTYPE is the length of binary */ - if (type->type == 0) { - pField->bytes = 0; - } else { - int32_t bytes = -(int32_t)(type->type); - if (bytes > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - // overflowed. set bytes to -1 so that error can be reported - bytes = -1; - } else { - bytes += VARSTR_HEADER_SIZE; - } - - pField->bytes = (int16_t)bytes; - } - } -} - -/* - * extract the select info out of sql string - */ -SSqlNode *tSetQuerySqlNode(SStrToken *pSelectToken, SArray *pSelNodeList, SRelationInfo *pFrom, tSqlExpr *pWhere, - SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, - SSessionWindowVal *pSession, SWindowStateVal *pWindowStateVal, SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, - SLimitVal *psLimit, tSqlExpr *pHaving) { - assert(pSelNodeList != NULL); - - SSqlNode *pSqlNode = calloc(1, sizeof(SSqlNode)); - - // all later sql string are belonged to the stream sql - pSqlNode->sqlstr = *pSelectToken; - pSqlNode->sqlstr.n = (uint32_t)strlen(pSqlNode->sqlstr.z); - - pSqlNode->pSelNodeList = pSelNodeList; - pSqlNode->from = pFrom; - pSqlNode->pGroupby = pGroupby; - pSqlNode->pSortOrder = pSortOrder; - pSqlNode->pWhere = pWhere; - pSqlNode->fillType = pFill; - pSqlNode->pHaving = pHaving; - - if (pLimit != NULL) { - pSqlNode->limit = *pLimit; - } else { - pSqlNode->limit.limit = -1; - pSqlNode->limit.offset = 0; - } - - if (psLimit != NULL) { - pSqlNode->slimit = *psLimit; - } else { - pSqlNode->slimit.limit = -1; - pSqlNode->slimit.offset = 0; - } - - if (pInterval != NULL) { - pSqlNode->interval = *pInterval; - } else { - TPARSER_SET_NONE_TOKEN(pSqlNode->interval.interval); - TPARSER_SET_NONE_TOKEN(pSqlNode->interval.offset); - } - - if (pSliding != NULL) { - pSqlNode->sliding = *pSliding; - } else { - TPARSER_SET_NONE_TOKEN(pSqlNode->sliding); - } - - if (pSession != NULL) { - pSqlNode->sessionVal = *pSession; - } else { - TPARSER_SET_NONE_TOKEN(pSqlNode->sessionVal.gap); - TPARSER_SET_NONE_TOKEN(pSqlNode->sessionVal.col); - } - - if (pWindowStateVal != NULL) { - pSqlNode->windowstateVal = *pWindowStateVal; - } else { - TPARSER_SET_NONE_TOKEN(pSqlNode->windowstateVal.col); - } - - return pSqlNode; -} - -static void freeVariant(void *pItem) { - tVariantListItem* p = (tVariantListItem*) pItem; - tVariantDestroy(&p->pVar); -} - -void freeCreateTableInfo(void* p) { - SCreatedTableInfo* pInfo = (SCreatedTableInfo*) p; - taosArrayDestroy(pInfo->pTagNames); - taosArrayDestroyEx(pInfo->pTagVals, freeVariant); - tfree(pInfo->fullname); - tfree(pInfo->tagdata.data); -} - -void destroySqlNode(SSqlNode *pSqlNode) { - if (pSqlNode == NULL) { - return; - } - - tSqlExprListDestroy(pSqlNode->pSelNodeList); - pSqlNode->pSelNodeList = NULL; - - tSqlExprDestroy(pSqlNode->pWhere); - pSqlNode->pWhere = NULL; - - taosArrayDestroyEx(pSqlNode->pSortOrder, freeVariant); - pSqlNode->pSortOrder = NULL; - - taosArrayDestroyEx(pSqlNode->pGroupby, freeVariant); - pSqlNode->pGroupby = NULL; - - pSqlNode->from = destroyRelationInfo(pSqlNode->from); - - taosArrayDestroyEx(pSqlNode->fillType, freeVariant); - pSqlNode->fillType = NULL; - - tSqlExprDestroy(pSqlNode->pHaving); - free(pSqlNode); -} - -void destroyAllSqlNode(SArray *pList) { - if (pList == NULL) { - return; - } - - size_t size = taosArrayGetSize(pList); - for(int32_t i = 0; i < size; ++i) { - SSqlNode *pNode = taosArrayGetP(pList, i); - destroySqlNode(pNode); - } - - taosArrayDestroy(pList); -} - -SCreateTableSql *tSetCreateTableInfo(SArray *pCols, SArray *pTags, SSqlNode *pSelect, int32_t type) { - SCreateTableSql *pCreate = calloc(1, sizeof(SCreateTableSql)); - - switch (type) { - case TSQL_CREATE_TABLE: { - pCreate->colInfo.pColumns = pCols; - assert(pTags == NULL); - break; - } - case TSQL_CREATE_STABLE: { - pCreate->colInfo.pColumns = pCols; - pCreate->colInfo.pTagColumns = pTags; - assert(pTags != NULL && pCols != NULL); - break; - } - case TSQL_CREATE_STREAM: { - pCreate->pSelect = pSelect; - break; - } - - case TSQL_CREATE_TABLE_FROM_STABLE: { - assert(0); - } - - default: - assert(false); - } - - pCreate->type = type; - return pCreate; -} - -SCreatedTableInfo createNewChildTableInfo(SStrToken *pTableName, SArray *pTagNames, SArray *pTagVals, SStrToken *pToken, SStrToken* igExists) { - SCreatedTableInfo info; - memset(&info, 0, sizeof(SCreatedTableInfo)); - - info.name = *pToken; - info.pTagNames = pTagNames; - info.pTagVals = pTagVals; - info.stableName = *pTableName; - info.igExist = (igExists->n > 0)? 1:0; - - return info; -} - -SAlterTableInfo *tSetAlterTableInfo(SStrToken *pTableName, SArray *pCols, SArray *pVals, int32_t type, int16_t tableType) { - SAlterTableInfo *pAlterTable = calloc(1, sizeof(SAlterTableInfo)); - - pAlterTable->name = *pTableName; - pAlterTable->type = type; - pAlterTable->tableType = tableType; - - if (type == TSDB_ALTER_TABLE_ADD_COLUMN || type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN || type == TSDB_ALTER_TABLE_CHANGE_COLUMN || type == TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN) { - pAlterTable->pAddColumns = pCols; - assert(pVals == NULL); - } else { - /* - * ALTER_TABLE_TAGS_CHG, ALTER_TABLE_TAGS_SET, ALTER_TABLE_TAGS_DROP, - * ALTER_TABLE_DROP_COLUMN - */ - pAlterTable->varList = pVals; - assert(pCols == NULL); - } - - return pAlterTable; -} - -void* destroyCreateTableSql(SCreateTableSql* pCreate) { - destroySqlNode(pCreate->pSelect); - - taosArrayDestroy(pCreate->colInfo.pColumns); - taosArrayDestroy(pCreate->colInfo.pTagColumns); - - taosArrayDestroyEx(pCreate->childTableInfo, freeCreateTableInfo); - tfree(pCreate); - - return NULL; -} - -void SqlInfoDestroy(SSqlInfo *pInfo) { - if (pInfo == NULL) return;; - taosArrayDestroy(pInfo->funcs); - if (pInfo->type == TSDB_SQL_SELECT) { - destroyAllSqlNode(pInfo->list); - } else if (pInfo->type == TSDB_SQL_CREATE_TABLE) { - pInfo->pCreateTableInfo = destroyCreateTableSql(pInfo->pCreateTableInfo); - } else if (pInfo->type == TSDB_SQL_ALTER_TABLE) { - taosArrayDestroyEx(pInfo->pAlterInfo->varList, freeVariant); - taosArrayDestroy(pInfo->pAlterInfo->pAddColumns); - tfree(pInfo->pAlterInfo->tagData.data); - tfree(pInfo->pAlterInfo); - } else if (pInfo->type == TSDB_SQL_COMPACT_VNODE) { - tSqlExprListDestroy(pInfo->list); - } else { - if (pInfo->pMiscInfo != NULL) { - taosArrayDestroy(pInfo->pMiscInfo->a); - } - - if (pInfo->pMiscInfo != NULL && (pInfo->type == TSDB_SQL_CREATE_DB || pInfo->type == TSDB_SQL_ALTER_DB)) { - taosArrayDestroyEx(pInfo->pMiscInfo->dbOpt.keep, freeVariant); - } - - tfree(pInfo->pMiscInfo); - } -} - -SArray* setSubclause(SArray* pList, void *pSqlNode) { - if (pList == NULL) { - - pList = taosArrayInit(1, POINTER_BYTES); - } - - taosArrayPush(pList, &pSqlNode); - return pList; -} - -SSqlInfo* setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pTableName, int32_t type) { - pInfo->type = type; - - if (type == TSDB_SQL_SELECT) { - pInfo->list = (SArray*) pSqlExprInfo; - } else { - pInfo->pCreateTableInfo = pSqlExprInfo; - } - - if (pTableName != NULL) { - pInfo->pCreateTableInfo->name = *pTableName; - } - - return pInfo; -} - -SArray* appendSelectClause(SArray *pList, void *pSubclause) { - taosArrayPush(pList, &pSubclause); - return pList; -} - -void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pTableNameToken, SStrToken *pIfNotExists) { - pInfo->pCreateTableInfo->name = *pTableNameToken; - pInfo->pCreateTableInfo->existCheck = (pIfNotExists->n != 0); -} - -void setDCLSqlElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) { - pInfo->type = type; - if (nParam == 0) { - return; - } - - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = (SMiscInfo *)calloc(1, sizeof(SMiscInfo)); - pInfo->pMiscInfo->a = taosArrayInit(4, sizeof(SStrToken)); - } - - va_list va; - va_start(va, nParam); - - while ((nParam--) > 0) { - SStrToken *pToken = va_arg(va, SStrToken *); - taosArrayPush(pInfo->pMiscInfo->a, pToken); - } - - va_end(va); -} - -void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck, int16_t dbType, int16_t tableType) { - pInfo->type = type; - - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = (SMiscInfo *)calloc(1, sizeof(SMiscInfo)); - pInfo->pMiscInfo->a = taosArrayInit(4, sizeof(SStrToken)); - } - - taosArrayPush(pInfo->pMiscInfo->a, pToken); - - pInfo->pMiscInfo->existsCheck = (existsCheck->n == 1); - pInfo->pMiscInfo->dbType = dbType; - pInfo->pMiscInfo->tableType = tableType; -} - -void setDropFuncInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken) { - pInfo->type = type; - - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = (SMiscInfo *)calloc(1, sizeof(SMiscInfo)); - pInfo->pMiscInfo->a = taosArrayInit(4, sizeof(SStrToken)); - } - - taosArrayPush(pInfo->pMiscInfo->a, pToken); -} - - -void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns) { - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); - } - - pInfo->type = TSDB_SQL_SHOW; - - SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; - pShowInfo->showType = type; - - if (prefix != NULL && prefix->type != 0) { - pShowInfo->prefix = *prefix; - } else { - pShowInfo->prefix.type = 0; - } - - if (pPatterns != NULL && pPatterns->type != 0) { - pShowInfo->pattern = *pPatterns; - } else { - pShowInfo->pattern.type = 0; - } -} - -void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDbInfo *pDB, SStrToken *pIgExists) { - pInfo->type = type; - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); - } - - pInfo->pMiscInfo->dbOpt = *pDB; - pInfo->pMiscInfo->dbOpt.dbname = *pToken; - pInfo->pMiscInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} -} - -void setCreateFuncInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPath, TAOS_FIELD *output, SStrToken* bufSize, int32_t funcType) { - pInfo->type = type; - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); - } - - pInfo->pMiscInfo->funcOpt.name = *pName; - pInfo->pMiscInfo->funcOpt.path = *pPath; - pInfo->pMiscInfo->funcOpt.output = *output; - pInfo->pMiscInfo->funcOpt.type = funcType; - if (bufSize->n > 0) { - pInfo->pMiscInfo->funcOpt.bufSize = strtol(bufSize->z, NULL, 10); - } else { - pInfo->pMiscInfo->funcOpt.bufSize = 0; - } -} - - -void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctInfo *pAcctInfo) { - pInfo->type = type; - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); - } - - pInfo->pMiscInfo->acctOpt = *pAcctInfo; - - assert(pName != NULL); - pInfo->pMiscInfo->user.user = *pName; - - if (pPwd != NULL) { - pInfo->pMiscInfo->user.passwd = *pPwd; - } -} -void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam) { - pInfo->type = type; - pInfo->list = pParam; -} - -void setCreateUserSql(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd) { - pInfo->type = TSDB_SQL_CREATE_USER; - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); - } - - assert(pName != NULL && pPasswd != NULL); - - pInfo->pMiscInfo->user.user = *pName; - pInfo->pMiscInfo->user.passwd = *pPasswd; -} - -void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege) { - pInfo->type = TSDB_SQL_ALTER_USER; - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); - } - - assert(pName != NULL); - - SUserInfo* pUser = &pInfo->pMiscInfo->user; - pUser->type = type; - pUser->user = *pName; - - if (pPwd != NULL) { - pUser->passwd = *pPwd; - } else { - pUser->passwd.type = TSDB_DATA_TYPE_NULL; - } - - if (pPrivilege != NULL) { - pUser->privilege = *pPrivilege; - } else { - pUser->privilege.type = TSDB_DATA_TYPE_NULL; - } -} - -void setKillSql(SSqlInfo *pInfo, int32_t type, SStrToken *id) { - pInfo->type = type; - if (pInfo->pMiscInfo == NULL) { - pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); - } - - assert(id != NULL); - pInfo->pMiscInfo->id = *id; -} - -void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) { - pDBInfo->compressionLevel = -1; - - pDBInfo->walLevel = -1; - pDBInfo->fsyncPeriod = -1; - pDBInfo->commitTime = -1; - pDBInfo->maxTablesPerVnode = -1; - - pDBInfo->cacheBlockSize = -1; - pDBInfo->numOfBlocks = -1; - pDBInfo->maxRowsPerBlock = -1; - pDBInfo->minRowsPerBlock = -1; - pDBInfo->daysPerFile = -1; - - pDBInfo->replica = -1; - pDBInfo->quorum = -1; - pDBInfo->keep = NULL; - - pDBInfo->update = -1; - pDBInfo->cachelast = -1; - - pDBInfo->dbType = -1; - pDBInfo->partitions = -1; - - memset(&pDBInfo->precision, 0, sizeof(SStrToken)); -} - -void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo) { - setDefaultCreateDbOption(pDBInfo); - - pDBInfo->dbType = TSDB_DB_TYPE_TOPIC; - pDBInfo->partitions = TSDB_DEFAULT_DB_PARTITON_OPTION; -} diff --git a/2.0/src/query/src/qTableMeta.c b/2.0/src/query/src/qTableMeta.c deleted file mode 100644 index 4e8954ed6fca766b606cb6345949918f6d2d14af..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qTableMeta.c +++ /dev/null @@ -1,107 +0,0 @@ - -#include "os.h" -#include "tmsg.h" -#include "qTableMeta.h" -#include "ttokendef.h" -#include "taosdef.h" -#include "tutil.h" - -int32_t tscGetNumOfTags(const STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - - if (pTableMeta->tableType == TSDB_NORMAL_TABLE) { - assert(tinfo.numOfTags == 0); - return 0; - } - - if (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE) { - return tinfo.numOfTags; - } - - assert(tinfo.numOfTags == 0); - return 0; -} - -int32_t tscGetNumOfColumns(const STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - - // table created according to super table, use data from super table - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - return tinfo.numOfColumns; -} - -SSchema *tscGetTableSchema(const STableMeta *pTableMeta) { - assert(pTableMeta != NULL); - return (SSchema*) pTableMeta->schema; -} - -SSchema* tscGetTableTagSchema(const STableMeta* pTableMeta) { - assert(pTableMeta != NULL && (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE)); - - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - assert(tinfo.numOfTags > 0); - - return tscGetTableColumnSchema(pTableMeta, tinfo.numOfColumns); -} - -STableComInfo tscGetTableInfo(const STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - return pTableMeta->tableInfo; -} - -SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) { - assert(pTableMeta != NULL); - - SSchema* pSchema = (SSchema*) pTableMeta->schema; - return &pSchema[colIndex]; -} - -// TODO for large number of columns, employ the binary search method -SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) { - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - - for(int32_t i = 0; i < tinfo.numOfColumns + tinfo.numOfTags; ++i) { - if (pTableMeta->schema[i].colId == colId) { - return &pTableMeta->schema[i]; - } - } - - return NULL; -} - -STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { - assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2); - - int32_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema); - STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize); - - pTableMeta->tableType = pTableMetaMsg->tableType; - pTableMeta->vgId = pTableMetaMsg->vgroup.vgId; - pTableMeta->suid = pTableMetaMsg->suid; - - pTableMeta->tableInfo = (STableComInfo) { - .numOfTags = pTableMetaMsg->numOfTags, - .precision = pTableMetaMsg->precision, - .numOfColumns = pTableMetaMsg->numOfColumns, - }; - - pTableMeta->id.tid = pTableMetaMsg->tid; - pTableMeta->id.uid = pTableMetaMsg->uid; - - pTableMeta->sversion = pTableMetaMsg->sversion; - pTableMeta->tversion = pTableMetaMsg->tversion; - - tstrncpy(pTableMeta->sTableName, pTableMetaMsg->sTableName, TSDB_TABLE_FNAME_LEN); - - memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize); - - int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns; - for(int32_t i = 0; i < numOfTotalCols; ++i) { - pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; - } - - return pTableMeta; -} - diff --git a/2.0/src/query/src/qTsbuf.c b/2.0/src/query/src/qTsbuf.c deleted file mode 100644 index 92ca8412722cc17614019edcabe027f8598dca5d..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/qTsbuf.c +++ /dev/null @@ -1,1109 +0,0 @@ -#include "qTsbuf.h" -#include "queryLog.h" -#include "taoserror.h" -#include "tcompression.h" -#include "tutil.h" - -static int32_t getDataStartOffset(); -static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); -static STSBuf* allocResForTSBuf(STSBuf* pTSBuf); -static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); - -/** - * todo error handling - * support auto closeable tmp file - * @param path - * @return - */ -STSBuf* tsBufCreate(bool autoDelete, int32_t order) { - STSBuf* pTSBuf = calloc(1, sizeof(STSBuf)); - if (pTSBuf == NULL) { - return NULL; - } - - pTSBuf->autoDelete = autoDelete; - - taosGetTmpfilePath("join", pTSBuf->path); - pTSBuf->f = fopen(pTSBuf->path, "wb+"); - if (pTSBuf->f == NULL) { - free(pTSBuf); - return NULL; - } - - if (!autoDelete) { - unlink(pTSBuf->path); - } - - if (NULL == allocResForTSBuf(pTSBuf)) { - return NULL; - } - - // update the header info - STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = TSDB_ORDER_ASC}; - STSBufUpdateHeader(pTSBuf, &header); - - tsBufResetPos(pTSBuf); - pTSBuf->cur.order = TSDB_ORDER_ASC; - - pTSBuf->tsOrder = order; - - return pTSBuf; -} - -STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { - STSBuf* pTSBuf = calloc(1, sizeof(STSBuf)); - if (pTSBuf == NULL) { - return NULL; - } - - pTSBuf->autoDelete = autoDelete; - - tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path)); - - pTSBuf->f = fopen(pTSBuf->path, "rb+"); - if (pTSBuf->f == NULL) { - free(pTSBuf); - return NULL; - } - - if (allocResForTSBuf(pTSBuf) == NULL) { - return NULL; - } - - // validate the file magic number - STSBufFileHeader header = {0}; - int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET); - UNUSED(ret); - size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f); - UNUSED(sz); - - // invalid file - if (header.magic != TS_COMP_FILE_MAGIC) { - tsBufDestroy(pTSBuf); - return NULL; - } - - if (header.numOfGroup > pTSBuf->numOfAlloc) { - pTSBuf->numOfAlloc = header.numOfGroup; - STSGroupBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc); - if (tmp == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->pData = tmp; - } - - pTSBuf->numOfGroups = header.numOfGroup; - - // check the ts order - pTSBuf->tsOrder = header.tsOrder; - if (pTSBuf->tsOrder != TSDB_ORDER_ASC && pTSBuf->tsOrder != TSDB_ORDER_DESC) { -// tscError("invalid order info in buf:%d", pTSBuf->tsOrder); - tsBufDestroy(pTSBuf); - return NULL; - } - - size_t infoSize = sizeof(STSGroupBlockInfo) * pTSBuf->numOfGroups; - - STSGroupBlockInfo* buf = (STSGroupBlockInfo*)calloc(1, infoSize); - if (buf == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - //int64_t pos = ftell(pTSBuf->f); //pos not used - sz = fread(buf, infoSize, 1, pTSBuf->f); - UNUSED(sz); - - // the length value for each vnode is not kept in file, so does not set the length value - for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) { - STSGroupBlockInfoEx* pBlockList = &pTSBuf->pData[i]; - memcpy(&pBlockList->info, &buf[i], sizeof(STSGroupBlockInfo)); - } - free(buf); - - ret = fseek(pTSBuf->f, 0, SEEK_END); - UNUSED(ret); - - struct stat fileStat; - if (fstat(fileno(pTSBuf->f), &fileStat) != 0) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->fileSize = (uint32_t)fileStat.st_size; - tsBufResetPos(pTSBuf); - - // ascending by default - pTSBuf->cur.order = TSDB_ORDER_ASC; - -// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f), -// pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); - - return pTSBuf; -} - -void* tsBufDestroy(STSBuf* pTSBuf) { - if (pTSBuf == NULL) { - return NULL; - } - - tfree(pTSBuf->assistBuf); - tfree(pTSBuf->tsData.rawBuf); - - tfree(pTSBuf->pData); - tfree(pTSBuf->block.payload); - - if (!pTSBuf->remainOpen) { - fclose(pTSBuf->f); - } - - if (pTSBuf->autoDelete) { -// ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path); - unlink(pTSBuf->path); - } else { -// tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); - } - - tVariantDestroy(&pTSBuf->block.tag); - free(pTSBuf); - return NULL; -} - -static STSGroupBlockInfoEx* tsBufGetLastGroupInfo(STSBuf* pTSBuf) { - int32_t last = pTSBuf->numOfGroups - 1; - - assert(last >= 0); - return &pTSBuf->pData[last]; -} - -static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { - if (pTSBuf->numOfAlloc <= pTSBuf->numOfGroups) { - uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5); - assert((int32_t)newSize > pTSBuf->numOfAlloc); - - STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); - if (tmp == NULL) { - return NULL; - } - - pTSBuf->pData = tmp; - pTSBuf->numOfAlloc = newSize; - memset(&pTSBuf->pData[pTSBuf->numOfGroups], 0, sizeof(STSGroupBlockInfoEx) * (newSize - pTSBuf->numOfGroups)); - } - - if (pTSBuf->numOfGroups > 0) { - STSGroupBlockInfoEx* pPrevBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - - // update prev vnode length info in file - TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pPrevBlockInfoEx->info); - } - - // set initial value for vnode block - STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfGroups].info; - pBlockInfo->id = id; - pBlockInfo->offset = pTSBuf->fileSize; - assert(pBlockInfo->offset >= getDataStartOffset()); - - // update vnode info in file - TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups, pBlockInfo); - - // add one vnode info - pTSBuf->numOfGroups += 1; - - // update the header info - STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; - - STSBufUpdateHeader(pTSBuf, &header); - return tsBufGetLastGroupInfo(pTSBuf); -} - -static void shrinkBuffer(STSList* ptsData) { - // shrink tmp buffer size if it consumes too many memory compared to the pre-defined size - if (ptsData->allocSize >= ptsData->threshold * 2) { - char* rawBuf = realloc(ptsData->rawBuf, MEM_BUF_SIZE); - if(rawBuf) { - ptsData->rawBuf = rawBuf; - ptsData->allocSize = MEM_BUF_SIZE; - } - } -} - -static int32_t getTagAreaLength(tVariant* pa) { - int32_t t = sizeof(pa->nLen) * 2 + sizeof(pa->nType); - if (pa->nType != TSDB_DATA_TYPE_NULL) { - t += pa->nLen; - } - - return t; -} - -static void writeDataToDisk(STSBuf* pTSBuf) { - if (pTSBuf->tsData.len == 0) { - return; - } - - STSBlock* pBlock = &pTSBuf->block; - STSList* pTsData = &pTSBuf->tsData; - - pBlock->numOfElem = pTsData->len / TSDB_KEYSIZE; - pBlock->compLen = - tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize, - TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - - int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); - assert(r == 0); - - /* - * format for output data: - * 1. tags, number of ts, size after compressed, payload, size after compressed - * 2. tags, number of ts, size after compressed, payload, size after compressed - * - * both side has the compressed length is used to support load data forwards/backwords. - */ - int32_t metaLen = 0; - metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); - - int32_t trueLen = pBlock->tag.nLen; - if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); - } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - float tfloat = (float)pBlock->tag.dKey; - metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); - } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(&pBlock->tag.i64, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); - } else { - trueLen = 0; - metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - } - - fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - - metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - assert(metaLen == getTagAreaLength(&pBlock->tag)); - - int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; - pTSBuf->fileSize += blockSize; - - pTSBuf->tsData.len = 0; - - STSGroupBlockInfoEx* pGroupBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - - pGroupBlockInfoEx->info.compLen += blockSize; - pGroupBlockInfoEx->info.numOfBlocks += 1; - - shrinkBuffer(&pTSBuf->tsData); -} - -static void expandBuffer(STSList* ptsData, int32_t inputSize) { - if (ptsData->allocSize - ptsData->len < inputSize) { - int32_t newSize = inputSize + ptsData->len; - char* tmp = realloc(ptsData->rawBuf, (size_t)newSize); - if (tmp == NULL) { - // todo - } - - ptsData->rawBuf = tmp; - ptsData->allocSize = newSize; - } -} - -STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { - STSBlock* pBlock = &pTSBuf->block; - - // clear the memory buffer - pBlock->compLen = 0; - pBlock->padding = 0; - pBlock->numOfElem = 0; - - int32_t offset = -1; - - if (order == TSDB_ORDER_DESC) { - /* - * set the right position for the reversed traverse, the reversed traverse is started from - * the end of each comp data block - */ - int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); - int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR); - size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f); - sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - UNUSED(sz); - - pBlock->compLen = pBlock->padding; - - offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - ret = fseek(pTSBuf->f, -offset, SEEK_CUR); - UNUSED(ret); - } - - fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); - fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); - - // NOTE: mix types tags are not supported - size_t sz = 0; - if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - char* tp = realloc(pBlock->tag.pz, pBlock->tag.nLen + 1); - assert(tp != NULL); - - memset(tp, 0, pBlock->tag.nLen + 1); - pBlock->tag.pz = tp; - - sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); - UNUSED(sz); - } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { - float tfloat = 0; - sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); - pBlock->tag.dKey = (double)tfloat; - UNUSED(sz); - } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value - sz = fread(&pBlock->tag.i64, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); - UNUSED(sz); - } - - sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); - UNUSED(sz); - sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - UNUSED(sz); - sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - - if (decomp) { - pTSBuf->tsData.len = - tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, - pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - } - - // read the comp length at the length of comp block - sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); - assert(pBlock->padding == pBlock->compLen); - - int32_t n = 0; - sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); - if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) { - assert(n == 0); - } else { - assert(n == pBlock->tag.nLen); - } - - UNUSED(sz); - - // for backwards traverse, set the start position at the end of previous block - if (order == TSDB_ORDER_DESC) { - int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR); - UNUSED(r); - } - - return pBlock; -} - -// set the order of ts buffer if the ts order has not been set yet -static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { - STSList* ptsData = &pTSBuf->tsData; - - if (pTSBuf->tsOrder == -1) { - if (ptsData->len > 0) { - TSKEY lastKey = *(TSKEY*)(ptsData->rawBuf + ptsData->len - TSDB_KEYSIZE); - - if (lastKey > *(TSKEY*)pData) { - pTSBuf->tsOrder = TSDB_ORDER_DESC; - } else { - pTSBuf->tsOrder = TSDB_ORDER_ASC; - } - } else if (len > TSDB_KEYSIZE) { - // no data in current vnode, more than one ts is added, check the orders - TSKEY k1 = *(TSKEY*)(pData); - TSKEY k2 = *(TSKEY*)(pData + TSDB_KEYSIZE); - - if (k1 < k2) { - pTSBuf->tsOrder = TSDB_ORDER_ASC; - } else if (k1 > k2) { - pTSBuf->tsOrder = TSDB_ORDER_DESC; - } else { - // todo handle error - } - } - } else { - // todo the timestamp order is set, check the asc/desc order of appended data - } - - return TSDB_CODE_SUCCESS; -} - -void tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, int32_t len) { - STSGroupBlockInfoEx* pBlockInfo = NULL; - STSList* ptsData = &pTSBuf->tsData; - - if (pTSBuf->numOfGroups == 0 || tsBufGetLastGroupInfo(pTSBuf)->info.id != id) { - writeDataToDisk(pTSBuf); - shrinkBuffer(ptsData); - - pBlockInfo = addOneGroupInfo(pTSBuf, id); - } else { - pBlockInfo = tsBufGetLastGroupInfo(pTSBuf); - } - - assert(pBlockInfo->info.id == id); - - if ((tVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { - // new arrived data with different tags value, save current value into disk first - writeDataToDisk(pTSBuf); - } else { - expandBuffer(ptsData, len); - } - - tVariantAssign(&pTSBuf->block.tag, tag); - memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len); - - // todo check return value - setCheckTSOrder(pTSBuf, pData, len); - - ptsData->len += len; - pBlockInfo->len += len; - - pTSBuf->numOfTotal += len / TSDB_KEYSIZE; - - // the size of raw data exceeds the size of the default prepared buffer, so - // during getBufBlock, the output buffer needs to be large enough. - if (ptsData->len >= ptsData->threshold) { - writeDataToDisk(pTSBuf); - shrinkBuffer(ptsData); - } - - tsBufResetPos(pTSBuf); -} - -void tsBufFlush(STSBuf* pTSBuf) { - if (pTSBuf->tsData.len <= 0) { - return; - } - - writeDataToDisk(pTSBuf); - shrinkBuffer(&pTSBuf->tsData); - - STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - - // update prev vnode length info in file - TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pBlockInfoEx->info); - - // save the ts order into header - STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; - STSBufUpdateHeader(pTSBuf, &header); - - taosFsync(fileno(pTSBuf->f)); -} - -static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t numOfGroups, int32_t id) { - int32_t j = -1; - for (int32_t i = 0; i < numOfGroups; ++i) { - if (pGroupInfoEx[i].info.id == id) { - j = i; - break; - } - } - - return j; -} - -// todo opt performance by cache blocks info -static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) { - if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { - return -1; - } - - // sequentially read the compressed data blocks, start from the beginning of the comp data block of this vnode - int32_t i = 0; - bool decomp = false; - - while ((i++) <= blockIndex) { - if (readDataFromDisk(pTSBuf, TSDB_ORDER_ASC, decomp) == NULL) { - return -1; - } - } - - // set the file position to be the end of previous comp block - if (pTSBuf->cur.order == TSDB_ORDER_DESC) { - STSBlock* pBlock = &pTSBuf->block; - int32_t compBlockSize = - pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR); - UNUSED(ret); - } - - return 0; -} - -static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, tVariant* tag) { - bool decomp = false; - - int64_t offset = 0; - if (pTSBuf->cur.order == TSDB_ORDER_ASC) { - offset = pBlockInfo->offset; - } else { // reversed traverse starts from the end of block - offset = pBlockInfo->offset + pBlockInfo->compLen; - } - - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { - return -1; - } - - for (int32_t i = 0; i < pBlockInfo->numOfBlocks; ++i) { - if (readDataFromDisk(pTSBuf, pTSBuf->cur.order, decomp) == NULL) { - return -1; - } - - if (tVariantCompare(&pTSBuf->block.tag, tag) == 0) { - return (pTSBuf->cur.order == TSDB_ORDER_ASC)? i: (pBlockInfo->numOfBlocks - (i + 1)); - } - } - - return -1; -} - -static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex) { - STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info; - if (pBlockInfo->numOfBlocks <= blockIndex) { - assert(false); - } - - STSCursor* pCur = &pTSBuf->cur; - if (pCur->vgroupIndex == groupIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) || - (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) { - int32_t i = 0; - bool decomp = false; - int32_t step = abs(blockIndex - pCur->blockIndex); - - while ((++i) <= step) { - if (readDataFromDisk(pTSBuf, pCur->order, decomp) == NULL) { - return; - } - } - } else { - if (tsBufFindBlock(pTSBuf, pBlockInfo, blockIndex) == -1) { - assert(false); - } - } - - STSBlock* pBlock = &pTSBuf->block; - - size_t s = pBlock->numOfElem * TSDB_KEYSIZE; - - /* - * In order to accommodate all the qualified data, the actual buffer size for one block with identical tags value - * may exceed the maximum allowed size during *tsBufAppend* function by invoking expandBuffer function - */ - if (s > pTSBuf->tsData.allocSize) { - expandBuffer(&pTSBuf->tsData, (int32_t)s); - } - - pTSBuf->tsData.len = - tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, - pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - - assert((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len)); - - pCur->vgroupIndex = groupIndex; - pCur->blockIndex = blockIndex; - - pCur->tsIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : pBlock->numOfElem - 1; -} - -static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockInfo* pVInfo) { - if (offset < 0 || offset >= getDataStartOffset()) { - return -1; - } - - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { - return -1; - } - - fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f); - return 0; -} - -STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) { - int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id); - if (j == -1) { - return NULL; - } - - return &pTSBuf->pData[j].info; -} - -int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { - if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { - return -1; - } - - assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC); - - int32_t r = fseek(pTSBuf->f, 0, SEEK_SET); - if (r != 0) { - qError("fseek failed, errno:%d", errno); - return -1; - } - - size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); - if (ws != 1) { - qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader)); - return -1; - } - return 0; -} - -bool tsBufNextPos(STSBuf* pTSBuf) { - if (pTSBuf == NULL || pTSBuf->numOfGroups == 0) { - return false; - } - - STSCursor* pCur = &pTSBuf->cur; - - // get the first/last position according to traverse order - if (pCur->vgroupIndex == -1) { - if (pCur->order == TSDB_ORDER_ASC) { - tsBufGetBlock(pTSBuf, 0, 0); - - if (pTSBuf->block.numOfElem == 0) { // the whole list is empty, return - tsBufResetPos(pTSBuf); - return false; - } else { - return true; - } - - } else { // get the last timestamp record in the last block of the last vnode - assert(pTSBuf->numOfGroups > 0); - - int32_t groupIndex = pTSBuf->numOfGroups - 1; - pCur->vgroupIndex = groupIndex; - - int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; - STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); - int32_t blockIndex = pBlockInfo->numOfBlocks - 1; - - tsBufGetBlock(pTSBuf, groupIndex, blockIndex); - - pCur->tsIndex = pTSBuf->block.numOfElem - 1; - if (pTSBuf->block.numOfElem == 0) { - tsBufResetPos(pTSBuf); - return false; - } else { - return true; - } - } - } - - int32_t step = pCur->order == TSDB_ORDER_ASC ? 1 : -1; - - while (1) { - assert(pTSBuf->tsData.len == pTSBuf->block.numOfElem * TSDB_KEYSIZE); - - if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || - (pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) { - int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; - - STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); - if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) || - (pCur->blockIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { - if ((pCur->vgroupIndex >= pTSBuf->numOfGroups - 1 && pCur->order == TSDB_ORDER_ASC) || - (pCur->vgroupIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { - pCur->vgroupIndex = -1; - return false; - } - - if (pBlockInfo == NULL) { - return false; - } - - int32_t blockIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : (pBlockInfo->numOfBlocks - 1); - tsBufGetBlock(pTSBuf, pCur->vgroupIndex + step, blockIndex); - break; - - } else { - tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex + step); - break; - } - } else { - pCur->tsIndex += step; - break; - } - } - - return true; -} - -void tsBufResetPos(STSBuf* pTSBuf) { - if (pTSBuf == NULL) { - return; - } - - pTSBuf->cur = (STSCursor){.tsIndex = -1, .blockIndex = -1, .vgroupIndex = -1, .order = pTSBuf->cur.order}; -} - -STSElem tsBufGetElem(STSBuf* pTSBuf) { - STSElem elem1 = {.id = -1}; - if (pTSBuf == NULL) { - return elem1; - } - - STSCursor* pCur = &pTSBuf->cur; - if (pCur != NULL && pCur->vgroupIndex < 0) { - return elem1; - } - - STSBlock* pBlock = &pTSBuf->block; - - elem1.id = pTSBuf->pData[pCur->vgroupIndex].info.id; - elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); - elem1.tag = &pBlock->tag; - - return elem1; -} - -/** - * current only support ts comp data from two vnode merge - * @param pDestBuf - * @param pSrcBuf - * @param id - * @return - */ -int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { - if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfGroups <= 0) { - return 0; - } - - if (pDestBuf->numOfGroups + pSrcBuf->numOfGroups > TS_COMP_FILE_GROUP_MAX) { - return -1; - } - - // src can only have one vnode index - assert(pSrcBuf->numOfGroups == 1); - - // there are data in buffer, flush to disk first - tsBufFlush(pDestBuf); - - // compared with the last vnode id - int32_t id = tsBufGetLastGroupInfo((STSBuf*) pSrcBuf)->info.id; - if (id != tsBufGetLastGroupInfo(pDestBuf)->info.id) { - int32_t oldSize = pDestBuf->numOfGroups; - int32_t newSize = oldSize + pSrcBuf->numOfGroups; - - if (pDestBuf->numOfAlloc < newSize) { - pDestBuf->numOfAlloc = newSize; - - STSGroupBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); - if (tmp == NULL) { - return -1; - } - - pDestBuf->pData = tmp; - } - - // directly copy the vnode index information - memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfGroups * sizeof(STSGroupBlockInfoEx)); - - // set the new offset value - for (int32_t i = 0; i < pSrcBuf->numOfGroups; ++i) { - STSGroupBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize]; - pBlockInfoEx->info.offset = (pSrcBuf->pData[i].info.offset - getDataStartOffset()) + pDestBuf->fileSize; - pBlockInfoEx->info.id = id; - } - - pDestBuf->numOfGroups = newSize; - } else { - STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pDestBuf); - - pBlockInfoEx->len += pSrcBuf->pData[0].len; - pBlockInfoEx->info.numOfBlocks += pSrcBuf->pData[0].info.numOfBlocks; - pBlockInfoEx->info.compLen += pSrcBuf->pData[0].info.compLen; - pBlockInfoEx->info.id = id; - } - - int32_t r = fseek(pDestBuf->f, 0, SEEK_END); - assert(r == 0); - - int64_t offset = getDataStartOffset(); - int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; - int64_t written = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size); - - if (written == -1 || written != size) { - return -1; - } - - pDestBuf->numOfTotal += pSrcBuf->numOfTotal; - - int32_t oldSize = pDestBuf->fileSize; - - // file meta data may be cached, close and reopen the file for accurate file size. - fclose(pDestBuf->f); - pDestBuf->f = fopen(pDestBuf->path, "rb+"); - if (pDestBuf->f == NULL) { - return -1; - } - - struct stat fileStat; - if (fstat(fileno(pDestBuf->f), &fileStat) != 0) { - return -1; - } - pDestBuf->fileSize = (uint32_t)fileStat.st_size; - - assert(pDestBuf->fileSize == oldSize + size); - - return 0; -} - -STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order, int32_t id) { - STSBuf* pTSBuf = tsBufCreate(true, order); - - STSGroupBlockInfo* pBlockInfo = &(addOneGroupInfo(pTSBuf, 0)->info); - pBlockInfo->numOfBlocks = numOfBlocks; - pBlockInfo->compLen = len; - pBlockInfo->offset = getDataStartOffset(); - pBlockInfo->id = id; - - // update prev vnode length info in file - TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo); - - int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET); - if (ret == -1) { - qError("fseek failed, errno:%d", errno); - tsBufDestroy(pTSBuf); - return NULL; - } - size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f); - if (sz != len) { - qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len); - tsBufDestroy(pTSBuf); - return NULL; - } - pTSBuf->fileSize += len; - - pTSBuf->tsOrder = order; - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - - STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; - if (STSBufUpdateHeader(pTSBuf, &header) < 0) { - tsBufDestroy(pTSBuf); - return NULL; - } - - if (taosFsync(fileno(pTSBuf->f)) == -1) { - qError("fsync failed, errno:%d", errno); - tsBufDestroy(pTSBuf); - return NULL; - } - - return pTSBuf; -} - -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag) { - STSElem elem = {.id = -1}; - - if (pTSBuf == NULL) { - return elem; - } - - int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id); - if (j == -1) { - return elem; - } - - // for debug purpose - // tsBufDisplay(pTSBuf); - - STSCursor* pCur = &pTSBuf->cur; - STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[j].info; - - int32_t blockIndex = tsBufFindBlockByTag(pTSBuf, pBlockInfo, tag); - if (blockIndex < 0) { - return elem; - } - - pCur->vgroupIndex = j; - pCur->blockIndex = blockIndex; - tsBufGetBlock(pTSBuf, j, blockIndex); - - return tsBufGetElem(pTSBuf); -} - -STSCursor tsBufGetCursor(STSBuf* pTSBuf) { - STSCursor c = {.vgroupIndex = -1}; - if (pTSBuf == NULL) { - return c; - } - - return pTSBuf->cur; -} - -void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur) { - if (pTSBuf == NULL || pCur == NULL) { - return; - } - - // assert(pCur->vgroupIndex != -1 && pCur->tsIndex >= 0 && pCur->blockIndex >= 0); - if (pCur->vgroupIndex != -1) { - tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex); - } - - pTSBuf->cur = *pCur; -} - -void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order) { - if (pTSBuf == NULL) { - return; - } - - pTSBuf->cur.order = order; -} - -STSBuf* tsBufClone(STSBuf* pTSBuf) { - if (pTSBuf == NULL) { - return NULL; - } - - tsBufFlush(pTSBuf); - - return tsBufCreateFromFile(pTSBuf->path, false); -} - -void tsBufDisplay(STSBuf* pTSBuf) { - printf("-------start of ts comp file-------\n"); - printf("number of vnode:%d\n", pTSBuf->numOfGroups); - - int32_t old = pTSBuf->cur.order; - pTSBuf->cur.order = TSDB_ORDER_ASC; - - tsBufResetPos(pTSBuf); - - while (tsBufNextPos(pTSBuf)) { - STSElem elem = tsBufGetElem(pTSBuf); - if (elem.tag->nType == TSDB_DATA_TYPE_BIGINT) { - printf("%d-%" PRId64 "-%" PRId64 "\n", elem.id, elem.tag->i64, elem.ts); - } - } - - pTSBuf->cur.order = old; - printf("-------end of ts comp file-------\n"); -} - -static int32_t getDataStartOffset() { - return sizeof(STSBufFileHeader) + TS_COMP_FILE_GROUP_MAX * sizeof(STSGroupBlockInfo); -} - -// update prev vnode length info in file -static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo) { - int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSGroupBlockInfo); - doUpdateGroupInfo(pTSBuf, offset, pBlockInfo); -} - -static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { - const int32_t INITIAL_GROUPINFO_SIZE = 4; - - pTSBuf->numOfAlloc = INITIAL_GROUPINFO_SIZE; - pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx)); - if (pTSBuf->pData == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->tsData.rawBuf = malloc(MEM_BUF_SIZE); - if (pTSBuf->tsData.rawBuf == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->bufSize = MEM_BUF_SIZE; - pTSBuf->tsData.threshold = MEM_BUF_SIZE; - pTSBuf->tsData.allocSize = MEM_BUF_SIZE; - - pTSBuf->assistBuf = malloc(MEM_BUF_SIZE); - if (pTSBuf->assistBuf == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->block.payload = malloc(MEM_BUF_SIZE); - if (pTSBuf->block.payload == NULL) { - tsBufDestroy(pTSBuf); - return NULL; - } - - pTSBuf->fileSize += getDataStartOffset(); - return pTSBuf; -} - -int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf) { - if (pTSBuf == NULL) { - return 0; - } - - return pTSBuf->numOfGroups; -} - -void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) { - int32_t size = tsBufGetNumOfGroup(pTSBuf); - if (num != NULL) { - *num = size; - } - - *id = NULL; - if (size == 0) { - return; - } - - (*id) = malloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t)); - - for(int32_t i = 0; i < size; ++i) { - (*id)[i] = pTSBuf->pData[i].info.id; - } -} - -int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, int32_t* len, int32_t* numOfBlocks) { - assert(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups); - STSGroupBlockInfo *pBlockInfo = &pTSBuf->pData[groupIndex].info; - - *len = 0; - *numOfBlocks = 0; - - if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { - int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); -// qError("%p: fseek failed: %s", pSql, tstrerror(code)); - return code; - } - - size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f); - if (s != pBlockInfo->compLen) { - int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); -// tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); - return code; - } - - *len = pBlockInfo->compLen; - *numOfBlocks = pBlockInfo->numOfBlocks; - - return TSDB_CODE_SUCCESS; -} - -STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag) { - STSElem el = {.id = -1}; - - for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) { - el = tsBufGetElemStartPos(pTSBuf, pTSBuf->pData[i].info.id, pTag); - if (el.id == pTSBuf->pData[i].info.id) { - return el; - } - } - - return el; -} - -bool tsBufIsValidElem(STSElem* pElem) { - return pElem->id >= 0; -} diff --git a/2.0/src/query/src/queryMain.c b/2.0/src/query/src/queryMain.c deleted file mode 100644 index e6d804cea375c4b4f2fc8e10e34fdf50e823483f..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/queryMain.c +++ /dev/null @@ -1,769 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tmsg.h" -#include "tcache.h" -#include "tglobal.h" - -#include "exception.h" -#include "hash.h" -#include "texpr.h" -#include "qExecutor.h" -#include "qUtil.h" -#include "query.h" -#include "queryLog.h" -#include "tlosertree.h" -#include "ttype.h" - -typedef struct SQueryMgmt { - pthread_mutex_t lock; - SCacheObj *qinfoPool; // query handle pool - int32_t vgId; - bool closed; -} SQueryMgmt; - -static void queryMgmtKillQueryFn(void* handle, void* param1) { - void** fp = (void**)handle; - qKillQuery(*fp); -} - -static void freeqinfoFn(void *qhandle) { - void** handle = qhandle; - if (handle == NULL || *handle == NULL) { - return; - } - - qKillQuery(*handle); - qDestroyQueryInfo(*handle); -} - -void freeParam(SQueryParam *param) { - tfree(param->sql); - tfree(param->tagCond); - tfree(param->tbnameCond); - tfree(param->pTableIdList); - taosArrayDestroy(param->pOperator); - tfree(param->pExprs); - tfree(param->pSecExprs); - - tfree(param->pExpr); - tfree(param->pSecExpr); - - tfree(param->pGroupColIndex); - tfree(param->pTagColumnInfo); - tfree(param->pGroupbyExpr); - tfree(param->prevResult); -} - -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo, uint64_t qId) { - assert(pQueryMsg != NULL && tsdb != NULL); - - int32_t code = TSDB_CODE_SUCCESS; - - SQueryParam param = {0}; - code = convertQueryMsg(pQueryMsg, ¶m); - if (code != TSDB_CODE_SUCCESS) { - goto _over; - } - - if (pQueryMsg->numOfTables <= 0) { - qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - if (param.pTableIdList == NULL || taosArrayGetSize(param.pTableIdList) == 0) { - qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; - if ((code = createQueryFunc(&info, pQueryMsg->numOfOutput, ¶m.pExprs, param.pExpr, param.pTagColumnInfo, - pQueryMsg->queryType, pQueryMsg, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - - if (param.pSecExpr != NULL) { - if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, ¶m.pSecExprs, param.pSecExpr, param.pExprs, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } - - if (param.colCond != NULL) { - if ((code = createQueryFilter(param.colCond, pQueryMsg->colCondLen, ¶m.pFilters)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } - - param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code); - if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { - goto _over; - } - - bool isSTableQuery = false; - STableGroupInfo tableGroupInfo = {0}; - int64_t st = taosGetTimestampUs(); - - if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); - - qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); - if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { - isSTableQuery = true; - - // also note there's possibility that only one table in the super table - if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); - - // group by normal column, do not pass the group by condition to tsdb to group table into different group - int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; - if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(param.pGroupColIndex->flag)) { - numOfGroupByCols = 0; - } - - qDebug("qmsg:%p query stable, uid:%"PRIu64", tid:%d", pQueryMsg, id->uid, id->tid); - code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, param.tagCond, pQueryMsg->tagCondLen, - pQueryMsg->tagNameRelType, param.tbnameCond, &tableGroupInfo, param.pGroupColIndex, numOfGroupByCols); - - if (code != TSDB_CODE_SUCCESS) { - qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); - goto _over; - } - } else { - code = tsdbGetTableGroupFromIdList(tsdb, param.pTableIdList, &tableGroupInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _over; - } - - qDebug("qmsg:%p query on %u tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); - } - - int64_t el = taosGetTimestampUs() - st; - qDebug("qmsg:%p tag filter completed, numOfTables:%u, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); - } else { - assert(0); - } - - code = checkForQueryBuf(tableGroupInfo.numOfTables); - if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort - goto _over; - } - - assert(pQueryMsg->stableQuery == isSTableQuery); - (*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo, - param.pTagColumnInfo, param.pFilters, vgId, param.sql, qId, param.pUdfInfo); - - param.sql = NULL; - param.pExprs = NULL; - param.pSecExprs = NULL; - param.pGroupbyExpr = NULL; - param.pTagColumnInfo = NULL; - param.pFilters = NULL; - - if ((*pQInfo) == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _over; - } - param.pUdfInfo = NULL; - - code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pQInfo, ¶m, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL); - - _over: - if (param.pGroupbyExpr != NULL) { - taosArrayDestroy(param.pGroupbyExpr->columnInfo); - } - - tfree(param.colCond); - - destroyUdfInfo(param.pUdfInfo); - - taosArrayDestroy(param.pTableIdList); - param.pTableIdList = NULL; - - freeParam(¶m); - - for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) { - SColumnInfo* column = pQueryMsg->tableCols + i; - freeColumnFilterInfo(column->flist.filterInfo, column->flist.numOfFilters); - } - - filterFreeInfo(param.pFilters); - - //pQInfo already freed in initQInfo, but *pQInfo may not pointer to null; - if (code != TSDB_CODE_SUCCESS) { - *pQInfo = NULL; - } - - // if failed to add ref for all tables in this query, abort current query - return code; -} - -#ifdef TEST_IMPL -// wait moment -int waitMoment(SQInfo* pQInfo){ - if(pQInfo->sql) { - int ms = 0; - char* pcnt = strstr(pQInfo->sql, " count(*)"); - if(pcnt) return 0; - - char* pos = strstr(pQInfo->sql, " t_"); - if(pos){ - pos += 3; - ms = atoi(pos); - while(*pos >= '0' && *pos <= '9'){ - pos ++; - } - char unit_char = *pos; - if(unit_char == 'h'){ - ms *= 3600*1000; - } else if(unit_char == 'm'){ - ms *= 60*1000; - } else if(unit_char == 's'){ - ms *= 1000; - } - } - if(ms == 0) return 0; - printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql); - - if(ms < 1000) { - taosMsleep(ms); - } else { - int used_ms = 0; - while(used_ms < ms) { - taosMsleep(1000); - used_ms += 1000; - if(isQueryKilled(pQInfo)){ - printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); - break; - } - } - } - } - return 1; -} -#endif - -bool qTableQuery(qinfo_t qinfo, uint64_t *qId) { - SQInfo *pQInfo = (SQInfo *)qinfo; - assert(pQInfo && pQInfo->signature == pQInfo); - int64_t threadId = taosGetSelfPthreadId(); - - int64_t curOwner = 0; - if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { - qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", pQInfo->qId, pQInfo, (void*) curOwner); - pQInfo->code = TSDB_CODE_QRY_IN_EXEC; - return false; - } - - *qId = pQInfo->qId; - if(pQInfo->startExecTs == 0) - pQInfo->startExecTs = taosGetTimestampMs(); - - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" it is already killed, abort", pQInfo->qId); - return doBuildResCheck(pQInfo); - } - - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { - qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", pQInfo->qId); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - return doBuildResCheck(pQInfo); - } - - // error occurs, record the error code and return to client - int32_t ret = setjmp(pQInfo->runtimeEnv.env); - if (ret != TSDB_CODE_SUCCESS) { - publishQueryAbortEvent(pQInfo, ret); - pQInfo->code = ret; - qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", pQInfo->qId, tstrerror(pQInfo->code)); - return doBuildResCheck(pQInfo); - } - - qDebug("QInfo:0x%"PRIx64" query task is launched", pQInfo->qId); - - bool newgroup = false; - publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_BEFORE_OPERATOR_EXEC); - - int64_t st = taosGetTimestampUs(); - pRuntimeEnv->outputBuf = pRuntimeEnv->proot->exec(pRuntimeEnv->proot, &newgroup); - pQInfo->summary.elapsedTime += (taosGetTimestampUs() - st); -#ifdef TEST_IMPL - waitMoment(pQInfo); -#endif - publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_AFTER_OPERATOR_EXEC); - pRuntimeEnv->resultInfo.total += GET_NUM_OF_RESULTS(pRuntimeEnv); - - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" query is killed", pQInfo->qId); - } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { - qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pQInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, - pRuntimeEnv->resultInfo.total); - } else { - qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pQInfo->qId, - GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total); - } - - return doBuildResCheck(pQInfo); -} - -int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - qError("QInfo invalid qhandle"); - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - *buildRes = false; - if (IS_QUERY_KILLED(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" query is killed, code:0x%08x", pQInfo->qId, pQInfo->code); - return pQInfo->code; - } - - int32_t code = TSDB_CODE_SUCCESS; - - if (tsRetrieveBlockingModel) { - pQInfo->rspContext = pRspContext; - tsem_wait(&pQInfo->ready); - *buildRes = true; - code = pQInfo->code; - } else { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - - pthread_mutex_lock(&pQInfo->lock); - - assert(pQInfo->rspContext == NULL); - if (pQInfo->dataReady == QUERY_RESULT_READY) { - *buildRes = true; - qDebug("QInfo:0x%"PRIx64" retrieve result info, rowsize:%d, rows:%d, code:%s", pQInfo->qId, pQueryAttr->resultRowSize, - GET_NUM_OF_RESULTS(pRuntimeEnv), tstrerror(pQInfo->code)); - } else { - *buildRes = false; - qDebug("QInfo:0x%"PRIx64" retrieve req set query return result after paused", pQInfo->qId); - pQInfo->rspContext = pRspContext; - assert(pQInfo->rspContext != NULL); - } - - code = pQInfo->code; - pthread_mutex_unlock(&pQInfo->lock); - } - - return code; -} - -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen, bool* continueExec) { - SQInfo *pQInfo = (SQInfo *)qinfo; - int32_t compLen = 0; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - - int32_t s = GET_NUM_OF_RESULTS(pRuntimeEnv); - size_t size = pQueryAttr->resultRowSize * s; - size += sizeof(int32_t); - size += sizeof(STableIdInfo) * taosHashGetSize(pRuntimeEnv->pTableRetrieveTsMap); - - *contLen = (int32_t)(size + sizeof(SRetrieveTableRsp)); - - // current solution only avoid crash, but cannot return error code to client - *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); - if (*pRsp == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - (*pRsp)->numOfRows = htonl((int32_t)s); - - if (pQInfo->code == TSDB_CODE_SUCCESS) { - (*pRsp)->offset = htobe64(pQInfo->runtimeEnv.currentOffset); - (*pRsp)->useconds = htobe64(pQInfo->summary.elapsedTime); - } else { - (*pRsp)->offset = 0; - (*pRsp)->useconds = htobe64(pQInfo->summary.elapsedTime); - } - - (*pRsp)->precision = htons(pQueryAttr->precision); - (*pRsp)->compressed = (int8_t)((tsCompressColData != -1) && checkNeedToCompressQueryCol(pQInfo)); - - if (GET_NUM_OF_RESULTS(&(pQInfo->runtimeEnv)) > 0 && pQInfo->code == TSDB_CODE_SUCCESS) { - doDumpQueryResult(pQInfo, (*pRsp)->data, (*pRsp)->compressed, &compLen); - } else { - setQueryStatus(pRuntimeEnv, QUERY_OVER); - } - - if ((*pRsp)->compressed && compLen != 0) { - int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput; - int32_t origSize = pQueryAttr->resultRowSize * s; - int32_t compSize = compLen + numOfCols * sizeof(int32_t); - *contLen = *contLen - origSize + compSize; - *pRsp = (SRetrieveTableRsp *)rpcReallocCont(*pRsp, *contLen); - qDebug("QInfo:0x%"PRIx64" compress col data, uncompressed size:%d, compressed size:%d, ratio:%.2f", - pQInfo->qId, origSize, compSize, (float)origSize / (float)compSize); - } - (*pRsp)->compLen = htonl(compLen); - - pQInfo->rspContext = NULL; - pQInfo->dataReady = QUERY_RESULT_NOT_READY; - - if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_OVER)) { - // here current thread hold the refcount, so it is safe to free tsdbQueryHandle. - *continueExec = false; - (*pRsp)->completed = 1; // notify no more result to client - qDebug("QInfo:0x%"PRIx64" no more results to retrieve", pQInfo->qId); - } else { - *continueExec = true; - qDebug("QInfo:0x%"PRIx64" has more results to retrieve", pQInfo->qId); - } - - // the memory should be freed if the code of pQInfo is not TSDB_CODE_SUCCESS - if (pQInfo->code != TSDB_CODE_SUCCESS) { - rpcFreeCont(*pRsp); - *pRsp = NULL; - } - - return pQInfo->code; -} - -void* qGetResultRetrieveMsg(qinfo_t qinfo) { - SQInfo* pQInfo = (SQInfo*) qinfo; - assert(pQInfo != NULL); - - return pQInfo->rspContext; -} - -int32_t qKillQuery(qinfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - qDebug("QInfo:0x%"PRIx64" query killed", pQInfo->qId); - setTaskKilled(pQInfo); - - // Wait for the query executing thread being stopped/ - // Once the query is stopped, the owner of qHandle will be cleared immediately. - while (pQInfo->owner != 0) { - taosMsleep(100); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t qQueryCompleted(qinfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQInfo->runtimeEnv.status, QUERY_OVER); -} - -void qDestroyQueryInfo(qinfo_t qHandle) { - SQInfo* pQInfo = (SQInfo*) qHandle; - if (!isValidQInfo(pQInfo)) { - return; - } - - qDebug("QInfo:0x%"PRIx64" query completed", pQInfo->qId); - queryCostStatis(pQInfo); // print the query cost summary - freeQInfo(pQInfo); -} - -void* qOpenQueryMgmt(int32_t vgId) { - const int32_t refreshHandleInterval = 30; // every 30 seconds, refresh handle pool - - char cacheName[128] = {0}; - sprintf(cacheName, "qhandle_%d", vgId); - - SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt)); - if (pQueryMgmt == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pQueryMgmt->qinfoPool = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshHandleInterval, true, freeqinfoFn, cacheName); - pQueryMgmt->closed = false; - pQueryMgmt->vgId = vgId; - - pthread_mutex_init(&pQueryMgmt->lock, NULL); - - qDebug("vgId:%d, open querymgmt success", vgId); - return pQueryMgmt; -} - -void qQueryMgmtNotifyClosed(void* pQMgmt) { - if (pQMgmt == NULL) { - return; - } - - SQueryMgmt* pQueryMgmt = pQMgmt; - qInfo("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId); - - pthread_mutex_lock(&pQueryMgmt->lock); - pQueryMgmt->closed = true; - pthread_mutex_unlock(&pQueryMgmt->lock); - - taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn, NULL); -} - -void qQueryMgmtReOpen(void *pQMgmt) { - if (pQMgmt == NULL) { - return; - } - - SQueryMgmt *pQueryMgmt = pQMgmt; - qInfo("vgId:%d, set querymgmt reopen", pQueryMgmt->vgId); - - pthread_mutex_lock(&pQueryMgmt->lock); - pQueryMgmt->closed = false; - pthread_mutex_unlock(&pQueryMgmt->lock); -} - -void qCleanupQueryMgmt(void* pQMgmt) { - if (pQMgmt == NULL) { - return; - } - - SQueryMgmt* pQueryMgmt = pQMgmt; - int32_t vgId = pQueryMgmt->vgId; - - assert(pQueryMgmt->closed); - - SCacheObj* pqinfoPool = pQueryMgmt->qinfoPool; - pQueryMgmt->qinfoPool = NULL; - - taosCacheCleanup(pqinfoPool); - pthread_mutex_destroy(&pQueryMgmt->lock); - tfree(pQueryMgmt); - - qDebug("vgId:%d, queryMgmt cleanup completed", vgId); -} - -void** qRegisterQInfo(void* pMgmt, uint64_t qId, void *qInfo) { - if (pMgmt == NULL) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } - - SQueryMgmt *pQueryMgmt = pMgmt; - if (pQueryMgmt->qinfoPool == NULL) { - qError("QInfo:0x%"PRIx64"-%p failed to add qhandle into qMgmt, since qMgmt is closed", qId, (void*)qInfo); - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } - - pthread_mutex_lock(&pQueryMgmt->lock); - if (pQueryMgmt->closed) { - pthread_mutex_unlock(&pQueryMgmt->lock); - qError("QInfo:0x%"PRIx64"-%p failed to add qhandle into cache, since qMgmt is colsing", qId, (void*)qInfo); - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } else { - void** handle = taosCachePut(pQueryMgmt->qinfoPool, &qId, sizeof(qId), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE), - (getMaximumIdleDurationSec()*1000)); - pthread_mutex_unlock(&pQueryMgmt->lock); - - return handle; - } -} - -void** qAcquireQInfo(void* pMgmt, uint64_t _key) { - SQueryMgmt *pQueryMgmt = pMgmt; - - if (pQueryMgmt->closed) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } - - if (pQueryMgmt->qinfoPool == NULL) { - terrno = TSDB_CODE_QRY_INVALID_QHANDLE; - return NULL; - } - - void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, &_key, sizeof(_key)); - if (handle == NULL || *handle == NULL) { - terrno = TSDB_CODE_QRY_INVALID_QHANDLE; - return NULL; - } else { - return handle; - } -} - -void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { - SQueryMgmt *pQueryMgmt = pMgmt; - if (pQueryMgmt->qinfoPool == NULL) { - return NULL; - } - - taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); - return 0; -} - -//kill by qid -int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount) { - int32_t error = TSDB_CODE_SUCCESS; - void** handle = qAcquireQInfo(pMgmt, qId); - if(handle == NULL) return terrno; - - SQInfo* pQInfo = (SQInfo*)(*handle); - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - qWarn("QId:0x%"PRIx64" be killed(no memory commit).", pQInfo->qId); - setTaskKilled(pQInfo); - - // wait query stop - int32_t loop = 0; - while (pQInfo->owner != 0) { - taosMsleep(waitMs); - if(loop++ > waitCount){ - error = TSDB_CODE_FAILED; - break; - } - } - - qReleaseQInfo(pMgmt, (void **)&handle, true); - return error; -} - -// local struct -typedef struct { - int64_t qId; - int64_t startExecTs; -} SLongQuery; - -// callbark for sort compare -static int compareLongQuery(const void* p1, const void* p2) { - // sort desc - SLongQuery* plq1 = *(SLongQuery**)p1; - SLongQuery* plq2 = *(SLongQuery**)p2; - if(plq1->startExecTs == plq2->startExecTs) { - return 0; - } else if(plq1->startExecTs > plq2->startExecTs) { - return 1; - } else { - return -1; - } -} - -// callback for taosCacheRefresh -static void cbFoundItem(void* handle, void* param1) { - SQInfo * qInfo = *(SQInfo**) handle; - if(qInfo == NULL) return ; - SArray* qids = (SArray*) param1; - if(qids == NULL) return ; - - bool usedMem = true; - bool usedIMem = true; - SMemTable* mem = qInfo->query.memRef.snapshot.omem; - SMemTable* imem = qInfo->query.memRef.snapshot.imem; - if(mem == NULL || T_REF_VAL_GET(mem) == 0) - usedMem = false; - if(imem == NULL || T_REF_VAL_GET(mem) == 0) - usedIMem = false ; - - if(!usedMem && !usedIMem) - return ; - - // push to qids - SLongQuery* plq = (SLongQuery*)malloc(sizeof(SLongQuery)); - plq->qId = qInfo->qId; - plq->startExecTs = qInfo->startExecTs; - taosArrayPush(qids, &plq); -} - -// longquery -void* qObtainLongQuery(void* param){ - SQueryMgmt* qMgmt = (SQueryMgmt*)param; - if(qMgmt == NULL || qMgmt->qinfoPool == NULL) - return NULL; - SArray* qids = taosArrayInit(4, sizeof(int64_t*)); - if(qids == NULL) return NULL; - // Get each item - taosCacheRefresh(qMgmt->qinfoPool, cbFoundItem, qids); - - size_t cnt = taosArrayGetSize(qids); - if(cnt == 0) { - taosArrayDestroy(qids); - return NULL; - } - if(cnt > 1) - taosArraySort(qids, compareLongQuery); - - return qids; -} - -//solve tsdb no block to commit -bool qFixedNoBlock(void* pRepo, void* pMgmt, int32_t longQueryMs) { - SQueryMgmt *pQueryMgmt = pMgmt; - bool fixed = false; - - // qid top list - SArray *qids = (SArray*)qObtainLongQuery(pQueryMgmt); - if(qids == NULL) return false; - - // kill Query - int64_t now = taosGetTimestampMs(); - size_t cnt = taosArrayGetSize(qids); - size_t i; - SLongQuery* plq; - for(i=0; i < cnt; i++) { - plq = (SLongQuery* )taosArrayGetP(qids, i); - if(plq->startExecTs > now) continue; - if(now - plq->startExecTs >= longQueryMs) { - qKillQueryByQId(pMgmt, plq->qId, 500, 10); // wait 50*100 ms - if(tsdbNoProblem(pRepo)) { - fixed = true; - qWarn("QId:0x%"PRIx64" fixed problem after kill this query.", plq->qId); - break; - } - } - } - - // free qids - for(i=0; i < cnt; i++) { - free(taosArrayGetP(qids, i)); - } - taosArrayDestroy(qids); - return fixed; -} - -//solve tsdb no block to commit -bool qSolveCommitNoBlock(void* pRepo, void* pMgmt) { - qWarn("pRepo=%p start solve problem.", pRepo); - if(qFixedNoBlock(pRepo, pMgmt, 10*60*1000)) { - return true; - } - if(qFixedNoBlock(pRepo, pMgmt, 2*60*1000)){ - return true; - } - if(qFixedNoBlock(pRepo, pMgmt, 30*1000)){ - return true; - } - qWarn("pRepo=%p solve problem failed.", pRepo); - return false; -} diff --git a/2.0/src/query/src/sql.c b/2.0/src/query/src/sql.c deleted file mode 100644 index 4fe3e860a6a90dbdd96bd95faacb5d48df2ee541..0000000000000000000000000000000000000000 --- a/2.0/src/query/src/sql.c +++ /dev/null @@ -1,4423 +0,0 @@ -/* -** 2000-05-29 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Driver template for the LEMON parser generator. -** -** The "lemon" program processes an LALR(1) input grammar file, then uses -** this template to construct a parser. The "lemon" program inserts text -** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the -** interstitial "-" characters) contained in this template is changed into -** the value of the %name directive from the grammar. Otherwise, the content -** of this template is copied straight through into the generate parser -** source file. -** -** The following is the concatenation of all %include directives from the -** input grammar file: -*/ -#include -#include -/************ Begin %include sections from the grammar ************************/ -#line 23 "sql.y" - -#include -#include -#include -#include -#include -#include "qSqlparser.h" -#include "tmsgtype.h" -#include "ttoken.h" -#include "ttokendef.h" -#include "tutil.h" -#include "tvariant.h" -#line 42 "sql.c" -/**************** End of %include directives **********************************/ -/* These constants specify the various numeric values for terminal symbols -** in a format understandable to "makeheaders". This section is blank unless -** "lemon" is run with the "-m" command-line option. -***************** Begin makeheaders token definitions *************************/ -/**************** End makeheaders token definitions ***************************/ - -/* The next sections is a series of control #defines. -** various aspects of the generated parser. -** YYCODETYPE is the data type used to store the integer codes -** that represent terminal and non-terminal symbols. -** "unsigned char" is used if there are fewer than -** 256 symbols. Larger types otherwise. -** YYNOCODE is a number of type YYCODETYPE that is not used for -** any terminal or nonterminal symbol. -** YYFALLBACK If defined, this indicates that one or more tokens -** (also known as: "terminal symbols") have fall-back -** values which should be used if the original symbol -** would not parse. This permits keywords to sometimes -** be used as identifiers, for example. -** YYACTIONTYPE is the data type used for "action codes" - numbers -** that indicate what to do in response to the next -** token. -** ParseTOKENTYPE is the data type used for minor type for terminal -** symbols. Background: A "minor type" is a semantic -** value associated with a terminal or non-terminal -** symbols. For example, for an "ID" terminal symbol, -** the minor type might be the name of the identifier. -** Each non-terminal can have a different minor type. -** Terminal symbols all have the same minor type, though. -** This macros defines the minor type for terminal -** symbols. -** YYMINORTYPE is the data type used for all minor types. -** This is typically a union of many types, one of -** which is ParseTOKENTYPE. The entry in the union -** for terminal symbols is called "yy0". -** YYSTACKDEPTH is the maximum depth of the parser's stack. If -** zero the stack is dynamically sized using realloc() -** ParseARG_SDECL A static variable declaration for the %extra_argument -** ParseARG_PDECL A parameter declaration for the %extra_argument -** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter -** ParseARG_STORE Code to store %extra_argument into yypParser -** ParseARG_FETCH Code to extract %extra_argument from yypParser -** ParseCTX_* As ParseARG_ except for %extra_context -** YYERRORSYMBOL is the code number of the error symbol. If not -** defined, then do no error processing. -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar -** YYNTOKEN Number of terminal symbols -** YY_MAX_SHIFT Maximum value for shift actions -** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions -** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions -** YY_ERROR_ACTION The yy_action[] code for syntax error -** YY_ACCEPT_ACTION The yy_action[] code for accept -** YY_NO_ACTION The yy_action[] code for no-op -** YY_MIN_REDUCE Minimum value for reduce actions -** YY_MAX_REDUCE Maximum value for reduce actions -*/ -#ifndef INTERFACE -# define INTERFACE 1 -#endif -/************* Begin control #defines *****************************************/ -#define YYCODETYPE unsigned short int -#define YYNOCODE 278 -#define YYACTIONTYPE unsigned short int -#define ParseTOKENTYPE SStrToken -typedef union { - int yyinit; - ParseTOKENTYPE yy0; - SRelationInfo* yy8; - SWindowStateVal yy40; - SSqlNode* yy56; - SCreateDbInfo yy90; - int yy96; - int32_t yy104; - SSessionWindowVal yy147; - SCreatedTableInfo yy152; - SLimitVal yy166; - SCreateAcctInfo yy171; - TAOS_FIELD yy183; - int64_t yy325; - SIntervalVal yy400; - SArray* yy421; - tVariant yy430; - SCreateTableSql* yy438; - tSqlExpr* yy439; -} YYMINORTYPE; -#ifndef YYSTACKDEPTH -#define YYSTACKDEPTH 100 -#endif -#define ParseARG_SDECL SSqlInfo* pInfo; -#define ParseARG_PDECL ,SSqlInfo* pInfo -#define ParseARG_PARAM ,pInfo -#define ParseARG_FETCH SSqlInfo* pInfo=yypParser->pInfo; -#define ParseARG_STORE yypParser->pInfo=pInfo; -#define ParseCTX_SDECL -#define ParseCTX_PDECL -#define ParseCTX_PARAM -#define ParseCTX_FETCH -#define ParseCTX_STORE -#define YYFALLBACK 1 -#define YYNSTATE 368 -#define YYNRULE 294 -#define YYNRULE_WITH_ACTION 294 -#define YYNTOKEN 197 -#define YY_MAX_SHIFT 367 -#define YY_MIN_SHIFTREDUCE 576 -#define YY_MAX_SHIFTREDUCE 869 -#define YY_ERROR_ACTION 870 -#define YY_ACCEPT_ACTION 871 -#define YY_NO_ACTION 872 -#define YY_MIN_REDUCE 873 -#define YY_MAX_REDUCE 1166 -/************* End control #defines *******************************************/ -#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) - -/* Define the yytestcase() macro to be a no-op if is not already defined -** otherwise. -** -** Applications can choose to define yytestcase() in the %include section -** to a macro that can assist in verifying code coverage. For production -** code the yytestcase() macro should be turned off. But it is useful -** for testing. -*/ -#ifndef yytestcase -# define yytestcase(X) -#endif - - -/* Next are the tables used to determine what action to take based on the -** current state and lookahead token. These tables are used to implement -** functions that take a state number and lookahead value and return an -** action integer. -** -** Suppose the action integer is N. Then the action is determined as -** follows -** -** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead -** token onto the stack and goto state N. -** -** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then -** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. -** -** N == YY_ERROR_ACTION A syntax error has occurred. -** -** N == YY_ACCEPT_ACTION The parser accepts its input. -** -** N == YY_NO_ACTION No such action. Denotes unused -** slots in the yy_action[] table. -** -** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE -** and YY_MAX_REDUCE -** -** The action table is constructed as a single large table named yy_action[]. -** Given state S and lookahead X, the action is computed as either: -** -** (A) N = yy_action[ yy_shift_ofst[S] + X ] -** (B) N = yy_default[S] -** -** The (A) formula is preferred. The B formula is used instead if -** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X. -** -** The formulas above are for computing the action when the lookahead is -** a terminal symbol. If the lookahead is a non-terminal (as occurs after -** a reduce action) then the yy_reduce_ofst[] array is used in place of -** the yy_shift_ofst[] array. -** -** The following are the tables generated in this section: -** -** yy_action[] A single table containing all actions. -** yy_lookahead[] A table containing the lookahead for each entry in -** yy_action. Used to detect hash collisions. -** yy_shift_ofst[] For each state, the offset into yy_action for -** shifting terminals. -** yy_reduce_ofst[] For each state, the offset into yy_action for -** shifting non-terminals after a reduce. -** yy_default[] Default action for each state. -** -*********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (773) -static const YYACTIONTYPE yy_action[] = { - /* 0 */ 23, 628, 366, 235, 1051, 208, 241, 712, 211, 629, - /* 10 */ 1029, 871, 367, 59, 60, 173, 63, 64, 1042, 1142, - /* 20 */ 255, 53, 52, 51, 628, 62, 324, 67, 65, 68, - /* 30 */ 66, 157, 629, 286, 238, 58, 57, 344, 343, 56, - /* 40 */ 55, 54, 59, 60, 247, 63, 64, 252, 1029, 255, - /* 50 */ 53, 52, 51, 209, 62, 324, 67, 65, 68, 66, - /* 60 */ 999, 1042, 997, 998, 58, 57, 664, 1000, 56, 55, - /* 70 */ 54, 1001, 1048, 1002, 1003, 58, 57, 277, 1015, 56, - /* 80 */ 55, 54, 59, 60, 164, 63, 64, 38, 82, 255, - /* 90 */ 53, 52, 51, 88, 62, 324, 67, 65, 68, 66, - /* 100 */ 284, 283, 249, 752, 58, 57, 1029, 211, 56, 55, - /* 110 */ 54, 38, 59, 61, 806, 63, 64, 1042, 1143, 255, - /* 120 */ 53, 52, 51, 628, 62, 324, 67, 65, 68, 66, - /* 130 */ 45, 629, 237, 239, 58, 57, 1026, 164, 56, 55, - /* 140 */ 54, 60, 1023, 63, 64, 771, 772, 255, 53, 52, - /* 150 */ 51, 95, 62, 324, 67, 65, 68, 66, 38, 1090, - /* 160 */ 1025, 296, 58, 57, 322, 83, 56, 55, 54, 577, - /* 170 */ 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, - /* 180 */ 588, 589, 590, 155, 322, 236, 63, 64, 756, 248, - /* 190 */ 255, 53, 52, 51, 628, 62, 324, 67, 65, 68, - /* 200 */ 66, 251, 629, 245, 354, 58, 57, 1026, 215, 56, - /* 210 */ 55, 54, 1089, 44, 320, 361, 360, 319, 318, 317, - /* 220 */ 359, 316, 315, 314, 358, 313, 357, 356, 808, 38, - /* 230 */ 1, 180, 24, 991, 979, 980, 981, 982, 983, 984, - /* 240 */ 985, 986, 987, 988, 989, 990, 992, 993, 256, 214, - /* 250 */ 38, 254, 821, 922, 100, 810, 222, 813, 164, 816, - /* 260 */ 192, 211, 139, 138, 137, 221, 809, 254, 821, 329, - /* 270 */ 88, 810, 1143, 813, 246, 816, 1028, 29, 1026, 67, - /* 280 */ 65, 68, 66, 38, 1162, 233, 234, 58, 57, 325, - /* 290 */ 1017, 56, 55, 54, 38, 333, 56, 55, 54, 1026, - /* 300 */ 269, 233, 234, 258, 5, 41, 182, 45, 211, 273, - /* 310 */ 272, 181, 106, 111, 102, 110, 164, 73, 736, 1143, - /* 320 */ 932, 733, 812, 734, 815, 735, 263, 192, 334, 276, - /* 330 */ 309, 80, 1026, 94, 69, 123, 117, 128, 229, 335, - /* 340 */ 362, 960, 127, 1026, 133, 136, 126, 202, 200, 198, - /* 350 */ 69, 260, 261, 130, 197, 143, 142, 141, 140, 74, - /* 360 */ 44, 97, 361, 360, 788, 923, 38, 359, 38, 822, - /* 370 */ 817, 358, 192, 357, 356, 38, 818, 38, 38, 259, - /* 380 */ 811, 257, 814, 332, 331, 822, 817, 264, 125, 298, - /* 390 */ 264, 93, 818, 326, 1012, 1013, 35, 1016, 178, 14, - /* 400 */ 354, 179, 265, 96, 262, 264, 339, 338, 154, 152, - /* 410 */ 151, 336, 749, 340, 81, 1026, 1027, 1026, 3, 193, - /* 420 */ 341, 787, 342, 346, 1026, 278, 1026, 1026, 365, 364, - /* 430 */ 148, 85, 86, 99, 76, 737, 738, 768, 9, 39, - /* 440 */ 778, 779, 722, 819, 301, 724, 216, 303, 1014, 723, - /* 450 */ 34, 159, 844, 823, 70, 26, 39, 253, 39, 70, - /* 460 */ 79, 98, 627, 70, 135, 134, 25, 25, 280, 280, - /* 470 */ 16, 116, 15, 115, 77, 18, 25, 17, 741, 6, - /* 480 */ 742, 274, 739, 304, 740, 20, 122, 19, 121, 22, - /* 490 */ 217, 21, 711, 1100, 1137, 1136, 1135, 825, 231, 156, - /* 500 */ 232, 820, 212, 213, 218, 210, 1099, 219, 220, 224, - /* 510 */ 225, 226, 223, 207, 1154, 243, 1096, 1095, 244, 345, - /* 520 */ 1050, 1061, 1043, 48, 1058, 1059, 1063, 153, 281, 158, - /* 530 */ 163, 292, 1024, 175, 1082, 174, 1081, 279, 84, 285, - /* 540 */ 1022, 310, 176, 240, 177, 171, 167, 937, 306, 307, - /* 550 */ 308, 767, 311, 312, 1040, 165, 166, 46, 287, 289, - /* 560 */ 297, 299, 205, 168, 42, 78, 75, 50, 323, 931, - /* 570 */ 330, 1161, 113, 1160, 295, 169, 293, 291, 1157, 183, - /* 580 */ 337, 1153, 119, 288, 1152, 1149, 184, 957, 43, 40, - /* 590 */ 47, 206, 919, 129, 49, 917, 131, 132, 915, 914, - /* 600 */ 266, 195, 196, 911, 910, 909, 908, 907, 906, 905, - /* 610 */ 199, 201, 902, 900, 898, 896, 203, 893, 204, 889, - /* 620 */ 355, 124, 89, 290, 1083, 347, 348, 349, 350, 351, - /* 630 */ 352, 353, 363, 869, 230, 250, 305, 267, 268, 868, - /* 640 */ 270, 227, 228, 271, 867, 850, 107, 936, 935, 108, - /* 650 */ 849, 275, 280, 300, 10, 282, 744, 87, 30, 90, - /* 660 */ 913, 912, 904, 186, 958, 190, 185, 187, 144, 191, - /* 670 */ 189, 188, 145, 146, 147, 903, 995, 895, 4, 894, - /* 680 */ 959, 769, 160, 33, 780, 170, 172, 2, 161, 162, - /* 690 */ 774, 91, 242, 776, 92, 1005, 294, 11, 12, 31, - /* 700 */ 32, 13, 27, 302, 28, 99, 101, 104, 36, 103, - /* 710 */ 642, 37, 105, 677, 675, 674, 673, 671, 670, 669, - /* 720 */ 666, 321, 109, 632, 7, 826, 824, 8, 327, 328, - /* 730 */ 112, 114, 71, 72, 118, 714, 39, 120, 713, 710, - /* 740 */ 658, 656, 648, 654, 650, 652, 646, 644, 680, 679, - /* 750 */ 678, 676, 672, 668, 667, 194, 630, 594, 873, 872, - /* 760 */ 872, 872, 872, 872, 872, 872, 872, 872, 872, 872, - /* 770 */ 872, 149, 150, -}; -static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 266, 1, 199, 200, 199, 266, 245, 5, 266, 9, - /* 10 */ 249, 197, 198, 13, 14, 253, 16, 17, 247, 277, - /* 20 */ 20, 21, 22, 23, 1, 25, 26, 27, 28, 29, - /* 30 */ 30, 199, 9, 271, 263, 35, 36, 35, 36, 39, - /* 40 */ 40, 41, 13, 14, 245, 16, 17, 206, 249, 20, - /* 50 */ 21, 22, 23, 266, 25, 26, 27, 28, 29, 30, - /* 60 */ 223, 247, 225, 226, 35, 36, 5, 230, 39, 40, - /* 70 */ 41, 234, 267, 236, 237, 35, 36, 263, 0, 39, - /* 80 */ 40, 41, 13, 14, 199, 16, 17, 199, 88, 20, - /* 90 */ 21, 22, 23, 84, 25, 26, 27, 28, 29, 30, - /* 100 */ 268, 269, 245, 39, 35, 36, 249, 266, 39, 40, - /* 110 */ 41, 199, 13, 14, 85, 16, 17, 247, 277, 20, - /* 120 */ 21, 22, 23, 1, 25, 26, 27, 28, 29, 30, - /* 130 */ 121, 9, 244, 263, 35, 36, 248, 199, 39, 40, - /* 140 */ 41, 14, 199, 16, 17, 127, 128, 20, 21, 22, - /* 150 */ 23, 250, 25, 26, 27, 28, 29, 30, 199, 274, - /* 160 */ 248, 276, 35, 36, 86, 264, 39, 40, 41, 47, - /* 170 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 180 */ 58, 59, 60, 61, 86, 63, 16, 17, 124, 246, - /* 190 */ 20, 21, 22, 23, 1, 25, 26, 27, 28, 29, - /* 200 */ 30, 206, 9, 244, 92, 35, 36, 248, 266, 39, - /* 210 */ 40, 41, 274, 100, 101, 102, 103, 104, 105, 106, - /* 220 */ 107, 108, 109, 110, 111, 112, 113, 114, 1, 199, - /* 230 */ 208, 209, 46, 223, 224, 225, 226, 227, 228, 229, - /* 240 */ 230, 231, 232, 233, 234, 235, 236, 237, 206, 63, - /* 250 */ 199, 1, 2, 205, 207, 5, 70, 7, 199, 9, - /* 260 */ 212, 266, 76, 77, 78, 79, 39, 1, 2, 83, - /* 270 */ 84, 5, 277, 7, 244, 9, 249, 84, 248, 27, - /* 280 */ 28, 29, 30, 199, 249, 35, 36, 35, 36, 39, - /* 290 */ 243, 39, 40, 41, 199, 244, 39, 40, 41, 248, - /* 300 */ 144, 35, 36, 70, 64, 65, 66, 121, 266, 153, - /* 310 */ 154, 71, 72, 73, 74, 75, 199, 99, 2, 277, - /* 320 */ 205, 5, 5, 7, 7, 9, 70, 212, 244, 143, - /* 330 */ 90, 145, 248, 274, 84, 64, 65, 66, 152, 244, - /* 340 */ 221, 222, 71, 248, 73, 74, 75, 64, 65, 66, - /* 350 */ 84, 35, 36, 82, 71, 72, 73, 74, 75, 141, - /* 360 */ 100, 207, 102, 103, 78, 205, 199, 107, 199, 119, - /* 370 */ 120, 111, 212, 113, 114, 199, 126, 199, 199, 146, - /* 380 */ 5, 148, 7, 150, 151, 119, 120, 199, 80, 272, - /* 390 */ 199, 274, 126, 15, 240, 241, 242, 243, 210, 84, - /* 400 */ 92, 210, 146, 88, 148, 199, 150, 151, 64, 65, - /* 410 */ 66, 244, 99, 244, 207, 248, 210, 248, 203, 204, - /* 420 */ 244, 135, 244, 244, 248, 85, 248, 248, 67, 68, - /* 430 */ 69, 85, 85, 118, 99, 119, 120, 85, 125, 99, - /* 440 */ 85, 85, 85, 126, 85, 85, 266, 85, 241, 85, - /* 450 */ 84, 99, 85, 85, 99, 99, 99, 62, 99, 99, - /* 460 */ 84, 99, 85, 99, 80, 81, 99, 99, 122, 122, - /* 470 */ 147, 147, 149, 149, 139, 147, 99, 149, 5, 84, - /* 480 */ 7, 199, 5, 117, 7, 147, 147, 149, 149, 147, - /* 490 */ 266, 149, 116, 239, 266, 266, 266, 119, 266, 199, - /* 500 */ 266, 126, 266, 266, 266, 266, 239, 266, 266, 266, - /* 510 */ 266, 266, 266, 266, 249, 239, 239, 239, 239, 239, - /* 520 */ 199, 199, 247, 265, 199, 199, 199, 62, 247, 199, - /* 530 */ 199, 199, 247, 199, 275, 251, 275, 201, 201, 270, - /* 540 */ 199, 91, 199, 270, 199, 255, 259, 199, 199, 199, - /* 550 */ 199, 126, 199, 199, 262, 261, 260, 199, 270, 270, - /* 560 */ 136, 133, 199, 258, 199, 138, 140, 137, 199, 199, - /* 570 */ 199, 199, 199, 199, 131, 257, 130, 129, 199, 199, - /* 580 */ 199, 199, 199, 132, 199, 199, 199, 199, 199, 199, - /* 590 */ 199, 199, 199, 199, 142, 199, 199, 199, 199, 199, - /* 600 */ 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, - /* 610 */ 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, - /* 620 */ 115, 98, 201, 201, 201, 97, 53, 94, 96, 57, - /* 630 */ 95, 93, 86, 5, 201, 201, 201, 155, 5, 5, - /* 640 */ 155, 201, 201, 5, 5, 102, 207, 211, 211, 207, - /* 650 */ 101, 144, 122, 117, 84, 99, 85, 123, 84, 99, - /* 660 */ 201, 201, 201, 218, 220, 216, 219, 214, 202, 213, - /* 670 */ 215, 217, 202, 202, 202, 201, 238, 201, 203, 201, - /* 680 */ 222, 85, 84, 252, 85, 256, 254, 208, 84, 99, - /* 690 */ 85, 84, 1, 85, 84, 238, 84, 134, 134, 99, - /* 700 */ 99, 84, 84, 117, 84, 118, 80, 72, 89, 88, - /* 710 */ 5, 89, 88, 9, 5, 5, 5, 5, 5, 5, - /* 720 */ 5, 15, 80, 87, 84, 119, 85, 84, 26, 61, - /* 730 */ 149, 149, 16, 16, 149, 5, 99, 149, 5, 85, - /* 740 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 750 */ 5, 5, 5, 5, 5, 99, 87, 62, 0, 278, - /* 760 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 770 */ 278, 21, 21, 278, 278, 278, 278, 278, 278, 278, - /* 780 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 790 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 800 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 810 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 820 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 830 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 840 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 850 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 860 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 870 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 880 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 890 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 900 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 910 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 920 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 930 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 940 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 950 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 960 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, -}; -#define YY_SHIFT_COUNT (367) -#define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (758) -static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 186, 113, 113, 260, 260, 98, 250, 266, 266, 193, - /* 10 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - /* 20 */ 23, 23, 23, 0, 122, 266, 316, 316, 316, 9, - /* 30 */ 9, 23, 23, 18, 23, 78, 23, 23, 23, 23, - /* 40 */ 308, 98, 112, 112, 61, 773, 773, 773, 266, 266, - /* 50 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, - /* 60 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, - /* 70 */ 316, 316, 316, 2, 2, 2, 2, 2, 2, 2, - /* 80 */ 23, 23, 23, 64, 23, 23, 23, 9, 9, 23, - /* 90 */ 23, 23, 23, 286, 286, 313, 9, 23, 23, 23, - /* 100 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - /* 110 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - /* 120 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - /* 130 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - /* 140 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - /* 150 */ 23, 23, 23, 23, 23, 23, 465, 465, 465, 425, - /* 160 */ 425, 425, 425, 465, 465, 427, 426, 428, 430, 424, - /* 170 */ 443, 446, 448, 451, 452, 465, 465, 465, 450, 450, - /* 180 */ 505, 98, 98, 465, 465, 523, 528, 573, 533, 532, - /* 190 */ 572, 535, 538, 505, 61, 465, 465, 546, 546, 465, - /* 200 */ 546, 465, 546, 465, 465, 773, 773, 29, 69, 69, - /* 210 */ 99, 69, 127, 170, 240, 252, 252, 252, 252, 252, - /* 220 */ 252, 271, 283, 40, 40, 40, 40, 233, 256, 156, - /* 230 */ 315, 257, 257, 317, 375, 361, 344, 340, 346, 347, - /* 240 */ 352, 355, 356, 218, 335, 357, 359, 360, 362, 364, - /* 250 */ 366, 367, 368, 227, 395, 378, 377, 323, 324, 328, - /* 260 */ 473, 477, 338, 339, 376, 342, 384, 628, 482, 633, - /* 270 */ 634, 485, 638, 639, 543, 549, 507, 530, 536, 570, - /* 280 */ 534, 571, 574, 556, 560, 596, 598, 599, 604, 605, - /* 290 */ 590, 607, 608, 610, 691, 612, 600, 563, 601, 564, - /* 300 */ 617, 536, 618, 586, 620, 587, 626, 619, 621, 635, - /* 310 */ 705, 622, 624, 704, 709, 710, 711, 712, 713, 714, - /* 320 */ 715, 636, 706, 642, 640, 641, 606, 643, 702, 668, - /* 330 */ 716, 581, 582, 637, 637, 637, 637, 717, 585, 588, - /* 340 */ 637, 637, 637, 730, 733, 654, 637, 735, 736, 737, - /* 350 */ 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, - /* 360 */ 748, 749, 656, 669, 750, 751, 695, 758, -}; -#define YY_REDUCE_COUNT (206) -#define YY_REDUCE_MIN (-266) -#define YY_REDUCE_MAX (479) -static const short yy_reduce_ofst[] = { - /* 0 */ -186, 10, 10, -163, -163, 154, -159, -5, 42, -168, - /* 10 */ -112, -115, 117, -41, 30, 51, 84, 95, 167, 169, - /* 20 */ 176, 178, 179, -195, -197, -258, -239, -201, -143, -229, - /* 30 */ -130, -62, 59, -238, -57, 47, 188, 191, 206, -88, - /* 40 */ 48, 207, 115, 160, 119, -99, 22, 215, -266, -261, - /* 50 */ -213, -58, 180, 224, 228, 229, 230, 232, 234, 236, - /* 60 */ 237, 238, 239, 241, 242, 243, 244, 245, 246, 247, - /* 70 */ 27, 35, 265, 254, 267, 276, 277, 278, 279, 280, - /* 80 */ 282, 300, 321, 258, 322, 325, 326, 275, 281, 327, - /* 90 */ 330, 331, 332, 259, 261, 284, 285, 334, 341, 343, - /* 100 */ 345, 348, 349, 350, 351, 353, 354, 358, 363, 365, - /* 110 */ 369, 370, 371, 372, 373, 374, 379, 380, 381, 382, - /* 120 */ 383, 385, 386, 387, 388, 389, 390, 391, 392, 393, - /* 130 */ 394, 396, 397, 398, 399, 400, 401, 402, 403, 404, - /* 140 */ 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - /* 150 */ 415, 416, 417, 418, 419, 420, 336, 337, 421, 269, - /* 160 */ 273, 288, 289, 422, 423, 292, 294, 296, 287, 305, - /* 170 */ 318, 429, 290, 432, 431, 433, 434, 435, 436, 437, - /* 180 */ 438, 439, 442, 440, 441, 444, 447, 445, 453, 454, - /* 190 */ 455, 449, 456, 457, 458, 459, 460, 466, 470, 461, - /* 200 */ 471, 474, 472, 476, 478, 479, 475, -}; -static const YYACTIONTYPE yy_default[] = { - /* 0 */ 870, 994, 933, 1004, 920, 930, 1145, 1145, 1145, 870, - /* 10 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 20 */ 870, 870, 870, 1052, 890, 1145, 870, 870, 870, 870, - /* 30 */ 870, 870, 870, 1067, 870, 930, 870, 870, 870, 870, - /* 40 */ 940, 930, 940, 940, 870, 1047, 978, 996, 870, 870, - /* 50 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 60 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 70 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 80 */ 870, 870, 870, 1054, 1060, 1057, 870, 870, 870, 1062, - /* 90 */ 870, 870, 870, 1086, 1086, 1045, 870, 870, 870, 870, - /* 100 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 110 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 120 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 918, - /* 130 */ 870, 916, 870, 870, 870, 870, 870, 870, 870, 870, - /* 140 */ 870, 870, 870, 870, 870, 870, 870, 870, 901, 870, - /* 150 */ 870, 870, 870, 870, 870, 888, 892, 892, 892, 870, - /* 160 */ 870, 870, 870, 892, 892, 1093, 1097, 1079, 1091, 1087, - /* 170 */ 1074, 1072, 1070, 1078, 1101, 892, 892, 892, 938, 938, - /* 180 */ 934, 930, 930, 892, 892, 956, 954, 952, 944, 950, - /* 190 */ 946, 948, 942, 921, 870, 892, 892, 928, 928, 892, - /* 200 */ 928, 892, 928, 892, 892, 978, 996, 870, 1102, 1092, - /* 210 */ 870, 1144, 1132, 1131, 870, 1140, 1139, 1138, 1130, 1129, - /* 220 */ 1128, 870, 870, 1124, 1127, 1126, 1125, 870, 870, 870, - /* 230 */ 870, 1134, 1133, 870, 870, 870, 870, 870, 870, 870, - /* 240 */ 870, 870, 870, 1098, 1094, 870, 870, 870, 870, 870, - /* 250 */ 870, 870, 870, 870, 1104, 870, 870, 870, 870, 870, - /* 260 */ 870, 870, 870, 870, 1006, 870, 870, 870, 870, 870, - /* 270 */ 870, 870, 870, 870, 870, 870, 870, 1044, 870, 870, - /* 280 */ 870, 870, 870, 1056, 1055, 870, 870, 870, 870, 870, - /* 290 */ 870, 870, 870, 870, 870, 870, 1088, 870, 1080, 870, - /* 300 */ 870, 1018, 870, 870, 870, 870, 870, 870, 870, 870, - /* 310 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 320 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 330 */ 870, 870, 870, 1163, 1158, 1159, 1156, 870, 870, 870, - /* 340 */ 1155, 1150, 1151, 870, 870, 870, 1148, 870, 870, 870, - /* 350 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 360 */ 870, 870, 962, 870, 899, 897, 870, 870, -}; -/********** End of lemon-generated parsing tables *****************************/ - -/* The next table maps tokens (terminal symbols) into fallback tokens. -** If a construct like the following: -** -** %fallback ID X Y Z. -** -** appears in the grammar, then ID becomes a fallback token for X, Y, -** and Z. Whenever one of the tokens X, Y, or Z is input to the parser -** but it does not parse, the type of the token is changed to ID and -** the parse is retried before an error is thrown. -** -** This feature can be used, for example, to cause some keywords in a language -** to revert to identifiers if they keyword does not apply in the context where -** it appears. -*/ -#ifdef YYFALLBACK -static const YYCODETYPE yyFallback[] = { - 0, /* $ => nothing */ - 0, /* ID => nothing */ - 1, /* BOOL => ID */ - 1, /* TINYINT => ID */ - 1, /* SMALLINT => ID */ - 1, /* INTEGER => ID */ - 1, /* BIGINT => ID */ - 1, /* FLOAT => ID */ - 1, /* DOUBLE => ID */ - 1, /* STRING => ID */ - 1, /* TIMESTAMP => ID */ - 1, /* BINARY => ID */ - 1, /* NCHAR => ID */ - 0, /* OR => nothing */ - 0, /* AND => nothing */ - 0, /* NOT => nothing */ - 0, /* EQ => nothing */ - 0, /* NE => nothing */ - 0, /* ISNULL => nothing */ - 0, /* NOTNULL => nothing */ - 0, /* IS => nothing */ - 1, /* LIKE => ID */ - 1, /* MATCH => ID */ - 1, /* NMATCH => ID */ - 1, /* GLOB => ID */ - 0, /* BETWEEN => nothing */ - 0, /* IN => nothing */ - 0, /* GT => nothing */ - 0, /* GE => nothing */ - 0, /* LT => nothing */ - 0, /* LE => nothing */ - 0, /* BITAND => nothing */ - 0, /* BITOR => nothing */ - 0, /* LSHIFT => nothing */ - 0, /* RSHIFT => nothing */ - 0, /* PLUS => nothing */ - 0, /* MINUS => nothing */ - 0, /* DIVIDE => nothing */ - 0, /* TIMES => nothing */ - 0, /* STAR => nothing */ - 0, /* SLASH => nothing */ - 0, /* REM => nothing */ - 0, /* CONCAT => nothing */ - 0, /* UMINUS => nothing */ - 0, /* UPLUS => nothing */ - 0, /* BITNOT => nothing */ - 0, /* SHOW => nothing */ - 0, /* DATABASES => nothing */ - 0, /* TOPICS => nothing */ - 0, /* FUNCTIONS => nothing */ - 0, /* MNODES => nothing */ - 0, /* DNODES => nothing */ - 0, /* ACCOUNTS => nothing */ - 0, /* USERS => nothing */ - 0, /* MODULES => nothing */ - 0, /* QUERIES => nothing */ - 0, /* CONNECTIONS => nothing */ - 0, /* STREAMS => nothing */ - 0, /* VARIABLES => nothing */ - 0, /* SCORES => nothing */ - 0, /* GRANTS => nothing */ - 0, /* VNODES => nothing */ - 0, /* DOT => nothing */ - 0, /* CREATE => nothing */ - 0, /* TABLE => nothing */ - 1, /* STABLE => ID */ - 1, /* DATABASE => ID */ - 0, /* TABLES => nothing */ - 0, /* STABLES => nothing */ - 0, /* VGROUPS => nothing */ - 0, /* DROP => nothing */ - 0, /* TOPIC => nothing */ - 0, /* FUNCTION => nothing */ - 0, /* DNODE => nothing */ - 0, /* USER => nothing */ - 0, /* ACCOUNT => nothing */ - 0, /* USE => nothing */ - 0, /* DESCRIBE => nothing */ - 1, /* DESC => ID */ - 0, /* ALTER => nothing */ - 0, /* PASS => nothing */ - 0, /* PRIVILEGE => nothing */ - 0, /* LOCAL => nothing */ - 0, /* COMPACT => nothing */ - 0, /* LP => nothing */ - 0, /* RP => nothing */ - 0, /* IF => nothing */ - 0, /* EXISTS => nothing */ - 0, /* AS => nothing */ - 0, /* OUTPUTTYPE => nothing */ - 0, /* AGGREGATE => nothing */ - 0, /* BUFSIZE => nothing */ - 0, /* PPS => nothing */ - 0, /* TSERIES => nothing */ - 0, /* DBS => nothing */ - 0, /* STORAGE => nothing */ - 0, /* QTIME => nothing */ - 0, /* CONNS => nothing */ - 0, /* STATE => nothing */ - 0, /* COMMA => nothing */ - 0, /* KEEP => nothing */ - 0, /* CACHE => nothing */ - 0, /* REPLICA => nothing */ - 0, /* QUORUM => nothing */ - 0, /* DAYS => nothing */ - 0, /* MINROWS => nothing */ - 0, /* MAXROWS => nothing */ - 0, /* BLOCKS => nothing */ - 0, /* CTIME => nothing */ - 0, /* WAL => nothing */ - 0, /* FSYNC => nothing */ - 0, /* COMP => nothing */ - 0, /* PRECISION => nothing */ - 0, /* UPDATE => nothing */ - 0, /* CACHELAST => nothing */ - 0, /* PARTITIONS => nothing */ - 0, /* UNSIGNED => nothing */ - 0, /* TAGS => nothing */ - 0, /* USING => nothing */ - 1, /* NULL => ID */ - 1, /* NOW => ID */ - 0, /* SELECT => nothing */ - 0, /* UNION => nothing */ - 1, /* ALL => ID */ - 0, /* DISTINCT => nothing */ - 0, /* FROM => nothing */ - 0, /* VARIABLE => nothing */ - 0, /* INTERVAL => nothing */ - 0, /* EVERY => nothing */ - 0, /* SESSION => nothing */ - 0, /* STATE_WINDOW => nothing */ - 0, /* FILL => nothing */ - 0, /* SLIDING => nothing */ - 0, /* ORDER => nothing */ - 0, /* BY => nothing */ - 1, /* ASC => ID */ - 0, /* GROUP => nothing */ - 0, /* HAVING => nothing */ - 0, /* LIMIT => nothing */ - 1, /* OFFSET => ID */ - 0, /* SLIMIT => nothing */ - 0, /* SOFFSET => nothing */ - 0, /* WHERE => nothing */ - 0, /* RESET => nothing */ - 0, /* QUERY => nothing */ - 0, /* SYNCDB => nothing */ - 0, /* ADD => nothing */ - 0, /* COLUMN => nothing */ - 0, /* MODIFY => nothing */ - 0, /* TAG => nothing */ - 0, /* CHANGE => nothing */ - 0, /* SET => nothing */ - 0, /* KILL => nothing */ - 0, /* CONNECTION => nothing */ - 0, /* STREAM => nothing */ - 0, /* COLON => nothing */ - 1, /* ABORT => ID */ - 1, /* AFTER => ID */ - 1, /* ATTACH => ID */ - 1, /* BEFORE => ID */ - 1, /* BEGIN => ID */ - 1, /* CASCADE => ID */ - 1, /* CLUSTER => ID */ - 1, /* CONFLICT => ID */ - 1, /* COPY => ID */ - 1, /* DEFERRED => ID */ - 1, /* DELIMITERS => ID */ - 1, /* DETACH => ID */ - 1, /* EACH => ID */ - 1, /* END => ID */ - 1, /* EXPLAIN => ID */ - 1, /* FAIL => ID */ - 1, /* FOR => ID */ - 1, /* IGNORE => ID */ - 1, /* IMMEDIATE => ID */ - 1, /* INITIALLY => ID */ - 1, /* INSTEAD => ID */ - 1, /* KEY => ID */ - 1, /* OF => ID */ - 1, /* RAISE => ID */ - 1, /* REPLACE => ID */ - 1, /* RESTRICT => ID */ - 1, /* ROW => ID */ - 1, /* STATEMENT => ID */ - 1, /* TRIGGER => ID */ - 1, /* VIEW => ID */ - 1, /* IPTOKEN => ID */ - 1, /* SEMI => ID */ - 1, /* NONE => ID */ - 1, /* PREV => ID */ - 1, /* LINEAR => ID */ - 1, /* IMPORT => ID */ - 1, /* TBNAME => ID */ - 1, /* JOIN => ID */ - 1, /* INSERT => ID */ - 1, /* INTO => ID */ - 1, /* VALUES => ID */ -}; -#endif /* YYFALLBACK */ - -/* The following structure represents a single element of the -** parser's stack. Information stored includes: -** -** + The state number for the parser at this level of the stack. -** -** + The value of the token stored at this level of the stack. -** (In other words, the "major" token.) -** -** + The semantic value stored at this level of the stack. This is -** the information used by the action routines in the grammar. -** It is sometimes called the "minor" token. -** -** After the "shift" half of a SHIFTREDUCE action, the stateno field -** actually contains the reduce action for the second half of the -** SHIFTREDUCE. -*/ -struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ - YYCODETYPE major; /* The major token value. This is the code - ** number for the token at this stack level */ - YYMINORTYPE minor; /* The user-supplied minor token value. This - ** is the value of the token */ -}; -typedef struct yyStackEntry yyStackEntry; - -/* The state of the parser is completely contained in an instance of -** the following structure */ -struct yyParser { - yyStackEntry *yytos; /* Pointer to top element of the stack */ -#ifdef YYTRACKMAXSTACKDEPTH - int yyhwm; /* High-water mark of the stack */ -#endif -#ifndef YYNOERRORRECOVERY - int yyerrcnt; /* Shifts left before out of the error */ -#endif - ParseARG_SDECL /* A place to hold %extra_argument */ - ParseCTX_SDECL /* A place to hold %extra_context */ -#if YYSTACKDEPTH<=0 - int yystksz; /* Current side of the stack */ - yyStackEntry *yystack; /* The parser's stack */ - yyStackEntry yystk0; /* First stack entry */ -#else - yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ - yyStackEntry *yystackEnd; /* Last entry in the stack */ -#endif -}; -typedef struct yyParser yyParser; - -#ifndef NDEBUG -#include -static FILE *yyTraceFILE = 0; -static char *yyTracePrompt = 0; -#endif /* NDEBUG */ - -#ifndef NDEBUG -/* -** Turn parser tracing on by giving a stream to which to write the trace -** and a prompt to preface each trace message. Tracing is turned off -** by making either argument NULL -** -** Inputs: -**
    -**
  • A FILE* to which trace output should be written. -** If NULL, then tracing is turned off. -**
  • A prefix string written at the beginning of every -** line of trace output. If NULL, then tracing is -** turned off. -**
-** -** Outputs: -** None. -*/ -void ParseTrace(FILE *TraceFILE, char *zTracePrompt){ - yyTraceFILE = TraceFILE; - yyTracePrompt = zTracePrompt; - if( yyTraceFILE==0 ) yyTracePrompt = 0; - else if( yyTracePrompt==0 ) yyTraceFILE = 0; -} -#endif /* NDEBUG */ - -#if defined(YYCOVERAGE) || !defined(NDEBUG) -/* For tracing shifts, the names of all terminals and nonterminals -** are required. The following table supplies these names */ -static const char *const yyTokenName[] = { - /* 0 */ "$", - /* 1 */ "ID", - /* 2 */ "BOOL", - /* 3 */ "TINYINT", - /* 4 */ "SMALLINT", - /* 5 */ "INTEGER", - /* 6 */ "BIGINT", - /* 7 */ "FLOAT", - /* 8 */ "DOUBLE", - /* 9 */ "STRING", - /* 10 */ "TIMESTAMP", - /* 11 */ "BINARY", - /* 12 */ "NCHAR", - /* 13 */ "OR", - /* 14 */ "AND", - /* 15 */ "NOT", - /* 16 */ "EQ", - /* 17 */ "NE", - /* 18 */ "ISNULL", - /* 19 */ "NOTNULL", - /* 20 */ "IS", - /* 21 */ "LIKE", - /* 22 */ "MATCH", - /* 23 */ "NMATCH", - /* 24 */ "GLOB", - /* 25 */ "BETWEEN", - /* 26 */ "IN", - /* 27 */ "GT", - /* 28 */ "GE", - /* 29 */ "LT", - /* 30 */ "LE", - /* 31 */ "BITAND", - /* 32 */ "BITOR", - /* 33 */ "LSHIFT", - /* 34 */ "RSHIFT", - /* 35 */ "PLUS", - /* 36 */ "MINUS", - /* 37 */ "DIVIDE", - /* 38 */ "TIMES", - /* 39 */ "STAR", - /* 40 */ "SLASH", - /* 41 */ "REM", - /* 42 */ "CONCAT", - /* 43 */ "UMINUS", - /* 44 */ "UPLUS", - /* 45 */ "BITNOT", - /* 46 */ "SHOW", - /* 47 */ "DATABASES", - /* 48 */ "TOPICS", - /* 49 */ "FUNCTIONS", - /* 50 */ "MNODES", - /* 51 */ "DNODES", - /* 52 */ "ACCOUNTS", - /* 53 */ "USERS", - /* 54 */ "MODULES", - /* 55 */ "QUERIES", - /* 56 */ "CONNECTIONS", - /* 57 */ "STREAMS", - /* 58 */ "VARIABLES", - /* 59 */ "SCORES", - /* 60 */ "GRANTS", - /* 61 */ "VNODES", - /* 62 */ "DOT", - /* 63 */ "CREATE", - /* 64 */ "TABLE", - /* 65 */ "STABLE", - /* 66 */ "DATABASE", - /* 67 */ "TABLES", - /* 68 */ "STABLES", - /* 69 */ "VGROUPS", - /* 70 */ "DROP", - /* 71 */ "TOPIC", - /* 72 */ "FUNCTION", - /* 73 */ "DNODE", - /* 74 */ "USER", - /* 75 */ "ACCOUNT", - /* 76 */ "USE", - /* 77 */ "DESCRIBE", - /* 78 */ "DESC", - /* 79 */ "ALTER", - /* 80 */ "PASS", - /* 81 */ "PRIVILEGE", - /* 82 */ "LOCAL", - /* 83 */ "COMPACT", - /* 84 */ "LP", - /* 85 */ "RP", - /* 86 */ "IF", - /* 87 */ "EXISTS", - /* 88 */ "AS", - /* 89 */ "OUTPUTTYPE", - /* 90 */ "AGGREGATE", - /* 91 */ "BUFSIZE", - /* 92 */ "PPS", - /* 93 */ "TSERIES", - /* 94 */ "DBS", - /* 95 */ "STORAGE", - /* 96 */ "QTIME", - /* 97 */ "CONNS", - /* 98 */ "STATE", - /* 99 */ "COMMA", - /* 100 */ "KEEP", - /* 101 */ "CACHE", - /* 102 */ "REPLICA", - /* 103 */ "QUORUM", - /* 104 */ "DAYS", - /* 105 */ "MINROWS", - /* 106 */ "MAXROWS", - /* 107 */ "BLOCKS", - /* 108 */ "CTIME", - /* 109 */ "WAL", - /* 110 */ "FSYNC", - /* 111 */ "COMP", - /* 112 */ "PRECISION", - /* 113 */ "UPDATE", - /* 114 */ "CACHELAST", - /* 115 */ "PARTITIONS", - /* 116 */ "UNSIGNED", - /* 117 */ "TAGS", - /* 118 */ "USING", - /* 119 */ "NULL", - /* 120 */ "NOW", - /* 121 */ "SELECT", - /* 122 */ "UNION", - /* 123 */ "ALL", - /* 124 */ "DISTINCT", - /* 125 */ "FROM", - /* 126 */ "VARIABLE", - /* 127 */ "INTERVAL", - /* 128 */ "EVERY", - /* 129 */ "SESSION", - /* 130 */ "STATE_WINDOW", - /* 131 */ "FILL", - /* 132 */ "SLIDING", - /* 133 */ "ORDER", - /* 134 */ "BY", - /* 135 */ "ASC", - /* 136 */ "GROUP", - /* 137 */ "HAVING", - /* 138 */ "LIMIT", - /* 139 */ "OFFSET", - /* 140 */ "SLIMIT", - /* 141 */ "SOFFSET", - /* 142 */ "WHERE", - /* 143 */ "RESET", - /* 144 */ "QUERY", - /* 145 */ "SYNCDB", - /* 146 */ "ADD", - /* 147 */ "COLUMN", - /* 148 */ "MODIFY", - /* 149 */ "TAG", - /* 150 */ "CHANGE", - /* 151 */ "SET", - /* 152 */ "KILL", - /* 153 */ "CONNECTION", - /* 154 */ "STREAM", - /* 155 */ "COLON", - /* 156 */ "ABORT", - /* 157 */ "AFTER", - /* 158 */ "ATTACH", - /* 159 */ "BEFORE", - /* 160 */ "BEGIN", - /* 161 */ "CASCADE", - /* 162 */ "CLUSTER", - /* 163 */ "CONFLICT", - /* 164 */ "COPY", - /* 165 */ "DEFERRED", - /* 166 */ "DELIMITERS", - /* 167 */ "DETACH", - /* 168 */ "EACH", - /* 169 */ "END", - /* 170 */ "EXPLAIN", - /* 171 */ "FAIL", - /* 172 */ "FOR", - /* 173 */ "IGNORE", - /* 174 */ "IMMEDIATE", - /* 175 */ "INITIALLY", - /* 176 */ "INSTEAD", - /* 177 */ "KEY", - /* 178 */ "OF", - /* 179 */ "RAISE", - /* 180 */ "REPLACE", - /* 181 */ "RESTRICT", - /* 182 */ "ROW", - /* 183 */ "STATEMENT", - /* 184 */ "TRIGGER", - /* 185 */ "VIEW", - /* 186 */ "IPTOKEN", - /* 187 */ "SEMI", - /* 188 */ "NONE", - /* 189 */ "PREV", - /* 190 */ "LINEAR", - /* 191 */ "IMPORT", - /* 192 */ "TBNAME", - /* 193 */ "JOIN", - /* 194 */ "INSERT", - /* 195 */ "INTO", - /* 196 */ "VALUES", - /* 197 */ "program", - /* 198 */ "cmd", - /* 199 */ "ids", - /* 200 */ "dbPrefix", - /* 201 */ "cpxName", - /* 202 */ "ifexists", - /* 203 */ "alter_db_optr", - /* 204 */ "alter_topic_optr", - /* 205 */ "acct_optr", - /* 206 */ "exprlist", - /* 207 */ "ifnotexists", - /* 208 */ "db_optr", - /* 209 */ "topic_optr", - /* 210 */ "typename", - /* 211 */ "bufsize", - /* 212 */ "pps", - /* 213 */ "tseries", - /* 214 */ "dbs", - /* 215 */ "streams", - /* 216 */ "storage", - /* 217 */ "qtime", - /* 218 */ "users", - /* 219 */ "conns", - /* 220 */ "state", - /* 221 */ "intitemlist", - /* 222 */ "intitem", - /* 223 */ "keep", - /* 224 */ "cache", - /* 225 */ "replica", - /* 226 */ "quorum", - /* 227 */ "days", - /* 228 */ "minrows", - /* 229 */ "maxrows", - /* 230 */ "blocks", - /* 231 */ "ctime", - /* 232 */ "wal", - /* 233 */ "fsync", - /* 234 */ "comp", - /* 235 */ "prec", - /* 236 */ "update", - /* 237 */ "cachelast", - /* 238 */ "partitions", - /* 239 */ "signed", - /* 240 */ "create_table_args", - /* 241 */ "create_stable_args", - /* 242 */ "create_table_list", - /* 243 */ "create_from_stable", - /* 244 */ "columnlist", - /* 245 */ "tagitemlist", - /* 246 */ "tagNamelist", - /* 247 */ "select", - /* 248 */ "column", - /* 249 */ "tagitem", - /* 250 */ "selcollist", - /* 251 */ "from", - /* 252 */ "where_opt", - /* 253 */ "interval_option", - /* 254 */ "sliding_opt", - /* 255 */ "session_option", - /* 256 */ "windowstate_option", - /* 257 */ "fill_opt", - /* 258 */ "groupby_opt", - /* 259 */ "having_opt", - /* 260 */ "orderby_opt", - /* 261 */ "slimit_opt", - /* 262 */ "limit_opt", - /* 263 */ "union", - /* 264 */ "sclp", - /* 265 */ "distinct", - /* 266 */ "expr", - /* 267 */ "as", - /* 268 */ "tablelist", - /* 269 */ "sub", - /* 270 */ "tmvar", - /* 271 */ "intervalKey", - /* 272 */ "sortlist", - /* 273 */ "sortitem", - /* 274 */ "item", - /* 275 */ "sortorder", - /* 276 */ "grouplist", - /* 277 */ "expritem", -}; -#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ - -#ifndef NDEBUG -/* For tracing reduce actions, the names of all rules are required. -*/ -static const char *const yyRuleName[] = { - /* 0 */ "program ::= cmd", - /* 1 */ "cmd ::= SHOW DATABASES", - /* 2 */ "cmd ::= SHOW TOPICS", - /* 3 */ "cmd ::= SHOW FUNCTIONS", - /* 4 */ "cmd ::= SHOW MNODES", - /* 5 */ "cmd ::= SHOW DNODES", - /* 6 */ "cmd ::= SHOW ACCOUNTS", - /* 7 */ "cmd ::= SHOW USERS", - /* 8 */ "cmd ::= SHOW MODULES", - /* 9 */ "cmd ::= SHOW QUERIES", - /* 10 */ "cmd ::= SHOW CONNECTIONS", - /* 11 */ "cmd ::= SHOW STREAMS", - /* 12 */ "cmd ::= SHOW VARIABLES", - /* 13 */ "cmd ::= SHOW SCORES", - /* 14 */ "cmd ::= SHOW GRANTS", - /* 15 */ "cmd ::= SHOW VNODES", - /* 16 */ "cmd ::= SHOW VNODES ids", - /* 17 */ "dbPrefix ::=", - /* 18 */ "dbPrefix ::= ids DOT", - /* 19 */ "cpxName ::=", - /* 20 */ "cpxName ::= DOT ids", - /* 21 */ "cmd ::= SHOW CREATE TABLE ids cpxName", - /* 22 */ "cmd ::= SHOW CREATE STABLE ids cpxName", - /* 23 */ "cmd ::= SHOW CREATE DATABASE ids", - /* 24 */ "cmd ::= SHOW dbPrefix TABLES", - /* 25 */ "cmd ::= SHOW dbPrefix TABLES LIKE ids", - /* 26 */ "cmd ::= SHOW dbPrefix STABLES", - /* 27 */ "cmd ::= SHOW dbPrefix STABLES LIKE ids", - /* 28 */ "cmd ::= SHOW dbPrefix VGROUPS", - /* 29 */ "cmd ::= SHOW dbPrefix VGROUPS ids", - /* 30 */ "cmd ::= DROP TABLE ifexists ids cpxName", - /* 31 */ "cmd ::= DROP STABLE ifexists ids cpxName", - /* 32 */ "cmd ::= DROP DATABASE ifexists ids", - /* 33 */ "cmd ::= DROP TOPIC ifexists ids", - /* 34 */ "cmd ::= DROP FUNCTION ids", - /* 35 */ "cmd ::= DROP DNODE ids", - /* 36 */ "cmd ::= DROP USER ids", - /* 37 */ "cmd ::= DROP ACCOUNT ids", - /* 38 */ "cmd ::= USE ids", - /* 39 */ "cmd ::= DESCRIBE ids cpxName", - /* 40 */ "cmd ::= DESC ids cpxName", - /* 41 */ "cmd ::= ALTER USER ids PASS ids", - /* 42 */ "cmd ::= ALTER USER ids PRIVILEGE ids", - /* 43 */ "cmd ::= ALTER DNODE ids ids", - /* 44 */ "cmd ::= ALTER DNODE ids ids ids", - /* 45 */ "cmd ::= ALTER LOCAL ids", - /* 46 */ "cmd ::= ALTER LOCAL ids ids", - /* 47 */ "cmd ::= ALTER DATABASE ids alter_db_optr", - /* 48 */ "cmd ::= ALTER TOPIC ids alter_topic_optr", - /* 49 */ "cmd ::= ALTER ACCOUNT ids acct_optr", - /* 50 */ "cmd ::= ALTER ACCOUNT ids PASS ids acct_optr", - /* 51 */ "cmd ::= COMPACT VNODES IN LP exprlist RP", - /* 52 */ "ids ::= ID", - /* 53 */ "ids ::= STRING", - /* 54 */ "ifexists ::= IF EXISTS", - /* 55 */ "ifexists ::=", - /* 56 */ "ifnotexists ::= IF NOT EXISTS", - /* 57 */ "ifnotexists ::=", - /* 58 */ "cmd ::= CREATE DNODE ids", - /* 59 */ "cmd ::= CREATE ACCOUNT ids PASS ids acct_optr", - /* 60 */ "cmd ::= CREATE DATABASE ifnotexists ids db_optr", - /* 61 */ "cmd ::= CREATE TOPIC ifnotexists ids topic_optr", - /* 62 */ "cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize", - /* 63 */ "cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize", - /* 64 */ "cmd ::= CREATE USER ids PASS ids", - /* 65 */ "bufsize ::=", - /* 66 */ "bufsize ::= BUFSIZE INTEGER", - /* 67 */ "pps ::=", - /* 68 */ "pps ::= PPS INTEGER", - /* 69 */ "tseries ::=", - /* 70 */ "tseries ::= TSERIES INTEGER", - /* 71 */ "dbs ::=", - /* 72 */ "dbs ::= DBS INTEGER", - /* 73 */ "streams ::=", - /* 74 */ "streams ::= STREAMS INTEGER", - /* 75 */ "storage ::=", - /* 76 */ "storage ::= STORAGE INTEGER", - /* 77 */ "qtime ::=", - /* 78 */ "qtime ::= QTIME INTEGER", - /* 79 */ "users ::=", - /* 80 */ "users ::= USERS INTEGER", - /* 81 */ "conns ::=", - /* 82 */ "conns ::= CONNS INTEGER", - /* 83 */ "state ::=", - /* 84 */ "state ::= STATE ids", - /* 85 */ "acct_optr ::= pps tseries storage streams qtime dbs users conns state", - /* 86 */ "intitemlist ::= intitemlist COMMA intitem", - /* 87 */ "intitemlist ::= intitem", - /* 88 */ "intitem ::= INTEGER", - /* 89 */ "keep ::= KEEP intitemlist", - /* 90 */ "cache ::= CACHE INTEGER", - /* 91 */ "replica ::= REPLICA INTEGER", - /* 92 */ "quorum ::= QUORUM INTEGER", - /* 93 */ "days ::= DAYS INTEGER", - /* 94 */ "minrows ::= MINROWS INTEGER", - /* 95 */ "maxrows ::= MAXROWS INTEGER", - /* 96 */ "blocks ::= BLOCKS INTEGER", - /* 97 */ "ctime ::= CTIME INTEGER", - /* 98 */ "wal ::= WAL INTEGER", - /* 99 */ "fsync ::= FSYNC INTEGER", - /* 100 */ "comp ::= COMP INTEGER", - /* 101 */ "prec ::= PRECISION STRING", - /* 102 */ "update ::= UPDATE INTEGER", - /* 103 */ "cachelast ::= CACHELAST INTEGER", - /* 104 */ "partitions ::= PARTITIONS INTEGER", - /* 105 */ "db_optr ::=", - /* 106 */ "db_optr ::= db_optr cache", - /* 107 */ "db_optr ::= db_optr replica", - /* 108 */ "db_optr ::= db_optr quorum", - /* 109 */ "db_optr ::= db_optr days", - /* 110 */ "db_optr ::= db_optr minrows", - /* 111 */ "db_optr ::= db_optr maxrows", - /* 112 */ "db_optr ::= db_optr blocks", - /* 113 */ "db_optr ::= db_optr ctime", - /* 114 */ "db_optr ::= db_optr wal", - /* 115 */ "db_optr ::= db_optr fsync", - /* 116 */ "db_optr ::= db_optr comp", - /* 117 */ "db_optr ::= db_optr prec", - /* 118 */ "db_optr ::= db_optr keep", - /* 119 */ "db_optr ::= db_optr update", - /* 120 */ "db_optr ::= db_optr cachelast", - /* 121 */ "topic_optr ::= db_optr", - /* 122 */ "topic_optr ::= topic_optr partitions", - /* 123 */ "alter_db_optr ::=", - /* 124 */ "alter_db_optr ::= alter_db_optr replica", - /* 125 */ "alter_db_optr ::= alter_db_optr quorum", - /* 126 */ "alter_db_optr ::= alter_db_optr keep", - /* 127 */ "alter_db_optr ::= alter_db_optr blocks", - /* 128 */ "alter_db_optr ::= alter_db_optr comp", - /* 129 */ "alter_db_optr ::= alter_db_optr update", - /* 130 */ "alter_db_optr ::= alter_db_optr cachelast", - /* 131 */ "alter_topic_optr ::= alter_db_optr", - /* 132 */ "alter_topic_optr ::= alter_topic_optr partitions", - /* 133 */ "typename ::= ids", - /* 134 */ "typename ::= ids LP signed RP", - /* 135 */ "typename ::= ids UNSIGNED", - /* 136 */ "signed ::= INTEGER", - /* 137 */ "signed ::= PLUS INTEGER", - /* 138 */ "signed ::= MINUS INTEGER", - /* 139 */ "cmd ::= CREATE TABLE create_table_args", - /* 140 */ "cmd ::= CREATE TABLE create_stable_args", - /* 141 */ "cmd ::= CREATE STABLE create_stable_args", - /* 142 */ "cmd ::= CREATE TABLE create_table_list", - /* 143 */ "create_table_list ::= create_from_stable", - /* 144 */ "create_table_list ::= create_table_list create_from_stable", - /* 145 */ "create_table_args ::= ifnotexists ids cpxName LP columnlist RP", - /* 146 */ "create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP", - /* 147 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP", - /* 148 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP", - /* 149 */ "tagNamelist ::= tagNamelist COMMA ids", - /* 150 */ "tagNamelist ::= ids", - /* 151 */ "create_table_args ::= ifnotexists ids cpxName AS select", - /* 152 */ "columnlist ::= columnlist COMMA column", - /* 153 */ "columnlist ::= column", - /* 154 */ "column ::= ids typename", - /* 155 */ "tagitemlist ::= tagitemlist COMMA tagitem", - /* 156 */ "tagitemlist ::= tagitem", - /* 157 */ "tagitem ::= INTEGER", - /* 158 */ "tagitem ::= FLOAT", - /* 159 */ "tagitem ::= STRING", - /* 160 */ "tagitem ::= BOOL", - /* 161 */ "tagitem ::= NULL", - /* 162 */ "tagitem ::= NOW", - /* 163 */ "tagitem ::= MINUS INTEGER", - /* 164 */ "tagitem ::= MINUS FLOAT", - /* 165 */ "tagitem ::= PLUS INTEGER", - /* 166 */ "tagitem ::= PLUS FLOAT", - /* 167 */ "select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt", - /* 168 */ "select ::= LP select RP", - /* 169 */ "union ::= select", - /* 170 */ "union ::= union UNION ALL select", - /* 171 */ "cmd ::= union", - /* 172 */ "select ::= SELECT selcollist", - /* 173 */ "sclp ::= selcollist COMMA", - /* 174 */ "sclp ::=", - /* 175 */ "selcollist ::= sclp distinct expr as", - /* 176 */ "selcollist ::= sclp STAR", - /* 177 */ "as ::= AS ids", - /* 178 */ "as ::= ids", - /* 179 */ "as ::=", - /* 180 */ "distinct ::= DISTINCT", - /* 181 */ "distinct ::=", - /* 182 */ "from ::= FROM tablelist", - /* 183 */ "from ::= FROM sub", - /* 184 */ "sub ::= LP union RP", - /* 185 */ "sub ::= LP union RP ids", - /* 186 */ "sub ::= sub COMMA LP union RP ids", - /* 187 */ "tablelist ::= ids cpxName", - /* 188 */ "tablelist ::= ids cpxName ids", - /* 189 */ "tablelist ::= tablelist COMMA ids cpxName", - /* 190 */ "tablelist ::= tablelist COMMA ids cpxName ids", - /* 191 */ "tmvar ::= VARIABLE", - /* 192 */ "interval_option ::= intervalKey LP tmvar RP", - /* 193 */ "interval_option ::= intervalKey LP tmvar COMMA tmvar RP", - /* 194 */ "interval_option ::=", - /* 195 */ "intervalKey ::= INTERVAL", - /* 196 */ "intervalKey ::= EVERY", - /* 197 */ "session_option ::=", - /* 198 */ "session_option ::= SESSION LP ids cpxName COMMA tmvar RP", - /* 199 */ "windowstate_option ::=", - /* 200 */ "windowstate_option ::= STATE_WINDOW LP ids RP", - /* 201 */ "fill_opt ::=", - /* 202 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 203 */ "fill_opt ::= FILL LP ID RP", - /* 204 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 205 */ "sliding_opt ::=", - /* 206 */ "orderby_opt ::=", - /* 207 */ "orderby_opt ::= ORDER BY sortlist", - /* 208 */ "sortlist ::= sortlist COMMA item sortorder", - /* 209 */ "sortlist ::= item sortorder", - /* 210 */ "item ::= ids cpxName", - /* 211 */ "sortorder ::= ASC", - /* 212 */ "sortorder ::= DESC", - /* 213 */ "sortorder ::=", - /* 214 */ "groupby_opt ::=", - /* 215 */ "groupby_opt ::= GROUP BY grouplist", - /* 216 */ "grouplist ::= grouplist COMMA item", - /* 217 */ "grouplist ::= item", - /* 218 */ "having_opt ::=", - /* 219 */ "having_opt ::= HAVING expr", - /* 220 */ "limit_opt ::=", - /* 221 */ "limit_opt ::= LIMIT signed", - /* 222 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 223 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 224 */ "slimit_opt ::=", - /* 225 */ "slimit_opt ::= SLIMIT signed", - /* 226 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 227 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 228 */ "where_opt ::=", - /* 229 */ "where_opt ::= WHERE expr", - /* 230 */ "expr ::= LP expr RP", - /* 231 */ "expr ::= ID", - /* 232 */ "expr ::= ID DOT ID", - /* 233 */ "expr ::= ID DOT STAR", - /* 234 */ "expr ::= INTEGER", - /* 235 */ "expr ::= MINUS INTEGER", - /* 236 */ "expr ::= PLUS INTEGER", - /* 237 */ "expr ::= FLOAT", - /* 238 */ "expr ::= MINUS FLOAT", - /* 239 */ "expr ::= PLUS FLOAT", - /* 240 */ "expr ::= STRING", - /* 241 */ "expr ::= NOW", - /* 242 */ "expr ::= VARIABLE", - /* 243 */ "expr ::= PLUS VARIABLE", - /* 244 */ "expr ::= MINUS VARIABLE", - /* 245 */ "expr ::= BOOL", - /* 246 */ "expr ::= NULL", - /* 247 */ "expr ::= ID LP exprlist RP", - /* 248 */ "expr ::= ID LP STAR RP", - /* 249 */ "expr ::= expr IS NULL", - /* 250 */ "expr ::= expr IS NOT NULL", - /* 251 */ "expr ::= expr LT expr", - /* 252 */ "expr ::= expr GT expr", - /* 253 */ "expr ::= expr LE expr", - /* 254 */ "expr ::= expr GE expr", - /* 255 */ "expr ::= expr NE expr", - /* 256 */ "expr ::= expr EQ expr", - /* 257 */ "expr ::= expr BETWEEN expr AND expr", - /* 258 */ "expr ::= expr AND expr", - /* 259 */ "expr ::= expr OR expr", - /* 260 */ "expr ::= expr PLUS expr", - /* 261 */ "expr ::= expr MINUS expr", - /* 262 */ "expr ::= expr STAR expr", - /* 263 */ "expr ::= expr SLASH expr", - /* 264 */ "expr ::= expr REM expr", - /* 265 */ "expr ::= expr LIKE expr", - /* 266 */ "expr ::= expr MATCH expr", - /* 267 */ "expr ::= expr NMATCH expr", - /* 268 */ "expr ::= expr IN LP exprlist RP", - /* 269 */ "exprlist ::= exprlist COMMA expritem", - /* 270 */ "exprlist ::= expritem", - /* 271 */ "expritem ::= expr", - /* 272 */ "expritem ::=", - /* 273 */ "cmd ::= RESET QUERY CACHE", - /* 274 */ "cmd ::= SYNCDB ids REPLICA", - /* 275 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 276 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 277 */ "cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist", - /* 278 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 279 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 280 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 281 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 282 */ "cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist", - /* 283 */ "cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist", - /* 284 */ "cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids", - /* 285 */ "cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist", - /* 286 */ "cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist", - /* 287 */ "cmd ::= ALTER STABLE ids cpxName DROP TAG ids", - /* 288 */ "cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids", - /* 289 */ "cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem", - /* 290 */ "cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist", - /* 291 */ "cmd ::= KILL CONNECTION INTEGER", - /* 292 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 293 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", -}; -#endif /* NDEBUG */ - - -#if YYSTACKDEPTH<=0 -/* -** Try to increase the size of the parser stack. Return the number -** of errors. Return 0 on success. -*/ -static int yyGrowStack(yyParser *p){ - int newSize; - int idx; - yyStackEntry *pNew; - - newSize = p->yystksz*2 + 100; - idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; - if( p->yystack==&p->yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->yystk0; - }else{ - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); - } - if( pNew ){ - p->yystack = pNew; - p->yytos = &p->yystack[idx]; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", - yyTracePrompt, p->yystksz, newSize); - } -#endif - p->yystksz = newSize; - } - return pNew==0; -} -#endif - -/* Datatype of the argument to the memory allocated passed as the -** second argument to ParseAlloc() below. This can be changed by -** putting an appropriate #define in the %include section of the input -** grammar. -*/ -#ifndef YYMALLOCARGTYPE -# define YYMALLOCARGTYPE size_t -#endif - -/* Initialize a new parser that has already been allocated. -*/ -void ParseInit(void *yypRawParser ParseCTX_PDECL){ - yyParser *yypParser = (yyParser*)yypRawParser; - ParseCTX_STORE -#ifdef YYTRACKMAXSTACKDEPTH - yypParser->yyhwm = 0; -#endif -#if YYSTACKDEPTH<=0 - yypParser->yytos = NULL; - yypParser->yystack = NULL; - yypParser->yystksz = 0; - if( yyGrowStack(yypParser) ){ - yypParser->yystack = &yypParser->yystk0; - yypParser->yystksz = 1; - } -#endif -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - yypParser->yytos = yypParser->yystack; - yypParser->yystack[0].stateno = 0; - yypParser->yystack[0].major = 0; -#if YYSTACKDEPTH>0 - yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; -#endif -} - -#ifndef Parse_ENGINEALWAYSONSTACK -/* -** This function allocates a new parser. -** The only argument is a pointer to a function which works like -** malloc. -** -** Inputs: -** A pointer to the function used to allocate memory. -** -** Outputs: -** A pointer to a parser. This pointer is used in subsequent calls -** to Parse and ParseFree. -*/ -void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){ - yyParser *yypParser; - yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); - if( yypParser ){ - ParseCTX_STORE - ParseInit(yypParser ParseCTX_PARAM); - } - return (void*)yypParser; -} -#endif /* Parse_ENGINEALWAYSONSTACK */ - - -/* The following function deletes the "minor type" or semantic value -** associated with a symbol. The symbol can be either a terminal -** or nonterminal. "yymajor" is the symbol code, and "yypminor" is -** a pointer to the value to be deleted. The code used to do the -** deletions is derived from the %destructor and/or %token_destructor -** directives of the input grammar. -*/ -static void yy_destructor( - yyParser *yypParser, /* The parser */ - YYCODETYPE yymajor, /* Type code for object to destroy */ - YYMINORTYPE *yypminor /* The object to be destroyed */ -){ - ParseARG_FETCH - ParseCTX_FETCH - switch( yymajor ){ - /* Here is inserted the actions which take place when a - ** terminal or non-terminal is destroyed. This can happen - ** when the symbol is popped from the stack during a - ** reduce or during error processing or when a parser is - ** being destroyed before it is finished parsing. - ** - ** Note: during a reduce, the only symbols destroyed are those - ** which appear on the RHS of the rule, but which are *not* used - ** inside the C code. - */ -/********* Begin destructor definitions ***************************************/ - case 206: /* exprlist */ - case 250: /* selcollist */ - case 264: /* sclp */ -{ -#line 762 "sql.y" -tSqlExprListDestroy((yypminor->yy421)); -#line 1521 "sql.c" -} - break; - case 221: /* intitemlist */ - case 223: /* keep */ - case 244: /* columnlist */ - case 245: /* tagitemlist */ - case 246: /* tagNamelist */ - case 257: /* fill_opt */ - case 258: /* groupby_opt */ - case 260: /* orderby_opt */ - case 272: /* sortlist */ - case 276: /* grouplist */ -{ -#line 256 "sql.y" -taosArrayDestroy((yypminor->yy421)); -#line 1537 "sql.c" -} - break; - case 242: /* create_table_list */ -{ -#line 364 "sql.y" -destroyCreateTableSql((yypminor->yy438)); -#line 1544 "sql.c" -} - break; - case 247: /* select */ -{ -#line 484 "sql.y" -destroySqlNode((yypminor->yy56)); -#line 1551 "sql.c" -} - break; - case 251: /* from */ - case 268: /* tablelist */ - case 269: /* sub */ -{ -#line 539 "sql.y" -destroyRelationInfo((yypminor->yy8)); -#line 1560 "sql.c" -} - break; - case 252: /* where_opt */ - case 259: /* having_opt */ - case 266: /* expr */ - case 277: /* expritem */ -{ -#line 691 "sql.y" -tSqlExprDestroy((yypminor->yy439)); -#line 1570 "sql.c" -} - break; - case 263: /* union */ -{ -#line 492 "sql.y" -destroyAllSqlNode((yypminor->yy421)); -#line 1577 "sql.c" -} - break; - case 273: /* sortitem */ -{ -#line 624 "sql.y" -tVariantDestroy(&(yypminor->yy430)); -#line 1584 "sql.c" -} - break; -/********* End destructor definitions *****************************************/ - default: break; /* If no destructor action specified: do nothing */ - } -} - -/* -** Pop the parser's stack once. -** -** If there is a destructor routine associated with the token which -** is popped from the stack, then call it. -*/ -static void yy_pop_parser_stack(yyParser *pParser){ - yyStackEntry *yytos; - assert( pParser->yytos!=0 ); - assert( pParser->yytos > pParser->yystack ); - yytos = pParser->yytos--; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sPopping %s\n", - yyTracePrompt, - yyTokenName[yytos->major]); - } -#endif - yy_destructor(pParser, yytos->major, &yytos->minor); -} - -/* -** Clear all secondary memory allocations from the parser -*/ -void ParseFinalize(void *p){ - yyParser *pParser = (yyParser*)p; - while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); -#endif -} - -#ifndef Parse_ENGINEALWAYSONSTACK -/* -** Deallocate and destroy a parser. Destructors are called for -** all stack elements before shutting the parser down. -** -** If the YYPARSEFREENEVERNULL macro exists (for example because it -** is defined in a %include section of the input grammar) then it is -** assumed that the input pointer is never NULL. -*/ -void ParseFree( - void *p, /* The parser to be deleted */ - void (*freeProc)(void*) /* Function used to reclaim memory */ -){ -#ifndef YYPARSEFREENEVERNULL - if( p==0 ) return; -#endif - ParseFinalize(p); - (*freeProc)(p); -} -#endif /* Parse_ENGINEALWAYSONSTACK */ - -/* -** Return the peak depth of the stack for a parser. -*/ -#ifdef YYTRACKMAXSTACKDEPTH -int ParseStackPeak(void *p){ - yyParser *pParser = (yyParser*)p; - return pParser->yyhwm; -} -#endif - -/* This array of booleans keeps track of the parser statement -** coverage. The element yycoverage[X][Y] is set when the parser -** is in state X and has a lookahead token Y. In a well-tested -** systems, every element of this matrix should end up being set. -*/ -#if defined(YYCOVERAGE) -static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; -#endif - -/* -** Write into out a description of every state/lookahead combination that -** -** (1) has not been used by the parser, and -** (2) is not a syntax error. -** -** Return the number of missed state/lookahead combinations. -*/ -#if defined(YYCOVERAGE) -int ParseCoverage(FILE *out){ - int stateno, iLookAhead, i; - int nMissed = 0; - for(stateno=0; statenoYY_MAX_SHIFT ) return stateno; - assert( stateno <= YY_SHIFT_COUNT ); -#if defined(YYCOVERAGE) - yycoverage[stateno][iLookAhead] = 1; -#endif - do{ - i = yy_shift_ofst[stateno]; - assert( i>=0 ); - assert( i<=YY_ACTTAB_COUNT ); - assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); - assert( iLookAhead!=YYNOCODE ); - assert( iLookAhead < YYNTOKEN ); - i += iLookAhead; - assert( i<(int)YY_NLOOKAHEAD ); - if( yy_lookahead[i]!=iLookAhead ){ -#ifdef YYFALLBACK - YYCODETYPE iFallback; /* Fallback token */ - assert( iLookAhead %s\n", - yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); - } -#endif - assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ - iLookAhead = iFallback; - continue; - } -#endif -#ifdef YYWILDCARD - { - int j = i - iLookAhead + YYWILDCARD; - assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); - if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", - yyTracePrompt, yyTokenName[iLookAhead], - yyTokenName[YYWILDCARD]); - } -#endif /* NDEBUG */ - return yy_action[j]; - } - } -#endif /* YYWILDCARD */ - return yy_default[stateno]; - }else{ - assert( i>=0 && iYY_REDUCE_COUNT ){ - return yy_default[stateno]; - } -#else - assert( stateno<=YY_REDUCE_COUNT ); -#endif - i = yy_reduce_ofst[stateno]; - assert( iLookAhead!=YYNOCODE ); - i += iLookAhead; -#ifdef YYERRORSYMBOL - if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ - return yy_default[stateno]; - } -#else - assert( i>=0 && iyytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); - /* Here code is inserted which will execute if the parser - ** stack every overflows */ -/******** Begin %stack_overflow code ******************************************/ -/******** End %stack_overflow code ********************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument var */ - ParseCTX_STORE -} - -/* -** Print tracing information for a SHIFT action -*/ -#ifndef NDEBUG -static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ - if( yyTraceFILE ){ - if( yyNewStateyytos->major], - yyNewState); - }else{ - fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n", - yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], - yyNewState - YY_MIN_REDUCE); - } - } -} -#else -# define yyTraceShift(X,Y,Z) -#endif - -/* -** Perform a shift action. -*/ -static void yy_shift( - yyParser *yypParser, /* The parser to be shifted */ - YYACTIONTYPE yyNewState, /* The new state to shift in */ - YYCODETYPE yyMajor, /* The major token to shift in */ - ParseTOKENTYPE yyMinor /* The minor token to shift in */ -){ - yyStackEntry *yytos; - yypParser->yytos++; -#ifdef YYTRACKMAXSTACKDEPTH - if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ - yypParser->yyhwm++; - assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); - } -#endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>yypParser->yystackEnd ){ - yypParser->yytos--; - yyStackOverflow(yypParser); - return; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ - if( yyGrowStack(yypParser) ){ - yypParser->yytos--; - yyStackOverflow(yypParser); - return; - } - } -#endif - if( yyNewState > YY_MAX_SHIFT ){ - yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; - } - yytos = yypParser->yytos; - yytos->stateno = yyNewState; - yytos->major = yyMajor; - yytos->minor.yy0 = yyMinor; - yyTraceShift(yypParser, yyNewState, "Shift"); -} - -/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side -** of that rule */ -static const YYCODETYPE yyRuleInfoLhs[] = { - 197, /* (0) program ::= cmd */ - 198, /* (1) cmd ::= SHOW DATABASES */ - 198, /* (2) cmd ::= SHOW TOPICS */ - 198, /* (3) cmd ::= SHOW FUNCTIONS */ - 198, /* (4) cmd ::= SHOW MNODES */ - 198, /* (5) cmd ::= SHOW DNODES */ - 198, /* (6) cmd ::= SHOW ACCOUNTS */ - 198, /* (7) cmd ::= SHOW USERS */ - 198, /* (8) cmd ::= SHOW MODULES */ - 198, /* (9) cmd ::= SHOW QUERIES */ - 198, /* (10) cmd ::= SHOW CONNECTIONS */ - 198, /* (11) cmd ::= SHOW STREAMS */ - 198, /* (12) cmd ::= SHOW VARIABLES */ - 198, /* (13) cmd ::= SHOW SCORES */ - 198, /* (14) cmd ::= SHOW GRANTS */ - 198, /* (15) cmd ::= SHOW VNODES */ - 198, /* (16) cmd ::= SHOW VNODES ids */ - 200, /* (17) dbPrefix ::= */ - 200, /* (18) dbPrefix ::= ids DOT */ - 201, /* (19) cpxName ::= */ - 201, /* (20) cpxName ::= DOT ids */ - 198, /* (21) cmd ::= SHOW CREATE TABLE ids cpxName */ - 198, /* (22) cmd ::= SHOW CREATE STABLE ids cpxName */ - 198, /* (23) cmd ::= SHOW CREATE DATABASE ids */ - 198, /* (24) cmd ::= SHOW dbPrefix TABLES */ - 198, /* (25) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - 198, /* (26) cmd ::= SHOW dbPrefix STABLES */ - 198, /* (27) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - 198, /* (28) cmd ::= SHOW dbPrefix VGROUPS */ - 198, /* (29) cmd ::= SHOW dbPrefix VGROUPS ids */ - 198, /* (30) cmd ::= DROP TABLE ifexists ids cpxName */ - 198, /* (31) cmd ::= DROP STABLE ifexists ids cpxName */ - 198, /* (32) cmd ::= DROP DATABASE ifexists ids */ - 198, /* (33) cmd ::= DROP TOPIC ifexists ids */ - 198, /* (34) cmd ::= DROP FUNCTION ids */ - 198, /* (35) cmd ::= DROP DNODE ids */ - 198, /* (36) cmd ::= DROP USER ids */ - 198, /* (37) cmd ::= DROP ACCOUNT ids */ - 198, /* (38) cmd ::= USE ids */ - 198, /* (39) cmd ::= DESCRIBE ids cpxName */ - 198, /* (40) cmd ::= DESC ids cpxName */ - 198, /* (41) cmd ::= ALTER USER ids PASS ids */ - 198, /* (42) cmd ::= ALTER USER ids PRIVILEGE ids */ - 198, /* (43) cmd ::= ALTER DNODE ids ids */ - 198, /* (44) cmd ::= ALTER DNODE ids ids ids */ - 198, /* (45) cmd ::= ALTER LOCAL ids */ - 198, /* (46) cmd ::= ALTER LOCAL ids ids */ - 198, /* (47) cmd ::= ALTER DATABASE ids alter_db_optr */ - 198, /* (48) cmd ::= ALTER TOPIC ids alter_topic_optr */ - 198, /* (49) cmd ::= ALTER ACCOUNT ids acct_optr */ - 198, /* (50) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - 198, /* (51) cmd ::= COMPACT VNODES IN LP exprlist RP */ - 199, /* (52) ids ::= ID */ - 199, /* (53) ids ::= STRING */ - 202, /* (54) ifexists ::= IF EXISTS */ - 202, /* (55) ifexists ::= */ - 207, /* (56) ifnotexists ::= IF NOT EXISTS */ - 207, /* (57) ifnotexists ::= */ - 198, /* (58) cmd ::= CREATE DNODE ids */ - 198, /* (59) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - 198, /* (60) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - 198, /* (61) cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ - 198, /* (62) cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ - 198, /* (63) cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ - 198, /* (64) cmd ::= CREATE USER ids PASS ids */ - 211, /* (65) bufsize ::= */ - 211, /* (66) bufsize ::= BUFSIZE INTEGER */ - 212, /* (67) pps ::= */ - 212, /* (68) pps ::= PPS INTEGER */ - 213, /* (69) tseries ::= */ - 213, /* (70) tseries ::= TSERIES INTEGER */ - 214, /* (71) dbs ::= */ - 214, /* (72) dbs ::= DBS INTEGER */ - 215, /* (73) streams ::= */ - 215, /* (74) streams ::= STREAMS INTEGER */ - 216, /* (75) storage ::= */ - 216, /* (76) storage ::= STORAGE INTEGER */ - 217, /* (77) qtime ::= */ - 217, /* (78) qtime ::= QTIME INTEGER */ - 218, /* (79) users ::= */ - 218, /* (80) users ::= USERS INTEGER */ - 219, /* (81) conns ::= */ - 219, /* (82) conns ::= CONNS INTEGER */ - 220, /* (83) state ::= */ - 220, /* (84) state ::= STATE ids */ - 205, /* (85) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - 221, /* (86) intitemlist ::= intitemlist COMMA intitem */ - 221, /* (87) intitemlist ::= intitem */ - 222, /* (88) intitem ::= INTEGER */ - 223, /* (89) keep ::= KEEP intitemlist */ - 224, /* (90) cache ::= CACHE INTEGER */ - 225, /* (91) replica ::= REPLICA INTEGER */ - 226, /* (92) quorum ::= QUORUM INTEGER */ - 227, /* (93) days ::= DAYS INTEGER */ - 228, /* (94) minrows ::= MINROWS INTEGER */ - 229, /* (95) maxrows ::= MAXROWS INTEGER */ - 230, /* (96) blocks ::= BLOCKS INTEGER */ - 231, /* (97) ctime ::= CTIME INTEGER */ - 232, /* (98) wal ::= WAL INTEGER */ - 233, /* (99) fsync ::= FSYNC INTEGER */ - 234, /* (100) comp ::= COMP INTEGER */ - 235, /* (101) prec ::= PRECISION STRING */ - 236, /* (102) update ::= UPDATE INTEGER */ - 237, /* (103) cachelast ::= CACHELAST INTEGER */ - 238, /* (104) partitions ::= PARTITIONS INTEGER */ - 208, /* (105) db_optr ::= */ - 208, /* (106) db_optr ::= db_optr cache */ - 208, /* (107) db_optr ::= db_optr replica */ - 208, /* (108) db_optr ::= db_optr quorum */ - 208, /* (109) db_optr ::= db_optr days */ - 208, /* (110) db_optr ::= db_optr minrows */ - 208, /* (111) db_optr ::= db_optr maxrows */ - 208, /* (112) db_optr ::= db_optr blocks */ - 208, /* (113) db_optr ::= db_optr ctime */ - 208, /* (114) db_optr ::= db_optr wal */ - 208, /* (115) db_optr ::= db_optr fsync */ - 208, /* (116) db_optr ::= db_optr comp */ - 208, /* (117) db_optr ::= db_optr prec */ - 208, /* (118) db_optr ::= db_optr keep */ - 208, /* (119) db_optr ::= db_optr update */ - 208, /* (120) db_optr ::= db_optr cachelast */ - 209, /* (121) topic_optr ::= db_optr */ - 209, /* (122) topic_optr ::= topic_optr partitions */ - 203, /* (123) alter_db_optr ::= */ - 203, /* (124) alter_db_optr ::= alter_db_optr replica */ - 203, /* (125) alter_db_optr ::= alter_db_optr quorum */ - 203, /* (126) alter_db_optr ::= alter_db_optr keep */ - 203, /* (127) alter_db_optr ::= alter_db_optr blocks */ - 203, /* (128) alter_db_optr ::= alter_db_optr comp */ - 203, /* (129) alter_db_optr ::= alter_db_optr update */ - 203, /* (130) alter_db_optr ::= alter_db_optr cachelast */ - 204, /* (131) alter_topic_optr ::= alter_db_optr */ - 204, /* (132) alter_topic_optr ::= alter_topic_optr partitions */ - 210, /* (133) typename ::= ids */ - 210, /* (134) typename ::= ids LP signed RP */ - 210, /* (135) typename ::= ids UNSIGNED */ - 239, /* (136) signed ::= INTEGER */ - 239, /* (137) signed ::= PLUS INTEGER */ - 239, /* (138) signed ::= MINUS INTEGER */ - 198, /* (139) cmd ::= CREATE TABLE create_table_args */ - 198, /* (140) cmd ::= CREATE TABLE create_stable_args */ - 198, /* (141) cmd ::= CREATE STABLE create_stable_args */ - 198, /* (142) cmd ::= CREATE TABLE create_table_list */ - 242, /* (143) create_table_list ::= create_from_stable */ - 242, /* (144) create_table_list ::= create_table_list create_from_stable */ - 240, /* (145) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ - 241, /* (146) create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ - 243, /* (147) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ - 243, /* (148) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ - 246, /* (149) tagNamelist ::= tagNamelist COMMA ids */ - 246, /* (150) tagNamelist ::= ids */ - 240, /* (151) create_table_args ::= ifnotexists ids cpxName AS select */ - 244, /* (152) columnlist ::= columnlist COMMA column */ - 244, /* (153) columnlist ::= column */ - 248, /* (154) column ::= ids typename */ - 245, /* (155) tagitemlist ::= tagitemlist COMMA tagitem */ - 245, /* (156) tagitemlist ::= tagitem */ - 249, /* (157) tagitem ::= INTEGER */ - 249, /* (158) tagitem ::= FLOAT */ - 249, /* (159) tagitem ::= STRING */ - 249, /* (160) tagitem ::= BOOL */ - 249, /* (161) tagitem ::= NULL */ - 249, /* (162) tagitem ::= NOW */ - 249, /* (163) tagitem ::= MINUS INTEGER */ - 249, /* (164) tagitem ::= MINUS FLOAT */ - 249, /* (165) tagitem ::= PLUS INTEGER */ - 249, /* (166) tagitem ::= PLUS FLOAT */ - 247, /* (167) select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ - 247, /* (168) select ::= LP select RP */ - 263, /* (169) union ::= select */ - 263, /* (170) union ::= union UNION ALL select */ - 198, /* (171) cmd ::= union */ - 247, /* (172) select ::= SELECT selcollist */ - 264, /* (173) sclp ::= selcollist COMMA */ - 264, /* (174) sclp ::= */ - 250, /* (175) selcollist ::= sclp distinct expr as */ - 250, /* (176) selcollist ::= sclp STAR */ - 267, /* (177) as ::= AS ids */ - 267, /* (178) as ::= ids */ - 267, /* (179) as ::= */ - 265, /* (180) distinct ::= DISTINCT */ - 265, /* (181) distinct ::= */ - 251, /* (182) from ::= FROM tablelist */ - 251, /* (183) from ::= FROM sub */ - 269, /* (184) sub ::= LP union RP */ - 269, /* (185) sub ::= LP union RP ids */ - 269, /* (186) sub ::= sub COMMA LP union RP ids */ - 268, /* (187) tablelist ::= ids cpxName */ - 268, /* (188) tablelist ::= ids cpxName ids */ - 268, /* (189) tablelist ::= tablelist COMMA ids cpxName */ - 268, /* (190) tablelist ::= tablelist COMMA ids cpxName ids */ - 270, /* (191) tmvar ::= VARIABLE */ - 253, /* (192) interval_option ::= intervalKey LP tmvar RP */ - 253, /* (193) interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ - 253, /* (194) interval_option ::= */ - 271, /* (195) intervalKey ::= INTERVAL */ - 271, /* (196) intervalKey ::= EVERY */ - 255, /* (197) session_option ::= */ - 255, /* (198) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ - 256, /* (199) windowstate_option ::= */ - 256, /* (200) windowstate_option ::= STATE_WINDOW LP ids RP */ - 257, /* (201) fill_opt ::= */ - 257, /* (202) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - 257, /* (203) fill_opt ::= FILL LP ID RP */ - 254, /* (204) sliding_opt ::= SLIDING LP tmvar RP */ - 254, /* (205) sliding_opt ::= */ - 260, /* (206) orderby_opt ::= */ - 260, /* (207) orderby_opt ::= ORDER BY sortlist */ - 272, /* (208) sortlist ::= sortlist COMMA item sortorder */ - 272, /* (209) sortlist ::= item sortorder */ - 274, /* (210) item ::= ids cpxName */ - 275, /* (211) sortorder ::= ASC */ - 275, /* (212) sortorder ::= DESC */ - 275, /* (213) sortorder ::= */ - 258, /* (214) groupby_opt ::= */ - 258, /* (215) groupby_opt ::= GROUP BY grouplist */ - 276, /* (216) grouplist ::= grouplist COMMA item */ - 276, /* (217) grouplist ::= item */ - 259, /* (218) having_opt ::= */ - 259, /* (219) having_opt ::= HAVING expr */ - 262, /* (220) limit_opt ::= */ - 262, /* (221) limit_opt ::= LIMIT signed */ - 262, /* (222) limit_opt ::= LIMIT signed OFFSET signed */ - 262, /* (223) limit_opt ::= LIMIT signed COMMA signed */ - 261, /* (224) slimit_opt ::= */ - 261, /* (225) slimit_opt ::= SLIMIT signed */ - 261, /* (226) slimit_opt ::= SLIMIT signed SOFFSET signed */ - 261, /* (227) slimit_opt ::= SLIMIT signed COMMA signed */ - 252, /* (228) where_opt ::= */ - 252, /* (229) where_opt ::= WHERE expr */ - 266, /* (230) expr ::= LP expr RP */ - 266, /* (231) expr ::= ID */ - 266, /* (232) expr ::= ID DOT ID */ - 266, /* (233) expr ::= ID DOT STAR */ - 266, /* (234) expr ::= INTEGER */ - 266, /* (235) expr ::= MINUS INTEGER */ - 266, /* (236) expr ::= PLUS INTEGER */ - 266, /* (237) expr ::= FLOAT */ - 266, /* (238) expr ::= MINUS FLOAT */ - 266, /* (239) expr ::= PLUS FLOAT */ - 266, /* (240) expr ::= STRING */ - 266, /* (241) expr ::= NOW */ - 266, /* (242) expr ::= VARIABLE */ - 266, /* (243) expr ::= PLUS VARIABLE */ - 266, /* (244) expr ::= MINUS VARIABLE */ - 266, /* (245) expr ::= BOOL */ - 266, /* (246) expr ::= NULL */ - 266, /* (247) expr ::= ID LP exprlist RP */ - 266, /* (248) expr ::= ID LP STAR RP */ - 266, /* (249) expr ::= expr IS NULL */ - 266, /* (250) expr ::= expr IS NOT NULL */ - 266, /* (251) expr ::= expr LT expr */ - 266, /* (252) expr ::= expr GT expr */ - 266, /* (253) expr ::= expr LE expr */ - 266, /* (254) expr ::= expr GE expr */ - 266, /* (255) expr ::= expr NE expr */ - 266, /* (256) expr ::= expr EQ expr */ - 266, /* (257) expr ::= expr BETWEEN expr AND expr */ - 266, /* (258) expr ::= expr AND expr */ - 266, /* (259) expr ::= expr OR expr */ - 266, /* (260) expr ::= expr PLUS expr */ - 266, /* (261) expr ::= expr MINUS expr */ - 266, /* (262) expr ::= expr STAR expr */ - 266, /* (263) expr ::= expr SLASH expr */ - 266, /* (264) expr ::= expr REM expr */ - 266, /* (265) expr ::= expr LIKE expr */ - 266, /* (266) expr ::= expr MATCH expr */ - 266, /* (267) expr ::= expr NMATCH expr */ - 266, /* (268) expr ::= expr IN LP exprlist RP */ - 206, /* (269) exprlist ::= exprlist COMMA expritem */ - 206, /* (270) exprlist ::= expritem */ - 277, /* (271) expritem ::= expr */ - 277, /* (272) expritem ::= */ - 198, /* (273) cmd ::= RESET QUERY CACHE */ - 198, /* (274) cmd ::= SYNCDB ids REPLICA */ - 198, /* (275) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - 198, /* (276) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - 198, /* (277) cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ - 198, /* (278) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - 198, /* (279) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - 198, /* (280) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - 198, /* (281) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - 198, /* (282) cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ - 198, /* (283) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ - 198, /* (284) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ - 198, /* (285) cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ - 198, /* (286) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ - 198, /* (287) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ - 198, /* (288) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ - 198, /* (289) cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ - 198, /* (290) cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ - 198, /* (291) cmd ::= KILL CONNECTION INTEGER */ - 198, /* (292) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - 198, /* (293) cmd ::= KILL QUERY INTEGER COLON INTEGER */ -}; - -/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number -** of symbols on the right-hand side of that rule. */ -static const signed char yyRuleInfoNRhs[] = { - -1, /* (0) program ::= cmd */ - -2, /* (1) cmd ::= SHOW DATABASES */ - -2, /* (2) cmd ::= SHOW TOPICS */ - -2, /* (3) cmd ::= SHOW FUNCTIONS */ - -2, /* (4) cmd ::= SHOW MNODES */ - -2, /* (5) cmd ::= SHOW DNODES */ - -2, /* (6) cmd ::= SHOW ACCOUNTS */ - -2, /* (7) cmd ::= SHOW USERS */ - -2, /* (8) cmd ::= SHOW MODULES */ - -2, /* (9) cmd ::= SHOW QUERIES */ - -2, /* (10) cmd ::= SHOW CONNECTIONS */ - -2, /* (11) cmd ::= SHOW STREAMS */ - -2, /* (12) cmd ::= SHOW VARIABLES */ - -2, /* (13) cmd ::= SHOW SCORES */ - -2, /* (14) cmd ::= SHOW GRANTS */ - -2, /* (15) cmd ::= SHOW VNODES */ - -3, /* (16) cmd ::= SHOW VNODES ids */ - 0, /* (17) dbPrefix ::= */ - -2, /* (18) dbPrefix ::= ids DOT */ - 0, /* (19) cpxName ::= */ - -2, /* (20) cpxName ::= DOT ids */ - -5, /* (21) cmd ::= SHOW CREATE TABLE ids cpxName */ - -5, /* (22) cmd ::= SHOW CREATE STABLE ids cpxName */ - -4, /* (23) cmd ::= SHOW CREATE DATABASE ids */ - -3, /* (24) cmd ::= SHOW dbPrefix TABLES */ - -5, /* (25) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - -3, /* (26) cmd ::= SHOW dbPrefix STABLES */ - -5, /* (27) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - -3, /* (28) cmd ::= SHOW dbPrefix VGROUPS */ - -4, /* (29) cmd ::= SHOW dbPrefix VGROUPS ids */ - -5, /* (30) cmd ::= DROP TABLE ifexists ids cpxName */ - -5, /* (31) cmd ::= DROP STABLE ifexists ids cpxName */ - -4, /* (32) cmd ::= DROP DATABASE ifexists ids */ - -4, /* (33) cmd ::= DROP TOPIC ifexists ids */ - -3, /* (34) cmd ::= DROP FUNCTION ids */ - -3, /* (35) cmd ::= DROP DNODE ids */ - -3, /* (36) cmd ::= DROP USER ids */ - -3, /* (37) cmd ::= DROP ACCOUNT ids */ - -2, /* (38) cmd ::= USE ids */ - -3, /* (39) cmd ::= DESCRIBE ids cpxName */ - -3, /* (40) cmd ::= DESC ids cpxName */ - -5, /* (41) cmd ::= ALTER USER ids PASS ids */ - -5, /* (42) cmd ::= ALTER USER ids PRIVILEGE ids */ - -4, /* (43) cmd ::= ALTER DNODE ids ids */ - -5, /* (44) cmd ::= ALTER DNODE ids ids ids */ - -3, /* (45) cmd ::= ALTER LOCAL ids */ - -4, /* (46) cmd ::= ALTER LOCAL ids ids */ - -4, /* (47) cmd ::= ALTER DATABASE ids alter_db_optr */ - -4, /* (48) cmd ::= ALTER TOPIC ids alter_topic_optr */ - -4, /* (49) cmd ::= ALTER ACCOUNT ids acct_optr */ - -6, /* (50) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - -6, /* (51) cmd ::= COMPACT VNODES IN LP exprlist RP */ - -1, /* (52) ids ::= ID */ - -1, /* (53) ids ::= STRING */ - -2, /* (54) ifexists ::= IF EXISTS */ - 0, /* (55) ifexists ::= */ - -3, /* (56) ifnotexists ::= IF NOT EXISTS */ - 0, /* (57) ifnotexists ::= */ - -3, /* (58) cmd ::= CREATE DNODE ids */ - -6, /* (59) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - -5, /* (60) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - -5, /* (61) cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ - -8, /* (62) cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ - -9, /* (63) cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ - -5, /* (64) cmd ::= CREATE USER ids PASS ids */ - 0, /* (65) bufsize ::= */ - -2, /* (66) bufsize ::= BUFSIZE INTEGER */ - 0, /* (67) pps ::= */ - -2, /* (68) pps ::= PPS INTEGER */ - 0, /* (69) tseries ::= */ - -2, /* (70) tseries ::= TSERIES INTEGER */ - 0, /* (71) dbs ::= */ - -2, /* (72) dbs ::= DBS INTEGER */ - 0, /* (73) streams ::= */ - -2, /* (74) streams ::= STREAMS INTEGER */ - 0, /* (75) storage ::= */ - -2, /* (76) storage ::= STORAGE INTEGER */ - 0, /* (77) qtime ::= */ - -2, /* (78) qtime ::= QTIME INTEGER */ - 0, /* (79) users ::= */ - -2, /* (80) users ::= USERS INTEGER */ - 0, /* (81) conns ::= */ - -2, /* (82) conns ::= CONNS INTEGER */ - 0, /* (83) state ::= */ - -2, /* (84) state ::= STATE ids */ - -9, /* (85) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - -3, /* (86) intitemlist ::= intitemlist COMMA intitem */ - -1, /* (87) intitemlist ::= intitem */ - -1, /* (88) intitem ::= INTEGER */ - -2, /* (89) keep ::= KEEP intitemlist */ - -2, /* (90) cache ::= CACHE INTEGER */ - -2, /* (91) replica ::= REPLICA INTEGER */ - -2, /* (92) quorum ::= QUORUM INTEGER */ - -2, /* (93) days ::= DAYS INTEGER */ - -2, /* (94) minrows ::= MINROWS INTEGER */ - -2, /* (95) maxrows ::= MAXROWS INTEGER */ - -2, /* (96) blocks ::= BLOCKS INTEGER */ - -2, /* (97) ctime ::= CTIME INTEGER */ - -2, /* (98) wal ::= WAL INTEGER */ - -2, /* (99) fsync ::= FSYNC INTEGER */ - -2, /* (100) comp ::= COMP INTEGER */ - -2, /* (101) prec ::= PRECISION STRING */ - -2, /* (102) update ::= UPDATE INTEGER */ - -2, /* (103) cachelast ::= CACHELAST INTEGER */ - -2, /* (104) partitions ::= PARTITIONS INTEGER */ - 0, /* (105) db_optr ::= */ - -2, /* (106) db_optr ::= db_optr cache */ - -2, /* (107) db_optr ::= db_optr replica */ - -2, /* (108) db_optr ::= db_optr quorum */ - -2, /* (109) db_optr ::= db_optr days */ - -2, /* (110) db_optr ::= db_optr minrows */ - -2, /* (111) db_optr ::= db_optr maxrows */ - -2, /* (112) db_optr ::= db_optr blocks */ - -2, /* (113) db_optr ::= db_optr ctime */ - -2, /* (114) db_optr ::= db_optr wal */ - -2, /* (115) db_optr ::= db_optr fsync */ - -2, /* (116) db_optr ::= db_optr comp */ - -2, /* (117) db_optr ::= db_optr prec */ - -2, /* (118) db_optr ::= db_optr keep */ - -2, /* (119) db_optr ::= db_optr update */ - -2, /* (120) db_optr ::= db_optr cachelast */ - -1, /* (121) topic_optr ::= db_optr */ - -2, /* (122) topic_optr ::= topic_optr partitions */ - 0, /* (123) alter_db_optr ::= */ - -2, /* (124) alter_db_optr ::= alter_db_optr replica */ - -2, /* (125) alter_db_optr ::= alter_db_optr quorum */ - -2, /* (126) alter_db_optr ::= alter_db_optr keep */ - -2, /* (127) alter_db_optr ::= alter_db_optr blocks */ - -2, /* (128) alter_db_optr ::= alter_db_optr comp */ - -2, /* (129) alter_db_optr ::= alter_db_optr update */ - -2, /* (130) alter_db_optr ::= alter_db_optr cachelast */ - -1, /* (131) alter_topic_optr ::= alter_db_optr */ - -2, /* (132) alter_topic_optr ::= alter_topic_optr partitions */ - -1, /* (133) typename ::= ids */ - -4, /* (134) typename ::= ids LP signed RP */ - -2, /* (135) typename ::= ids UNSIGNED */ - -1, /* (136) signed ::= INTEGER */ - -2, /* (137) signed ::= PLUS INTEGER */ - -2, /* (138) signed ::= MINUS INTEGER */ - -3, /* (139) cmd ::= CREATE TABLE create_table_args */ - -3, /* (140) cmd ::= CREATE TABLE create_stable_args */ - -3, /* (141) cmd ::= CREATE STABLE create_stable_args */ - -3, /* (142) cmd ::= CREATE TABLE create_table_list */ - -1, /* (143) create_table_list ::= create_from_stable */ - -2, /* (144) create_table_list ::= create_table_list create_from_stable */ - -6, /* (145) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ - -10, /* (146) create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ - -10, /* (147) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ - -13, /* (148) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ - -3, /* (149) tagNamelist ::= tagNamelist COMMA ids */ - -1, /* (150) tagNamelist ::= ids */ - -5, /* (151) create_table_args ::= ifnotexists ids cpxName AS select */ - -3, /* (152) columnlist ::= columnlist COMMA column */ - -1, /* (153) columnlist ::= column */ - -2, /* (154) column ::= ids typename */ - -3, /* (155) tagitemlist ::= tagitemlist COMMA tagitem */ - -1, /* (156) tagitemlist ::= tagitem */ - -1, /* (157) tagitem ::= INTEGER */ - -1, /* (158) tagitem ::= FLOAT */ - -1, /* (159) tagitem ::= STRING */ - -1, /* (160) tagitem ::= BOOL */ - -1, /* (161) tagitem ::= NULL */ - -1, /* (162) tagitem ::= NOW */ - -2, /* (163) tagitem ::= MINUS INTEGER */ - -2, /* (164) tagitem ::= MINUS FLOAT */ - -2, /* (165) tagitem ::= PLUS INTEGER */ - -2, /* (166) tagitem ::= PLUS FLOAT */ - -14, /* (167) select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ - -3, /* (168) select ::= LP select RP */ - -1, /* (169) union ::= select */ - -4, /* (170) union ::= union UNION ALL select */ - -1, /* (171) cmd ::= union */ - -2, /* (172) select ::= SELECT selcollist */ - -2, /* (173) sclp ::= selcollist COMMA */ - 0, /* (174) sclp ::= */ - -4, /* (175) selcollist ::= sclp distinct expr as */ - -2, /* (176) selcollist ::= sclp STAR */ - -2, /* (177) as ::= AS ids */ - -1, /* (178) as ::= ids */ - 0, /* (179) as ::= */ - -1, /* (180) distinct ::= DISTINCT */ - 0, /* (181) distinct ::= */ - -2, /* (182) from ::= FROM tablelist */ - -2, /* (183) from ::= FROM sub */ - -3, /* (184) sub ::= LP union RP */ - -4, /* (185) sub ::= LP union RP ids */ - -6, /* (186) sub ::= sub COMMA LP union RP ids */ - -2, /* (187) tablelist ::= ids cpxName */ - -3, /* (188) tablelist ::= ids cpxName ids */ - -4, /* (189) tablelist ::= tablelist COMMA ids cpxName */ - -5, /* (190) tablelist ::= tablelist COMMA ids cpxName ids */ - -1, /* (191) tmvar ::= VARIABLE */ - -4, /* (192) interval_option ::= intervalKey LP tmvar RP */ - -6, /* (193) interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ - 0, /* (194) interval_option ::= */ - -1, /* (195) intervalKey ::= INTERVAL */ - -1, /* (196) intervalKey ::= EVERY */ - 0, /* (197) session_option ::= */ - -7, /* (198) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ - 0, /* (199) windowstate_option ::= */ - -4, /* (200) windowstate_option ::= STATE_WINDOW LP ids RP */ - 0, /* (201) fill_opt ::= */ - -6, /* (202) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - -4, /* (203) fill_opt ::= FILL LP ID RP */ - -4, /* (204) sliding_opt ::= SLIDING LP tmvar RP */ - 0, /* (205) sliding_opt ::= */ - 0, /* (206) orderby_opt ::= */ - -3, /* (207) orderby_opt ::= ORDER BY sortlist */ - -4, /* (208) sortlist ::= sortlist COMMA item sortorder */ - -2, /* (209) sortlist ::= item sortorder */ - -2, /* (210) item ::= ids cpxName */ - -1, /* (211) sortorder ::= ASC */ - -1, /* (212) sortorder ::= DESC */ - 0, /* (213) sortorder ::= */ - 0, /* (214) groupby_opt ::= */ - -3, /* (215) groupby_opt ::= GROUP BY grouplist */ - -3, /* (216) grouplist ::= grouplist COMMA item */ - -1, /* (217) grouplist ::= item */ - 0, /* (218) having_opt ::= */ - -2, /* (219) having_opt ::= HAVING expr */ - 0, /* (220) limit_opt ::= */ - -2, /* (221) limit_opt ::= LIMIT signed */ - -4, /* (222) limit_opt ::= LIMIT signed OFFSET signed */ - -4, /* (223) limit_opt ::= LIMIT signed COMMA signed */ - 0, /* (224) slimit_opt ::= */ - -2, /* (225) slimit_opt ::= SLIMIT signed */ - -4, /* (226) slimit_opt ::= SLIMIT signed SOFFSET signed */ - -4, /* (227) slimit_opt ::= SLIMIT signed COMMA signed */ - 0, /* (228) where_opt ::= */ - -2, /* (229) where_opt ::= WHERE expr */ - -3, /* (230) expr ::= LP expr RP */ - -1, /* (231) expr ::= ID */ - -3, /* (232) expr ::= ID DOT ID */ - -3, /* (233) expr ::= ID DOT STAR */ - -1, /* (234) expr ::= INTEGER */ - -2, /* (235) expr ::= MINUS INTEGER */ - -2, /* (236) expr ::= PLUS INTEGER */ - -1, /* (237) expr ::= FLOAT */ - -2, /* (238) expr ::= MINUS FLOAT */ - -2, /* (239) expr ::= PLUS FLOAT */ - -1, /* (240) expr ::= STRING */ - -1, /* (241) expr ::= NOW */ - -1, /* (242) expr ::= VARIABLE */ - -2, /* (243) expr ::= PLUS VARIABLE */ - -2, /* (244) expr ::= MINUS VARIABLE */ - -1, /* (245) expr ::= BOOL */ - -1, /* (246) expr ::= NULL */ - -4, /* (247) expr ::= ID LP exprlist RP */ - -4, /* (248) expr ::= ID LP STAR RP */ - -3, /* (249) expr ::= expr IS NULL */ - -4, /* (250) expr ::= expr IS NOT NULL */ - -3, /* (251) expr ::= expr LT expr */ - -3, /* (252) expr ::= expr GT expr */ - -3, /* (253) expr ::= expr LE expr */ - -3, /* (254) expr ::= expr GE expr */ - -3, /* (255) expr ::= expr NE expr */ - -3, /* (256) expr ::= expr EQ expr */ - -5, /* (257) expr ::= expr BETWEEN expr AND expr */ - -3, /* (258) expr ::= expr AND expr */ - -3, /* (259) expr ::= expr OR expr */ - -3, /* (260) expr ::= expr PLUS expr */ - -3, /* (261) expr ::= expr MINUS expr */ - -3, /* (262) expr ::= expr STAR expr */ - -3, /* (263) expr ::= expr SLASH expr */ - -3, /* (264) expr ::= expr REM expr */ - -3, /* (265) expr ::= expr LIKE expr */ - -3, /* (266) expr ::= expr MATCH expr */ - -3, /* (267) expr ::= expr NMATCH expr */ - -5, /* (268) expr ::= expr IN LP exprlist RP */ - -3, /* (269) exprlist ::= exprlist COMMA expritem */ - -1, /* (270) exprlist ::= expritem */ - -1, /* (271) expritem ::= expr */ - 0, /* (272) expritem ::= */ - -3, /* (273) cmd ::= RESET QUERY CACHE */ - -3, /* (274) cmd ::= SYNCDB ids REPLICA */ - -7, /* (275) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - -7, /* (276) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - -7, /* (277) cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ - -7, /* (278) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - -7, /* (279) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - -8, /* (280) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - -9, /* (281) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - -7, /* (282) cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ - -7, /* (283) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ - -7, /* (284) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ - -7, /* (285) cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ - -7, /* (286) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ - -7, /* (287) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ - -8, /* (288) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ - -9, /* (289) cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ - -7, /* (290) cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ - -3, /* (291) cmd ::= KILL CONNECTION INTEGER */ - -5, /* (292) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - -5, /* (293) cmd ::= KILL QUERY INTEGER COLON INTEGER */ -}; - -static void yy_accept(yyParser*); /* Forward Declaration */ - -/* -** Perform a reduce action and the shift that must immediately -** follow the reduce. -** -** The yyLookahead and yyLookaheadToken parameters provide reduce actions -** access to the lookahead token (if any). The yyLookahead will be YYNOCODE -** if the lookahead token has already been consumed. As this procedure is -** only called from one place, optimizing compilers will in-line it, which -** means that the extra parameters have no performance impact. -*/ -static YYACTIONTYPE yy_reduce( - yyParser *yypParser, /* The parser */ - unsigned int yyruleno, /* Number of the rule by which to reduce */ - int yyLookahead, /* Lookahead token, or YYNOCODE if none */ - ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ - ParseCTX_PDECL /* %extra_context */ -){ - int yygoto; /* The next state */ - YYACTIONTYPE yyact; /* The next action */ - yyStackEntry *yymsp; /* The top of the parser's stack */ - int yysize; /* Amount to pop the stack */ - ParseARG_FETCH - (void)yyLookahead; - (void)yyLookaheadToken; - yymsp = yypParser->yytos; -#ifndef NDEBUG - if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfoNRhs[yyruleno]; - if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", - yyTracePrompt, - yyruleno, yyRuleName[yyruleno], - yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ - yypParser->yyhwm++; - assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack)); - } -#endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>=yypParser->yystackEnd ){ - yyStackOverflow(yypParser); - /* The call to yyStackOverflow() above pops the stack until it is - ** empty, causing the main parser loop to exit. So the return value - ** is never used and does not matter. */ - return 0; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ - if( yyGrowStack(yypParser) ){ - yyStackOverflow(yypParser); - /* The call to yyStackOverflow() above pops the stack until it is - ** empty, causing the main parser loop to exit. So the return value - ** is never used and does not matter. */ - return 0; - } - yymsp = yypParser->yytos; - } -#endif - } - - switch( yyruleno ){ - /* Beginning here are the reduction cases. A typical example - ** follows: - ** case 0: - ** #line - ** { ... } // User supplied code - ** #line - ** break; - */ -/********** Begin reduce actions **********************************************/ - YYMINORTYPE yylhsminor; - case 0: /* program ::= cmd */ - case 139: /* cmd ::= CREATE TABLE create_table_args */ yytestcase(yyruleno==139); - case 140: /* cmd ::= CREATE TABLE create_stable_args */ yytestcase(yyruleno==140); - case 141: /* cmd ::= CREATE STABLE create_stable_args */ yytestcase(yyruleno==141); -#line 63 "sql.y" -{} -#line 2561 "sql.c" - break; - case 1: /* cmd ::= SHOW DATABASES */ -#line 66 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);} -#line 2566 "sql.c" - break; - case 2: /* cmd ::= SHOW TOPICS */ -#line 67 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);} -#line 2571 "sql.c" - break; - case 3: /* cmd ::= SHOW FUNCTIONS */ -#line 68 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);} -#line 2576 "sql.c" - break; - case 4: /* cmd ::= SHOW MNODES */ -#line 69 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);} -#line 2581 "sql.c" - break; - case 5: /* cmd ::= SHOW DNODES */ -#line 70 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);} -#line 2586 "sql.c" - break; - case 6: /* cmd ::= SHOW ACCOUNTS */ -#line 71 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);} -#line 2591 "sql.c" - break; - case 7: /* cmd ::= SHOW USERS */ -#line 72 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_USER, 0, 0);} -#line 2596 "sql.c" - break; - case 8: /* cmd ::= SHOW MODULES */ -#line 74 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_MODULE, 0, 0); } -#line 2601 "sql.c" - break; - case 9: /* cmd ::= SHOW QUERIES */ -#line 75 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_QUERIES, 0, 0); } -#line 2606 "sql.c" - break; - case 10: /* cmd ::= SHOW CONNECTIONS */ -#line 76 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONNS, 0, 0);} -#line 2611 "sql.c" - break; - case 11: /* cmd ::= SHOW STREAMS */ -#line 77 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } -#line 2616 "sql.c" - break; - case 12: /* cmd ::= SHOW VARIABLES */ -#line 78 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } -#line 2621 "sql.c" - break; - case 13: /* cmd ::= SHOW SCORES */ -#line 79 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } -#line 2626 "sql.c" - break; - case 14: /* cmd ::= SHOW GRANTS */ -#line 80 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); } -#line 2631 "sql.c" - break; - case 15: /* cmd ::= SHOW VNODES */ -#line 82 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, 0, 0); } -#line 2636 "sql.c" - break; - case 16: /* cmd ::= SHOW VNODES ids */ -#line 83 "sql.y" -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &yymsp[0].minor.yy0, 0); } -#line 2641 "sql.c" - break; - case 17: /* dbPrefix ::= */ -#line 87 "sql.y" -{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.type = 0;} -#line 2646 "sql.c" - break; - case 18: /* dbPrefix ::= ids DOT */ -#line 88 "sql.y" -{yylhsminor.yy0 = yymsp[-1].minor.yy0; } -#line 2651 "sql.c" - yymsp[-1].minor.yy0 = yylhsminor.yy0; - break; - case 19: /* cpxName ::= */ -#line 91 "sql.y" -{yymsp[1].minor.yy0.n = 0; } -#line 2657 "sql.c" - break; - case 20: /* cpxName ::= DOT ids */ -#line 92 "sql.y" -{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n += 1; } -#line 2662 "sql.c" - break; - case 21: /* cmd ::= SHOW CREATE TABLE ids cpxName */ -#line 94 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_TABLE, 1, &yymsp[-1].minor.yy0); -} -#line 2670 "sql.c" - break; - case 22: /* cmd ::= SHOW CREATE STABLE ids cpxName */ -#line 98 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_STABLE, 1, &yymsp[-1].minor.yy0); -} -#line 2678 "sql.c" - break; - case 23: /* cmd ::= SHOW CREATE DATABASE ids */ -#line 103 "sql.y" -{ - setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_DATABASE, 1, &yymsp[0].minor.yy0); -} -#line 2685 "sql.c" - break; - case 24: /* cmd ::= SHOW dbPrefix TABLES */ -#line 107 "sql.y" -{ - setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &yymsp[-1].minor.yy0, 0); -} -#line 2692 "sql.c" - break; - case 25: /* cmd ::= SHOW dbPrefix TABLES LIKE ids */ -#line 111 "sql.y" -{ - setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0); -} -#line 2699 "sql.c" - break; - case 26: /* cmd ::= SHOW dbPrefix STABLES */ -#line 115 "sql.y" -{ - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &yymsp[-1].minor.yy0, 0); -} -#line 2706 "sql.c" - break; - case 27: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ -#line 119 "sql.y" -{ - SStrToken token; - tSetDbName(&token, &yymsp[-3].minor.yy0); - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &yymsp[0].minor.yy0); -} -#line 2715 "sql.c" - break; - case 28: /* cmd ::= SHOW dbPrefix VGROUPS */ -#line 125 "sql.y" -{ - SStrToken token; - tSetDbName(&token, &yymsp[-1].minor.yy0); - setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); -} -#line 2724 "sql.c" - break; - case 29: /* cmd ::= SHOW dbPrefix VGROUPS ids */ -#line 131 "sql.y" -{ - SStrToken token; - tSetDbName(&token, &yymsp[-2].minor.yy0); - setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &yymsp[0].minor.yy0); -} -#line 2733 "sql.c" - break; - case 30: /* cmd ::= DROP TABLE ifexists ids cpxName */ -#line 138 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0, -1, -1); -} -#line 2741 "sql.c" - break; - case 31: /* cmd ::= DROP STABLE ifexists ids cpxName */ -#line 144 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0, -1, TSDB_SUPER_TABLE); -} -#line 2749 "sql.c" - break; - case 32: /* cmd ::= DROP DATABASE ifexists ids */ -#line 149 "sql.y" -{ setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0, TSDB_DB_TYPE_DEFAULT, -1); } -#line 2754 "sql.c" - break; - case 33: /* cmd ::= DROP TOPIC ifexists ids */ -#line 150 "sql.y" -{ setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0, TSDB_DB_TYPE_TOPIC, -1); } -#line 2759 "sql.c" - break; - case 34: /* cmd ::= DROP FUNCTION ids */ -#line 151 "sql.y" -{ setDropFuncInfo(pInfo, TSDB_SQL_DROP_FUNCTION, &yymsp[0].minor.yy0); } -#line 2764 "sql.c" - break; - case 35: /* cmd ::= DROP DNODE ids */ -#line 153 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &yymsp[0].minor.yy0); } -#line 2769 "sql.c" - break; - case 36: /* cmd ::= DROP USER ids */ -#line 154 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_DROP_USER, 1, &yymsp[0].minor.yy0); } -#line 2774 "sql.c" - break; - case 37: /* cmd ::= DROP ACCOUNT ids */ -#line 155 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &yymsp[0].minor.yy0); } -#line 2779 "sql.c" - break; - case 38: /* cmd ::= USE ids */ -#line 158 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_USE_DB, 1, &yymsp[0].minor.yy0);} -#line 2784 "sql.c" - break; - case 39: /* cmd ::= DESCRIBE ids cpxName */ - case 40: /* cmd ::= DESC ids cpxName */ yytestcase(yyruleno==40); -#line 161 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &yymsp[-1].minor.yy0); -} -#line 2793 "sql.c" - break; - case 41: /* cmd ::= ALTER USER ids PASS ids */ -#line 170 "sql.y" -{ setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, NULL); } -#line 2798 "sql.c" - break; - case 42: /* cmd ::= ALTER USER ids PRIVILEGE ids */ -#line 171 "sql.y" -{ setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &yymsp[-2].minor.yy0, NULL, &yymsp[0].minor.yy0);} -#line 2803 "sql.c" - break; - case 43: /* cmd ::= ALTER DNODE ids ids */ -#line 172 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } -#line 2808 "sql.c" - break; - case 44: /* cmd ::= ALTER DNODE ids ids ids */ -#line 173 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } -#line 2813 "sql.c" - break; - case 45: /* cmd ::= ALTER LOCAL ids */ -#line 174 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &yymsp[0].minor.yy0); } -#line 2818 "sql.c" - break; - case 46: /* cmd ::= ALTER LOCAL ids ids */ -#line 175 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } -#line 2823 "sql.c" - break; - case 47: /* cmd ::= ALTER DATABASE ids alter_db_optr */ - case 48: /* cmd ::= ALTER TOPIC ids alter_topic_optr */ yytestcase(yyruleno==48); -#line 176 "sql.y" -{ SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy90, &t);} -#line 2829 "sql.c" - break; - case 49: /* cmd ::= ALTER ACCOUNT ids acct_optr */ -#line 179 "sql.y" -{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy171);} -#line 2834 "sql.c" - break; - case 50: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ -#line 180 "sql.y" -{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy171);} -#line 2839 "sql.c" - break; - case 51: /* cmd ::= COMPACT VNODES IN LP exprlist RP */ -#line 184 "sql.y" -{ setCompactVnodeSql(pInfo, TSDB_SQL_COMPACT_VNODE, yymsp[-1].minor.yy421);} -#line 2844 "sql.c" - break; - case 52: /* ids ::= ID */ - case 53: /* ids ::= STRING */ yytestcase(yyruleno==53); -#line 190 "sql.y" -{yylhsminor.yy0 = yymsp[0].minor.yy0; } -#line 2850 "sql.c" - yymsp[0].minor.yy0 = yylhsminor.yy0; - break; - case 54: /* ifexists ::= IF EXISTS */ -#line 194 "sql.y" -{ yymsp[-1].minor.yy0.n = 1;} -#line 2856 "sql.c" - break; - case 55: /* ifexists ::= */ - case 57: /* ifnotexists ::= */ yytestcase(yyruleno==57); - case 181: /* distinct ::= */ yytestcase(yyruleno==181); -#line 195 "sql.y" -{ yymsp[1].minor.yy0.n = 0;} -#line 2863 "sql.c" - break; - case 56: /* ifnotexists ::= IF NOT EXISTS */ -#line 198 "sql.y" -{ yymsp[-2].minor.yy0.n = 1;} -#line 2868 "sql.c" - break; - case 58: /* cmd ::= CREATE DNODE ids */ -#line 203 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);} -#line 2873 "sql.c" - break; - case 59: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ -#line 205 "sql.y" -{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy171);} -#line 2878 "sql.c" - break; - case 60: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - case 61: /* cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ yytestcase(yyruleno==61); -#line 206 "sql.y" -{ setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy90, &yymsp[-2].minor.yy0);} -#line 2884 "sql.c" - break; - case 62: /* cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ -#line 208 "sql.y" -{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy183, &yymsp[0].minor.yy0, 1);} -#line 2889 "sql.c" - break; - case 63: /* cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ -#line 209 "sql.y" -{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy183, &yymsp[0].minor.yy0, 2);} -#line 2894 "sql.c" - break; - case 64: /* cmd ::= CREATE USER ids PASS ids */ -#line 210 "sql.y" -{ setCreateUserSql(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} -#line 2899 "sql.c" - break; - case 65: /* bufsize ::= */ - case 67: /* pps ::= */ yytestcase(yyruleno==67); - case 69: /* tseries ::= */ yytestcase(yyruleno==69); - case 71: /* dbs ::= */ yytestcase(yyruleno==71); - case 73: /* streams ::= */ yytestcase(yyruleno==73); - case 75: /* storage ::= */ yytestcase(yyruleno==75); - case 77: /* qtime ::= */ yytestcase(yyruleno==77); - case 79: /* users ::= */ yytestcase(yyruleno==79); - case 81: /* conns ::= */ yytestcase(yyruleno==81); - case 83: /* state ::= */ yytestcase(yyruleno==83); -#line 212 "sql.y" -{ yymsp[1].minor.yy0.n = 0; } -#line 2913 "sql.c" - break; - case 66: /* bufsize ::= BUFSIZE INTEGER */ - case 68: /* pps ::= PPS INTEGER */ yytestcase(yyruleno==68); - case 70: /* tseries ::= TSERIES INTEGER */ yytestcase(yyruleno==70); - case 72: /* dbs ::= DBS INTEGER */ yytestcase(yyruleno==72); - case 74: /* streams ::= STREAMS INTEGER */ yytestcase(yyruleno==74); - case 76: /* storage ::= STORAGE INTEGER */ yytestcase(yyruleno==76); - case 78: /* qtime ::= QTIME INTEGER */ yytestcase(yyruleno==78); - case 80: /* users ::= USERS INTEGER */ yytestcase(yyruleno==80); - case 82: /* conns ::= CONNS INTEGER */ yytestcase(yyruleno==82); - case 84: /* state ::= STATE ids */ yytestcase(yyruleno==84); -#line 213 "sql.y" -{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } -#line 2927 "sql.c" - break; - case 85: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ -#line 243 "sql.y" -{ - yylhsminor.yy171.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; - yylhsminor.yy171.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; - yylhsminor.yy171.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; - yylhsminor.yy171.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; - yylhsminor.yy171.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; - yylhsminor.yy171.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy171.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy171.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; - yylhsminor.yy171.stat = yymsp[0].minor.yy0; -} -#line 2942 "sql.c" - yymsp[-8].minor.yy171 = yylhsminor.yy171; - break; - case 86: /* intitemlist ::= intitemlist COMMA intitem */ - case 155: /* tagitemlist ::= tagitemlist COMMA tagitem */ yytestcase(yyruleno==155); -#line 259 "sql.y" -{ yylhsminor.yy421 = tVariantListAppend(yymsp[-2].minor.yy421, &yymsp[0].minor.yy430, -1); } -#line 2949 "sql.c" - yymsp[-2].minor.yy421 = yylhsminor.yy421; - break; - case 87: /* intitemlist ::= intitem */ - case 156: /* tagitemlist ::= tagitem */ yytestcase(yyruleno==156); -#line 260 "sql.y" -{ yylhsminor.yy421 = tVariantListAppend(NULL, &yymsp[0].minor.yy430, -1); } -#line 2956 "sql.c" - yymsp[0].minor.yy421 = yylhsminor.yy421; - break; - case 88: /* intitem ::= INTEGER */ - case 157: /* tagitem ::= INTEGER */ yytestcase(yyruleno==157); - case 158: /* tagitem ::= FLOAT */ yytestcase(yyruleno==158); - case 159: /* tagitem ::= STRING */ yytestcase(yyruleno==159); - case 160: /* tagitem ::= BOOL */ yytestcase(yyruleno==160); -#line 262 "sql.y" -{ toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy430, &yymsp[0].minor.yy0); } -#line 2966 "sql.c" - yymsp[0].minor.yy430 = yylhsminor.yy430; - break; - case 89: /* keep ::= KEEP intitemlist */ -#line 266 "sql.y" -{ yymsp[-1].minor.yy421 = yymsp[0].minor.yy421; } -#line 2972 "sql.c" - break; - case 90: /* cache ::= CACHE INTEGER */ - case 91: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==91); - case 92: /* quorum ::= QUORUM INTEGER */ yytestcase(yyruleno==92); - case 93: /* days ::= DAYS INTEGER */ yytestcase(yyruleno==93); - case 94: /* minrows ::= MINROWS INTEGER */ yytestcase(yyruleno==94); - case 95: /* maxrows ::= MAXROWS INTEGER */ yytestcase(yyruleno==95); - case 96: /* blocks ::= BLOCKS INTEGER */ yytestcase(yyruleno==96); - case 97: /* ctime ::= CTIME INTEGER */ yytestcase(yyruleno==97); - case 98: /* wal ::= WAL INTEGER */ yytestcase(yyruleno==98); - case 99: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==99); - case 100: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==100); - case 101: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==101); - case 102: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==102); - case 103: /* cachelast ::= CACHELAST INTEGER */ yytestcase(yyruleno==103); - case 104: /* partitions ::= PARTITIONS INTEGER */ yytestcase(yyruleno==104); -#line 268 "sql.y" -{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } -#line 2991 "sql.c" - break; - case 105: /* db_optr ::= */ -#line 285 "sql.y" -{setDefaultCreateDbOption(&yymsp[1].minor.yy90); yymsp[1].minor.yy90.dbType = TSDB_DB_TYPE_DEFAULT;} -#line 2996 "sql.c" - break; - case 106: /* db_optr ::= db_optr cache */ -#line 287 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3001 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 107: /* db_optr ::= db_optr replica */ - case 124: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==124); -#line 288 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3008 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 108: /* db_optr ::= db_optr quorum */ - case 125: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==125); -#line 289 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3015 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 109: /* db_optr ::= db_optr days */ -#line 290 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3021 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 110: /* db_optr ::= db_optr minrows */ -#line 291 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } -#line 3027 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 111: /* db_optr ::= db_optr maxrows */ -#line 292 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } -#line 3033 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 112: /* db_optr ::= db_optr blocks */ - case 127: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==127); -#line 293 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3040 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 113: /* db_optr ::= db_optr ctime */ -#line 294 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3046 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 114: /* db_optr ::= db_optr wal */ -#line 295 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3052 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 115: /* db_optr ::= db_optr fsync */ -#line 296 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3058 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 116: /* db_optr ::= db_optr comp */ - case 128: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==128); -#line 297 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3065 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 117: /* db_optr ::= db_optr prec */ -#line 298 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.precision = yymsp[0].minor.yy0; } -#line 3071 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 118: /* db_optr ::= db_optr keep */ - case 126: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==126); -#line 299 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.keep = yymsp[0].minor.yy421; } -#line 3078 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 119: /* db_optr ::= db_optr update */ - case 129: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==129); -#line 300 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3085 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 120: /* db_optr ::= db_optr cachelast */ - case 130: /* alter_db_optr ::= alter_db_optr cachelast */ yytestcase(yyruleno==130); -#line 301 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.cachelast = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3092 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 121: /* topic_optr ::= db_optr */ - case 131: /* alter_topic_optr ::= alter_db_optr */ yytestcase(yyruleno==131); -#line 305 "sql.y" -{ yylhsminor.yy90 = yymsp[0].minor.yy90; yylhsminor.yy90.dbType = TSDB_DB_TYPE_TOPIC; } -#line 3099 "sql.c" - yymsp[0].minor.yy90 = yylhsminor.yy90; - break; - case 122: /* topic_optr ::= topic_optr partitions */ - case 132: /* alter_topic_optr ::= alter_topic_optr partitions */ yytestcase(yyruleno==132); -#line 306 "sql.y" -{ yylhsminor.yy90 = yymsp[-1].minor.yy90; yylhsminor.yy90.partitions = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3106 "sql.c" - yymsp[-1].minor.yy90 = yylhsminor.yy90; - break; - case 123: /* alter_db_optr ::= */ -#line 309 "sql.y" -{ setDefaultCreateDbOption(&yymsp[1].minor.yy90); yymsp[1].minor.yy90.dbType = TSDB_DB_TYPE_DEFAULT;} -#line 3112 "sql.c" - break; - case 133: /* typename ::= ids */ -#line 329 "sql.y" -{ - yymsp[0].minor.yy0.type = 0; - tSetColumnType (&yylhsminor.yy183, &yymsp[0].minor.yy0); -} -#line 3120 "sql.c" - yymsp[0].minor.yy183 = yylhsminor.yy183; - break; - case 134: /* typename ::= ids LP signed RP */ -#line 335 "sql.y" -{ - if (yymsp[-1].minor.yy325 <= 0) { - yymsp[-3].minor.yy0.type = 0; - tSetColumnType(&yylhsminor.yy183, &yymsp[-3].minor.yy0); - } else { - yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy325; // negative value of name length - tSetColumnType(&yylhsminor.yy183, &yymsp[-3].minor.yy0); - } -} -#line 3134 "sql.c" - yymsp[-3].minor.yy183 = yylhsminor.yy183; - break; - case 135: /* typename ::= ids UNSIGNED */ -#line 346 "sql.y" -{ - yymsp[-1].minor.yy0.type = 0; - yymsp[-1].minor.yy0.n = ((yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z); - tSetColumnType (&yylhsminor.yy183, &yymsp[-1].minor.yy0); -} -#line 3144 "sql.c" - yymsp[-1].minor.yy183 = yylhsminor.yy183; - break; - case 136: /* signed ::= INTEGER */ -#line 353 "sql.y" -{ yylhsminor.yy325 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3150 "sql.c" - yymsp[0].minor.yy325 = yylhsminor.yy325; - break; - case 137: /* signed ::= PLUS INTEGER */ -#line 354 "sql.y" -{ yymsp[-1].minor.yy325 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } -#line 3156 "sql.c" - break; - case 138: /* signed ::= MINUS INTEGER */ -#line 355 "sql.y" -{ yymsp[-1].minor.yy325 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} -#line 3161 "sql.c" - break; - case 142: /* cmd ::= CREATE TABLE create_table_list */ -#line 361 "sql.y" -{ pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy438;} -#line 3166 "sql.c" - break; - case 143: /* create_table_list ::= create_from_stable */ -#line 365 "sql.y" -{ - SCreateTableSql* pCreateTable = calloc(1, sizeof(SCreateTableSql)); - pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); - - taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy152); - pCreateTable->type = TSQL_CREATE_TABLE_FROM_STABLE; - yylhsminor.yy438 = pCreateTable; -} -#line 3178 "sql.c" - yymsp[0].minor.yy438 = yylhsminor.yy438; - break; - case 144: /* create_table_list ::= create_table_list create_from_stable */ -#line 374 "sql.y" -{ - taosArrayPush(yymsp[-1].minor.yy438->childTableInfo, &yymsp[0].minor.yy152); - yylhsminor.yy438 = yymsp[-1].minor.yy438; -} -#line 3187 "sql.c" - yymsp[-1].minor.yy438 = yylhsminor.yy438; - break; - case 145: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ -#line 380 "sql.y" -{ - yylhsminor.yy438 = tSetCreateTableInfo(yymsp[-1].minor.yy421, NULL, NULL, TSQL_CREATE_TABLE); - setSqlInfo(pInfo, yylhsminor.yy438, NULL, TSDB_SQL_CREATE_TABLE); - - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - setCreatedTableName(pInfo, &yymsp[-4].minor.yy0, &yymsp[-5].minor.yy0); -} -#line 3199 "sql.c" - yymsp[-5].minor.yy438 = yylhsminor.yy438; - break; - case 146: /* create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ -#line 390 "sql.y" -{ - yylhsminor.yy438 = tSetCreateTableInfo(yymsp[-5].minor.yy421, yymsp[-1].minor.yy421, NULL, TSQL_CREATE_STABLE); - setSqlInfo(pInfo, yylhsminor.yy438, NULL, TSDB_SQL_CREATE_TABLE); - - yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; - setCreatedTableName(pInfo, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); -} -#line 3211 "sql.c" - yymsp[-9].minor.yy438 = yylhsminor.yy438; - break; - case 147: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ -#line 401 "sql.y" -{ - yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; - yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; - yylhsminor.yy152 = createNewChildTableInfo(&yymsp[-5].minor.yy0, NULL, yymsp[-1].minor.yy421, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); -} -#line 3221 "sql.c" - yymsp[-9].minor.yy152 = yylhsminor.yy152; - break; - case 148: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ -#line 407 "sql.y" -{ - yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; - yymsp[-11].minor.yy0.n += yymsp[-10].minor.yy0.n; - yylhsminor.yy152 = createNewChildTableInfo(&yymsp[-8].minor.yy0, yymsp[-5].minor.yy421, yymsp[-1].minor.yy421, &yymsp[-11].minor.yy0, &yymsp[-12].minor.yy0); -} -#line 3231 "sql.c" - yymsp[-12].minor.yy152 = yylhsminor.yy152; - break; - case 149: /* tagNamelist ::= tagNamelist COMMA ids */ -#line 415 "sql.y" -{taosArrayPush(yymsp[-2].minor.yy421, &yymsp[0].minor.yy0); yylhsminor.yy421 = yymsp[-2].minor.yy421; } -#line 3237 "sql.c" - yymsp[-2].minor.yy421 = yylhsminor.yy421; - break; - case 150: /* tagNamelist ::= ids */ -#line 416 "sql.y" -{yylhsminor.yy421 = taosArrayInit(4, sizeof(SStrToken)); taosArrayPush(yylhsminor.yy421, &yymsp[0].minor.yy0);} -#line 3243 "sql.c" - yymsp[0].minor.yy421 = yylhsminor.yy421; - break; - case 151: /* create_table_args ::= ifnotexists ids cpxName AS select */ -#line 420 "sql.y" -{ - yylhsminor.yy438 = tSetCreateTableInfo(NULL, NULL, yymsp[0].minor.yy56, TSQL_CREATE_STREAM); - setSqlInfo(pInfo, yylhsminor.yy438, NULL, TSDB_SQL_CREATE_TABLE); - - yymsp[-3].minor.yy0.n += yymsp[-2].minor.yy0.n; - setCreatedTableName(pInfo, &yymsp[-3].minor.yy0, &yymsp[-4].minor.yy0); -} -#line 3255 "sql.c" - yymsp[-4].minor.yy438 = yylhsminor.yy438; - break; - case 152: /* columnlist ::= columnlist COMMA column */ -#line 431 "sql.y" -{taosArrayPush(yymsp[-2].minor.yy421, &yymsp[0].minor.yy183); yylhsminor.yy421 = yymsp[-2].minor.yy421; } -#line 3261 "sql.c" - yymsp[-2].minor.yy421 = yylhsminor.yy421; - break; - case 153: /* columnlist ::= column */ -#line 432 "sql.y" -{yylhsminor.yy421 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy421, &yymsp[0].minor.yy183);} -#line 3267 "sql.c" - yymsp[0].minor.yy421 = yylhsminor.yy421; - break; - case 154: /* column ::= ids typename */ -#line 436 "sql.y" -{ - tSetColumnInfo(&yylhsminor.yy183, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy183); -} -#line 3275 "sql.c" - yymsp[-1].minor.yy183 = yylhsminor.yy183; - break; - case 161: /* tagitem ::= NULL */ -#line 451 "sql.y" -{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy430, &yymsp[0].minor.yy0); } -#line 3281 "sql.c" - yymsp[0].minor.yy430 = yylhsminor.yy430; - break; - case 162: /* tagitem ::= NOW */ -#line 452 "sql.y" -{ yymsp[0].minor.yy0.type = TSDB_DATA_TYPE_TIMESTAMP; tVariantCreate(&yylhsminor.yy430, &yymsp[0].minor.yy0);} -#line 3287 "sql.c" - yymsp[0].minor.yy430 = yylhsminor.yy430; - break; - case 163: /* tagitem ::= MINUS INTEGER */ - case 164: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==164); - case 165: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==165); - case 166: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==166); -#line 454 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; - toTSDBType(yymsp[-1].minor.yy0.type); - tVariantCreate(&yylhsminor.yy430, &yymsp[-1].minor.yy0); -} -#line 3301 "sql.c" - yymsp[-1].minor.yy430 = yylhsminor.yy430; - break; - case 167: /* select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ -#line 485 "sql.y" -{ - yylhsminor.yy56 = tSetQuerySqlNode(&yymsp[-13].minor.yy0, yymsp[-12].minor.yy421, yymsp[-11].minor.yy8, yymsp[-10].minor.yy439, yymsp[-4].minor.yy421, yymsp[-2].minor.yy421, &yymsp[-9].minor.yy400, &yymsp[-7].minor.yy147, &yymsp[-6].minor.yy40, &yymsp[-8].minor.yy0, yymsp[-5].minor.yy421, &yymsp[0].minor.yy166, &yymsp[-1].minor.yy166, yymsp[-3].minor.yy439); -} -#line 3309 "sql.c" - yymsp[-13].minor.yy56 = yylhsminor.yy56; - break; - case 168: /* select ::= LP select RP */ -#line 489 "sql.y" -{yymsp[-2].minor.yy56 = yymsp[-1].minor.yy56;} -#line 3315 "sql.c" - break; - case 169: /* union ::= select */ -#line 493 "sql.y" -{ yylhsminor.yy421 = setSubclause(NULL, yymsp[0].minor.yy56); } -#line 3320 "sql.c" - yymsp[0].minor.yy421 = yylhsminor.yy421; - break; - case 170: /* union ::= union UNION ALL select */ -#line 494 "sql.y" -{ yylhsminor.yy421 = appendSelectClause(yymsp[-3].minor.yy421, yymsp[0].minor.yy56); } -#line 3326 "sql.c" - yymsp[-3].minor.yy421 = yylhsminor.yy421; - break; - case 171: /* cmd ::= union */ -#line 496 "sql.y" -{ setSqlInfo(pInfo, yymsp[0].minor.yy421, NULL, TSDB_SQL_SELECT); } -#line 3332 "sql.c" - break; - case 172: /* select ::= SELECT selcollist */ -#line 503 "sql.y" -{ - yylhsminor.yy56 = tSetQuerySqlNode(&yymsp[-1].minor.yy0, yymsp[0].minor.yy421, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); -} -#line 3339 "sql.c" - yymsp[-1].minor.yy56 = yylhsminor.yy56; - break; - case 173: /* sclp ::= selcollist COMMA */ -#line 515 "sql.y" -{yylhsminor.yy421 = yymsp[-1].minor.yy421;} -#line 3345 "sql.c" - yymsp[-1].minor.yy421 = yylhsminor.yy421; - break; - case 174: /* sclp ::= */ - case 206: /* orderby_opt ::= */ yytestcase(yyruleno==206); -#line 516 "sql.y" -{yymsp[1].minor.yy421 = 0;} -#line 3352 "sql.c" - break; - case 175: /* selcollist ::= sclp distinct expr as */ -#line 517 "sql.y" -{ - yylhsminor.yy421 = tSqlExprListAppend(yymsp[-3].minor.yy421, yymsp[-1].minor.yy439, yymsp[-2].minor.yy0.n? &yymsp[-2].minor.yy0:0, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); -} -#line 3359 "sql.c" - yymsp[-3].minor.yy421 = yylhsminor.yy421; - break; - case 176: /* selcollist ::= sclp STAR */ -#line 521 "sql.y" -{ - tSqlExpr *pNode = tSqlExprCreateIdValue(NULL, TK_ALL); - yylhsminor.yy421 = tSqlExprListAppend(yymsp[-1].minor.yy421, pNode, 0, 0); -} -#line 3368 "sql.c" - yymsp[-1].minor.yy421 = yylhsminor.yy421; - break; - case 177: /* as ::= AS ids */ -#line 529 "sql.y" -{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } -#line 3374 "sql.c" - break; - case 178: /* as ::= ids */ -#line 530 "sql.y" -{ yylhsminor.yy0 = yymsp[0].minor.yy0; } -#line 3379 "sql.c" - yymsp[0].minor.yy0 = yylhsminor.yy0; - break; - case 179: /* as ::= */ -#line 531 "sql.y" -{ yymsp[1].minor.yy0.n = 0; } -#line 3385 "sql.c" - break; - case 180: /* distinct ::= DISTINCT */ -#line 534 "sql.y" -{ yylhsminor.yy0 = yymsp[0].minor.yy0; } -#line 3390 "sql.c" - yymsp[0].minor.yy0 = yylhsminor.yy0; - break; - case 182: /* from ::= FROM tablelist */ - case 183: /* from ::= FROM sub */ yytestcase(yyruleno==183); -#line 540 "sql.y" -{yymsp[-1].minor.yy8 = yymsp[0].minor.yy8;} -#line 3397 "sql.c" - break; - case 184: /* sub ::= LP union RP */ -#line 545 "sql.y" -{yymsp[-2].minor.yy8 = addSubqueryElem(NULL, yymsp[-1].minor.yy421, NULL);} -#line 3402 "sql.c" - break; - case 185: /* sub ::= LP union RP ids */ -#line 546 "sql.y" -{yymsp[-3].minor.yy8 = addSubqueryElem(NULL, yymsp[-2].minor.yy421, &yymsp[0].minor.yy0);} -#line 3407 "sql.c" - break; - case 186: /* sub ::= sub COMMA LP union RP ids */ -#line 547 "sql.y" -{yylhsminor.yy8 = addSubqueryElem(yymsp[-5].minor.yy8, yymsp[-2].minor.yy421, &yymsp[0].minor.yy0);} -#line 3412 "sql.c" - yymsp[-5].minor.yy8 = yylhsminor.yy8; - break; - case 187: /* tablelist ::= ids cpxName */ -#line 551 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy8 = setTableNameList(NULL, &yymsp[-1].minor.yy0, NULL); -} -#line 3421 "sql.c" - yymsp[-1].minor.yy8 = yylhsminor.yy8; - break; - case 188: /* tablelist ::= ids cpxName ids */ -#line 556 "sql.y" -{ - yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy8 = setTableNameList(NULL, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); -} -#line 3430 "sql.c" - yymsp[-2].minor.yy8 = yylhsminor.yy8; - break; - case 189: /* tablelist ::= tablelist COMMA ids cpxName */ -#line 561 "sql.y" -{ - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy8 = setTableNameList(yymsp[-3].minor.yy8, &yymsp[-1].minor.yy0, NULL); -} -#line 3439 "sql.c" - yymsp[-3].minor.yy8 = yylhsminor.yy8; - break; - case 190: /* tablelist ::= tablelist COMMA ids cpxName ids */ -#line 566 "sql.y" -{ - yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy8 = setTableNameList(yymsp[-4].minor.yy8, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); -} -#line 3448 "sql.c" - yymsp[-4].minor.yy8 = yylhsminor.yy8; - break; - case 191: /* tmvar ::= VARIABLE */ -#line 573 "sql.y" -{yylhsminor.yy0 = yymsp[0].minor.yy0;} -#line 3454 "sql.c" - yymsp[0].minor.yy0 = yylhsminor.yy0; - break; - case 192: /* interval_option ::= intervalKey LP tmvar RP */ -#line 576 "sql.y" -{yylhsminor.yy400.interval = yymsp[-1].minor.yy0; yylhsminor.yy400.offset.n = 0; yylhsminor.yy400.token = yymsp[-3].minor.yy104;} -#line 3460 "sql.c" - yymsp[-3].minor.yy400 = yylhsminor.yy400; - break; - case 193: /* interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ -#line 577 "sql.y" -{yylhsminor.yy400.interval = yymsp[-3].minor.yy0; yylhsminor.yy400.offset = yymsp[-1].minor.yy0; yylhsminor.yy400.token = yymsp[-5].minor.yy104;} -#line 3466 "sql.c" - yymsp[-5].minor.yy400 = yylhsminor.yy400; - break; - case 194: /* interval_option ::= */ -#line 578 "sql.y" -{memset(&yymsp[1].minor.yy400, 0, sizeof(yymsp[1].minor.yy400));} -#line 3472 "sql.c" - break; - case 195: /* intervalKey ::= INTERVAL */ -#line 581 "sql.y" -{yymsp[0].minor.yy104 = TK_INTERVAL;} -#line 3477 "sql.c" - break; - case 196: /* intervalKey ::= EVERY */ -#line 582 "sql.y" -{yymsp[0].minor.yy104 = TK_EVERY; } -#line 3482 "sql.c" - break; - case 197: /* session_option ::= */ -#line 585 "sql.y" -{yymsp[1].minor.yy147.col.n = 0; yymsp[1].minor.yy147.gap.n = 0;} -#line 3487 "sql.c" - break; - case 198: /* session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ -#line 586 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - yymsp[-6].minor.yy147.col = yymsp[-4].minor.yy0; - yymsp[-6].minor.yy147.gap = yymsp[-1].minor.yy0; -} -#line 3496 "sql.c" - break; - case 199: /* windowstate_option ::= */ -#line 593 "sql.y" -{ yymsp[1].minor.yy40.col.n = 0; yymsp[1].minor.yy40.col.z = NULL;} -#line 3501 "sql.c" - break; - case 200: /* windowstate_option ::= STATE_WINDOW LP ids RP */ -#line 594 "sql.y" -{ yymsp[-3].minor.yy40.col = yymsp[-1].minor.yy0; } -#line 3506 "sql.c" - break; - case 201: /* fill_opt ::= */ -#line 598 "sql.y" -{ yymsp[1].minor.yy421 = 0; } -#line 3511 "sql.c" - break; - case 202: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ -#line 599 "sql.y" -{ - tVariant A = {0}; - toTSDBType(yymsp[-3].minor.yy0.type); - tVariantCreate(&A, &yymsp[-3].minor.yy0); - - tVariantListInsert(yymsp[-1].minor.yy421, &A, -1, 0); - yymsp[-5].minor.yy421 = yymsp[-1].minor.yy421; -} -#line 3523 "sql.c" - break; - case 203: /* fill_opt ::= FILL LP ID RP */ -#line 608 "sql.y" -{ - toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-3].minor.yy421 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); -} -#line 3531 "sql.c" - break; - case 204: /* sliding_opt ::= SLIDING LP tmvar RP */ -#line 614 "sql.y" -{yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } -#line 3536 "sql.c" - break; - case 205: /* sliding_opt ::= */ -#line 615 "sql.y" -{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } -#line 3541 "sql.c" - break; - case 207: /* orderby_opt ::= ORDER BY sortlist */ -#line 627 "sql.y" -{yymsp[-2].minor.yy421 = yymsp[0].minor.yy421;} -#line 3546 "sql.c" - break; - case 208: /* sortlist ::= sortlist COMMA item sortorder */ -#line 629 "sql.y" -{ - yylhsminor.yy421 = tVariantListAppend(yymsp[-3].minor.yy421, &yymsp[-1].minor.yy430, yymsp[0].minor.yy96); -} -#line 3553 "sql.c" - yymsp[-3].minor.yy421 = yylhsminor.yy421; - break; - case 209: /* sortlist ::= item sortorder */ -#line 633 "sql.y" -{ - yylhsminor.yy421 = tVariantListAppend(NULL, &yymsp[-1].minor.yy430, yymsp[0].minor.yy96); -} -#line 3561 "sql.c" - yymsp[-1].minor.yy421 = yylhsminor.yy421; - break; - case 210: /* item ::= ids cpxName */ -#line 638 "sql.y" -{ - toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - - tVariantCreate(&yylhsminor.yy430, &yymsp[-1].minor.yy0); -} -#line 3572 "sql.c" - yymsp[-1].minor.yy430 = yylhsminor.yy430; - break; - case 211: /* sortorder ::= ASC */ -#line 646 "sql.y" -{ yymsp[0].minor.yy96 = TSDB_ORDER_ASC; } -#line 3578 "sql.c" - break; - case 212: /* sortorder ::= DESC */ -#line 647 "sql.y" -{ yymsp[0].minor.yy96 = TSDB_ORDER_DESC;} -#line 3583 "sql.c" - break; - case 213: /* sortorder ::= */ -#line 648 "sql.y" -{ yymsp[1].minor.yy96 = TSDB_ORDER_ASC; } -#line 3588 "sql.c" - break; - case 214: /* groupby_opt ::= */ -#line 656 "sql.y" -{ yymsp[1].minor.yy421 = 0;} -#line 3593 "sql.c" - break; - case 215: /* groupby_opt ::= GROUP BY grouplist */ -#line 657 "sql.y" -{ yymsp[-2].minor.yy421 = yymsp[0].minor.yy421;} -#line 3598 "sql.c" - break; - case 216: /* grouplist ::= grouplist COMMA item */ -#line 659 "sql.y" -{ - yylhsminor.yy421 = tVariantListAppend(yymsp[-2].minor.yy421, &yymsp[0].minor.yy430, -1); -} -#line 3605 "sql.c" - yymsp[-2].minor.yy421 = yylhsminor.yy421; - break; - case 217: /* grouplist ::= item */ -#line 663 "sql.y" -{ - yylhsminor.yy421 = tVariantListAppend(NULL, &yymsp[0].minor.yy430, -1); -} -#line 3613 "sql.c" - yymsp[0].minor.yy421 = yylhsminor.yy421; - break; - case 218: /* having_opt ::= */ - case 228: /* where_opt ::= */ yytestcase(yyruleno==228); - case 272: /* expritem ::= */ yytestcase(yyruleno==272); -#line 670 "sql.y" -{yymsp[1].minor.yy439 = 0;} -#line 3621 "sql.c" - break; - case 219: /* having_opt ::= HAVING expr */ - case 229: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==229); -#line 671 "sql.y" -{yymsp[-1].minor.yy439 = yymsp[0].minor.yy439;} -#line 3627 "sql.c" - break; - case 220: /* limit_opt ::= */ - case 224: /* slimit_opt ::= */ yytestcase(yyruleno==224); -#line 675 "sql.y" -{yymsp[1].minor.yy166.limit = -1; yymsp[1].minor.yy166.offset = 0;} -#line 3633 "sql.c" - break; - case 221: /* limit_opt ::= LIMIT signed */ - case 225: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==225); -#line 676 "sql.y" -{yymsp[-1].minor.yy166.limit = yymsp[0].minor.yy325; yymsp[-1].minor.yy166.offset = 0;} -#line 3639 "sql.c" - break; - case 222: /* limit_opt ::= LIMIT signed OFFSET signed */ -#line 678 "sql.y" -{ yymsp[-3].minor.yy166.limit = yymsp[-2].minor.yy325; yymsp[-3].minor.yy166.offset = yymsp[0].minor.yy325;} -#line 3644 "sql.c" - break; - case 223: /* limit_opt ::= LIMIT signed COMMA signed */ -#line 680 "sql.y" -{ yymsp[-3].minor.yy166.limit = yymsp[0].minor.yy325; yymsp[-3].minor.yy166.offset = yymsp[-2].minor.yy325;} -#line 3649 "sql.c" - break; - case 226: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ -#line 686 "sql.y" -{yymsp[-3].minor.yy166.limit = yymsp[-2].minor.yy325; yymsp[-3].minor.yy166.offset = yymsp[0].minor.yy325;} -#line 3654 "sql.c" - break; - case 227: /* slimit_opt ::= SLIMIT signed COMMA signed */ -#line 688 "sql.y" -{yymsp[-3].minor.yy166.limit = yymsp[0].minor.yy325; yymsp[-3].minor.yy166.offset = yymsp[-2].minor.yy325;} -#line 3659 "sql.c" - break; - case 230: /* expr ::= LP expr RP */ -#line 701 "sql.y" -{yylhsminor.yy439 = yymsp[-1].minor.yy439; yylhsminor.yy439->exprToken.z = yymsp[-2].minor.yy0.z; yylhsminor.yy439->exprToken.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} -#line 3664 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 231: /* expr ::= ID */ -#line 703 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_ID);} -#line 3670 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 232: /* expr ::= ID DOT ID */ -#line 704 "sql.y" -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ID);} -#line 3676 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 233: /* expr ::= ID DOT STAR */ -#line 705 "sql.y" -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ALL);} -#line 3682 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 234: /* expr ::= INTEGER */ -#line 707 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_INTEGER);} -#line 3688 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 235: /* expr ::= MINUS INTEGER */ - case 236: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==236); -#line 708 "sql.y" -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_INTEGER);} -#line 3695 "sql.c" - yymsp[-1].minor.yy439 = yylhsminor.yy439; - break; - case 237: /* expr ::= FLOAT */ -#line 710 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_FLOAT);} -#line 3701 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 238: /* expr ::= MINUS FLOAT */ - case 239: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==239); -#line 711 "sql.y" -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_FLOAT);} -#line 3708 "sql.c" - yymsp[-1].minor.yy439 = yylhsminor.yy439; - break; - case 240: /* expr ::= STRING */ -#line 713 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_STRING);} -#line 3714 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 241: /* expr ::= NOW */ -#line 714 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NOW); } -#line 3720 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 242: /* expr ::= VARIABLE */ -#line 715 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_VARIABLE);} -#line 3726 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 243: /* expr ::= PLUS VARIABLE */ - case 244: /* expr ::= MINUS VARIABLE */ yytestcase(yyruleno==244); -#line 716 "sql.y" -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_VARIABLE; yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_VARIABLE);} -#line 3733 "sql.c" - yymsp[-1].minor.yy439 = yylhsminor.yy439; - break; - case 245: /* expr ::= BOOL */ -#line 718 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_BOOL);} -#line 3739 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 246: /* expr ::= NULL */ -#line 719 "sql.y" -{ yylhsminor.yy439 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NULL);} -#line 3745 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 247: /* expr ::= ID LP exprlist RP */ -#line 722 "sql.y" -{ tStrTokenAppend(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy439 = tSqlExprCreateFunction(yymsp[-1].minor.yy421, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } -#line 3751 "sql.c" - yymsp[-3].minor.yy439 = yylhsminor.yy439; - break; - case 248: /* expr ::= ID LP STAR RP */ -#line 725 "sql.y" -{ tStrTokenAppend(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy439 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } -#line 3757 "sql.c" - yymsp[-3].minor.yy439 = yylhsminor.yy439; - break; - case 249: /* expr ::= expr IS NULL */ -#line 728 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, NULL, TK_ISNULL);} -#line 3763 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 250: /* expr ::= expr IS NOT NULL */ -#line 729 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-3].minor.yy439, NULL, TK_NOTNULL);} -#line 3769 "sql.c" - yymsp[-3].minor.yy439 = yylhsminor.yy439; - break; - case 251: /* expr ::= expr LT expr */ -#line 732 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_LT);} -#line 3775 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 252: /* expr ::= expr GT expr */ -#line 733 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_GT);} -#line 3781 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 253: /* expr ::= expr LE expr */ -#line 734 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_LE);} -#line 3787 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 254: /* expr ::= expr GE expr */ -#line 735 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_GE);} -#line 3793 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 255: /* expr ::= expr NE expr */ -#line 736 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_NE);} -#line 3799 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 256: /* expr ::= expr EQ expr */ -#line 737 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_EQ);} -#line 3805 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 257: /* expr ::= expr BETWEEN expr AND expr */ -#line 739 "sql.y" -{ tSqlExpr* X2 = tSqlExprClone(yymsp[-4].minor.yy439); yylhsminor.yy439 = tSqlExprCreate(tSqlExprCreate(yymsp[-4].minor.yy439, yymsp[-2].minor.yy439, TK_GE), tSqlExprCreate(X2, yymsp[0].minor.yy439, TK_LE), TK_AND);} -#line 3811 "sql.c" - yymsp[-4].minor.yy439 = yylhsminor.yy439; - break; - case 258: /* expr ::= expr AND expr */ -#line 741 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_AND);} -#line 3817 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 259: /* expr ::= expr OR expr */ -#line 742 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_OR); } -#line 3823 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 260: /* expr ::= expr PLUS expr */ -#line 745 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_PLUS); } -#line 3829 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 261: /* expr ::= expr MINUS expr */ -#line 746 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_MINUS); } -#line 3835 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 262: /* expr ::= expr STAR expr */ -#line 747 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_STAR); } -#line 3841 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 263: /* expr ::= expr SLASH expr */ -#line 748 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_DIVIDE);} -#line 3847 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 264: /* expr ::= expr REM expr */ -#line 749 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_REM); } -#line 3853 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 265: /* expr ::= expr LIKE expr */ -#line 752 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_LIKE); } -#line 3859 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 266: /* expr ::= expr MATCH expr */ -#line 755 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_MATCH); } -#line 3865 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 267: /* expr ::= expr NMATCH expr */ -#line 756 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-2].minor.yy439, yymsp[0].minor.yy439, TK_NMATCH); } -#line 3871 "sql.c" - yymsp[-2].minor.yy439 = yylhsminor.yy439; - break; - case 268: /* expr ::= expr IN LP exprlist RP */ -#line 759 "sql.y" -{yylhsminor.yy439 = tSqlExprCreate(yymsp[-4].minor.yy439, (tSqlExpr*)yymsp[-1].minor.yy421, TK_IN); } -#line 3877 "sql.c" - yymsp[-4].minor.yy439 = yylhsminor.yy439; - break; - case 269: /* exprlist ::= exprlist COMMA expritem */ -#line 767 "sql.y" -{yylhsminor.yy421 = tSqlExprListAppend(yymsp[-2].minor.yy421,yymsp[0].minor.yy439,0, 0);} -#line 3883 "sql.c" - yymsp[-2].minor.yy421 = yylhsminor.yy421; - break; - case 270: /* exprlist ::= expritem */ -#line 768 "sql.y" -{yylhsminor.yy421 = tSqlExprListAppend(0,yymsp[0].minor.yy439,0, 0);} -#line 3889 "sql.c" - yymsp[0].minor.yy421 = yylhsminor.yy421; - break; - case 271: /* expritem ::= expr */ -#line 769 "sql.y" -{yylhsminor.yy439 = yymsp[0].minor.yy439;} -#line 3895 "sql.c" - yymsp[0].minor.yy439 = yylhsminor.yy439; - break; - case 273: /* cmd ::= RESET QUERY CACHE */ -#line 773 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} -#line 3901 "sql.c" - break; - case 274: /* cmd ::= SYNCDB ids REPLICA */ -#line 776 "sql.y" -{ setDCLSqlElems(pInfo, TSDB_SQL_SYNC_DB_REPLICA, 1, &yymsp[-1].minor.yy0);} -#line 3906 "sql.c" - break; - case 275: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ -#line 779 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3915 "sql.c" - break; - case 276: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ -#line 785 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - - toTSDBType(yymsp[0].minor.yy0.type); - SArray* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3928 "sql.c" - break; - case 277: /* cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ -#line 795 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3937 "sql.c" - break; - case 278: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ -#line 802 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3946 "sql.c" - break; - case 279: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ -#line 807 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - - toTSDBType(yymsp[0].minor.yy0.type); - SArray* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3959 "sql.c" - break; - case 280: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ -#line 817 "sql.y" -{ - yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; - - toTSDBType(yymsp[-1].minor.yy0.type); - SArray* A = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); - - toTSDBType(yymsp[0].minor.yy0.type); - A = tVariantListAppendToken(A, &yymsp[0].minor.yy0, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3975 "sql.c" - break; - case 281: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ -#line 830 "sql.y" -{ - yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; - - toTSDBType(yymsp[-2].minor.yy0.type); - SArray* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy430, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3989 "sql.c" - break; - case 282: /* cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ -#line 841 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, -1); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 3998 "sql.c" - break; - case 283: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ -#line 848 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4007 "sql.c" - break; - case 284: /* cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ -#line 854 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - - toTSDBType(yymsp[0].minor.yy0.type); - SArray* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4020 "sql.c" - break; - case 285: /* cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ -#line 864 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4029 "sql.c" - break; - case 286: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ -#line 871 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4038 "sql.c" - break; - case 287: /* cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ -#line 876 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - - toTSDBType(yymsp[0].minor.yy0.type); - SArray* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4051 "sql.c" - break; - case 288: /* cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ -#line 886 "sql.y" -{ - yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; - - toTSDBType(yymsp[-1].minor.yy0.type); - SArray* A = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); - - toTSDBType(yymsp[0].minor.yy0.type); - A = tVariantListAppendToken(A, &yymsp[0].minor.yy0, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4067 "sql.c" - break; - case 289: /* cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ -#line 899 "sql.y" -{ - yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; - - toTSDBType(yymsp[-2].minor.yy0.type); - SArray* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy430, -1); - - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4081 "sql.c" - break; - case 290: /* cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ -#line 910 "sql.y" -{ - yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, TSDB_SUPER_TABLE); - setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); -} -#line 4090 "sql.c" - break; - case 291: /* cmd ::= KILL CONNECTION INTEGER */ -#line 917 "sql.y" -{setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} -#line 4095 "sql.c" - break; - case 292: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ -#line 918 "sql.y" -{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} -#line 4100 "sql.c" - break; - case 293: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ -#line 919 "sql.y" -{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} -#line 4105 "sql.c" - break; - default: - break; -/********** End reduce actions ************************************************/ - }; - assert( yyrulenoYY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) ); - - /* It is not possible for a REDUCE to be followed by an error */ - assert( yyact!=YY_ERROR_ACTION ); - - yymsp += yysize+1; - yypParser->yytos = yymsp; - yymsp->stateno = (YYACTIONTYPE)yyact; - yymsp->major = (YYCODETYPE)yygoto; - yyTraceShift(yypParser, yyact, "... then shift"); - return yyact; -} - -/* -** The following code executes when the parse fails -*/ -#ifndef YYNOERRORRECOVERY -static void yy_parse_failed( - yyParser *yypParser /* The parser */ -){ - ParseARG_FETCH - ParseCTX_FETCH -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); - } -#endif - while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); - /* Here code is inserted which will be executed whenever the - ** parser fails */ -/************ Begin %parse_failure code ***************************************/ -/************ End %parse_failure code *****************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE -} -#endif /* YYNOERRORRECOVERY */ - -/* -** The following code executes when a syntax error first occurs. -*/ -static void yy_syntax_error( - yyParser *yypParser, /* The parser */ - int yymajor, /* The major type of the error token */ - ParseTOKENTYPE yyminor /* The minor type of the error token */ -){ - ParseARG_FETCH - ParseCTX_FETCH -#define TOKEN yyminor -/************ Begin %syntax_error code ****************************************/ -#line 37 "sql.y" - - pInfo->valid = false; - int32_t outputBufLen = tListLen(pInfo->msg); - int32_t len = 0; - - if(TOKEN.z) { - char msg[] = "syntax error near \"%s\""; - int32_t sqlLen = strlen(&TOKEN.z[0]); - - if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > outputBufLen) { - char tmpstr[128] = {0}; - memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1); - len = sprintf(pInfo->msg, msg, tmpstr); - } else { - len = sprintf(pInfo->msg, msg, &TOKEN.z[0]); - } - - } else { - len = sprintf(pInfo->msg, "Incomplete SQL statement"); - } - - assert(len <= outputBufLen); -#line 4190 "sql.c" -/************ End %syntax_error code ******************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE -} - -/* -** The following is executed when the parser accepts -*/ -static void yy_accept( - yyParser *yypParser /* The parser */ -){ - ParseARG_FETCH - ParseCTX_FETCH -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); - } -#endif -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - assert( yypParser->yytos==yypParser->yystack ); - /* Here code is inserted which will be executed whenever the - ** parser accepts */ -/*********** Begin %parse_accept code *****************************************/ -#line 61 "sql.y" -#line 4217 "sql.c" -/*********** End %parse_accept code *******************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE -} - -/* The main parser program. -** The first argument is a pointer to a structure obtained from -** "ParseAlloc" which describes the current state of the parser. -** The second argument is the major token number. The third is -** the minor token. The fourth optional argument is whatever the -** user wants (and specified in the grammar) and is available for -** use by the action routines. -** -** Inputs: -**
    -**
  • A pointer to the parser (an opaque structure.) -**
  • The major token number. -**
  • The minor token number. -**
  • An option argument of a grammar-specified type. -**
-** -** Outputs: -** None. -*/ -void Parse( - void *yyp, /* The parser */ - int yymajor, /* The major token code number */ - ParseTOKENTYPE yyminor /* The value for the token */ - ParseARG_PDECL /* Optional %extra_argument parameter */ -){ - YYMINORTYPE yyminorunion; - YYACTIONTYPE yyact; /* The parser action. */ -#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) - int yyendofinput; /* True if we are at the end of input */ -#endif -#ifdef YYERRORSYMBOL - int yyerrorhit = 0; /* True if yymajor has invoked an error */ -#endif - yyParser *yypParser = (yyParser*)yyp; /* The parser */ - ParseCTX_FETCH - ParseARG_STORE - - assert( yypParser->yytos!=0 ); -#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) - yyendofinput = (yymajor==0); -#endif - - yyact = yypParser->yytos->stateno; -#ifndef NDEBUG - if( yyTraceFILE ){ - if( yyact < YY_MIN_REDUCE ){ - fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", - yyTracePrompt,yyTokenName[yymajor],yyact); - }else{ - fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", - yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); - } - } -#endif - - do{ - assert( yyact==yypParser->yytos->stateno ); - yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); - if( yyact >= YY_MIN_REDUCE ){ - yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, - yyminor ParseCTX_PARAM); - }else if( yyact <= YY_MAX_SHIFTREDUCE ){ - yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt--; -#endif - break; - }else if( yyact==YY_ACCEPT_ACTION ){ - yypParser->yytos--; - yy_accept(yypParser); - return; - }else{ - assert( yyact == YY_ERROR_ACTION ); - yyminorunion.yy0 = yyminor; -#ifdef YYERRORSYMBOL - int yymx; -#endif -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); - } -#endif -#ifdef YYERRORSYMBOL - /* A syntax error has occurred. - ** The response to an error depends upon whether or not the - ** grammar defines an error token "ERROR". - ** - ** This is what we do if the grammar does define ERROR: - ** - ** * Call the %syntax_error function. - ** - ** * Begin popping the stack until we enter a state where - ** it is legal to shift the error symbol, then shift - ** the error symbol. - ** - ** * Set the error count to three. - ** - ** * Begin accepting and shifting new tokens. No new error - ** processing will occur until three tokens have been - ** shifted successfully. - ** - */ - if( yypParser->yyerrcnt<0 ){ - yy_syntax_error(yypParser,yymajor,yyminor); - } - yymx = yypParser->yytos->major; - if( yymx==YYERRORSYMBOL || yyerrorhit ){ -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sDiscard input token %s\n", - yyTracePrompt,yyTokenName[yymajor]); - } -#endif - yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); - yymajor = YYNOCODE; - }else{ - while( yypParser->yytos >= yypParser->yystack - && (yyact = yy_find_reduce_action( - yypParser->yytos->stateno, - YYERRORSYMBOL)) > YY_MAX_SHIFTREDUCE - ){ - yy_pop_parser_stack(yypParser); - } - if( yypParser->yytos < yypParser->yystack || yymajor==0 ){ - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - yy_parse_failed(yypParser); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - yymajor = YYNOCODE; - }else if( yymx!=YYERRORSYMBOL ){ - yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); - } - } - yypParser->yyerrcnt = 3; - yyerrorhit = 1; - if( yymajor==YYNOCODE ) break; - yyact = yypParser->yytos->stateno; -#elif defined(YYNOERRORRECOVERY) - /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to - ** do any kind of error recovery. Instead, simply invoke the syntax - ** error routine and continue going as if nothing had happened. - ** - ** Applications can set this macro (for example inside %include) if - ** they intend to abandon the parse upon the first syntax error seen. - */ - yy_syntax_error(yypParser,yymajor, yyminor); - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - break; -#else /* YYERRORSYMBOL is not defined */ - /* This is what we do if the grammar does not define ERROR: - ** - ** * Report an error message, and throw away the input token. - ** - ** * If the input token is $, then fail the parse. - ** - ** As before, subsequent error messages are suppressed until - ** three input tokens have been successfully shifted. - */ - if( yypParser->yyerrcnt<=0 ){ - yy_syntax_error(yypParser,yymajor, yyminor); - } - yypParser->yyerrcnt = 3; - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - if( yyendofinput ){ - yy_parse_failed(yypParser); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - } - break; -#endif - } - }while( yypParser->yytos>yypParser->yystack ); -#ifndef NDEBUG - if( yyTraceFILE ){ - yyStackEntry *i; - char cDiv = '['; - fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); - for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ - fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); - cDiv = ' '; - } - fprintf(yyTraceFILE,"]\n"); - } -#endif - return; -} - -/* -** Return the fallback token corresponding to canonical token iToken, or -** 0 if iToken has no fallback. -*/ -int ParseFallback(int iToken){ -#ifdef YYFALLBACK - assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); - return yyFallback[iToken]; -#else - (void)iToken; - return 0; -#endif -} diff --git a/2.0/src/query/tests/CMakeLists.txt b/2.0/src/query/tests/CMakeLists.txt deleted file mode 100644 index 349d511f1570e3df835494ebd4e3e86d7795c873..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) -FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64) -FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64) - -IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) - MESSAGE(STATUS "gTest library found, build unit test") - - # GoogleTest requires at least C++11 - SET(CMAKE_CXX_STANDARD 11) - - INCLUDE_DIRECTORIES(/usr/include /usr/local/include) - LINK_DIRECTORIES(/usr/lib /usr/local/lib) - - INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) - AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - - ADD_EXECUTABLE(queryTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread) -ENDIF() - -SET_SOURCE_FILES_PROPERTIES(./astTest.cpp PROPERTIES COMPILE_FLAGS -w) -SET_SOURCE_FILES_PROPERTIES(./histogramTest.cpp PROPERTIES COMPILE_FLAGS -w) -SET_SOURCE_FILES_PROPERTIES(./percentileTest.cpp PROPERTIES COMPILE_FLAGS -w) -SET_SOURCE_FILES_PROPERTIES(./resultBufferTest.cpp PROPERTIES COMPILE_FLAGS -w) -SET_SOURCE_FILES_PROPERTIES(./tsBufTest.cpp PROPERTIES COMPILE_FLAGS -w) -SET_SOURCE_FILES_PROPERTIES(./unitTest.cpp PROPERTIES COMPILE_FLAGS -w) -SET_SOURCE_FILES_PROPERTIES(./rangeMergeTest.cpp PROPERTIES COMPILE_FLAGS -w) diff --git a/2.0/src/query/tests/astTest.cpp b/2.0/src/query/tests/astTest.cpp deleted file mode 100644 index 1109d25f298d1975e15c8a8c5462445e60670811..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/astTest.cpp +++ /dev/null @@ -1,637 +0,0 @@ -#include -#include -#include -#include - -#include "texpr.h" -#include "tmsg.h" -#include "tsdb.h" -#include "tskiplist.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wunused-function" - -typedef struct ResultObj { - int32_t numOfResult; - char * resultName[64]; -} ResultObj; - -static void initSchema(SSchema *pSchema, int32_t numOfCols); - -static void initSchema_binary(SSchema *schema, int32_t numOfCols); - -static SSkipList *createSkipList(SSchema *pSchema, int32_t numOfTags); -static SSkipList *createSkipList_binary(SSchema *pSchema, int32_t numOfTags); - -static void dropMeter(SSkipList *pSkipList); - -static void Right2LeftTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList); - -static void Left2RightTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList); - -static void IllegalExprTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList); - -static void Left2RightTest_binary(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList); -static void Right2LeftTest_binary(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList); - -void setValue(ResultObj *pResult, int32_t num, char **val) { - pResult->numOfResult = num; - for (int32_t i = 0; i < num; ++i) { - pResult->resultName[i] = val[i]; - } -} - -static void initSchema_binary(SSchema *schema, int32_t numOfCols) { - schema[0].type = TSDB_DATA_TYPE_BINARY; - schema[0].bytes = 8; - strcpy(schema[0].name, "a"); - - schema[1].type = TSDB_DATA_TYPE_DOUBLE; - schema[1].bytes = 8; - strcpy(schema[1].name, "b"); - - schema[2].type = TSDB_DATA_TYPE_INT; - schema[2].bytes = 20; - strcpy(schema[2].name, "c"); - - schema[3].type = TSDB_DATA_TYPE_BIGINT; - schema[3].bytes = 8; - strcpy(schema[3].name, "d"); - - schema[4].type = TSDB_DATA_TYPE_SMALLINT; - schema[4].bytes = 2; - strcpy(schema[4].name, "e"); - - schema[5].type = TSDB_DATA_TYPE_TINYINT; - schema[5].bytes = 1; - strcpy(schema[5].name, "f"); - - schema[6].type = TSDB_DATA_TYPE_FLOAT; - schema[6].bytes = 4; - strcpy(schema[6].name, "g"); - - schema[7].type = TSDB_DATA_TYPE_BOOL; - schema[7].bytes = 1; - strcpy(schema[7].name, "h"); -} - -static void initSchema(SSchema *schema, int32_t numOfCols) { - schema[0].type = TSDB_DATA_TYPE_INT; - schema[0].bytes = 8; - strcpy(schema[0].name, "a"); - - schema[1].type = TSDB_DATA_TYPE_DOUBLE; - schema[1].bytes = 8; - strcpy(schema[1].name, "b"); - - schema[2].type = TSDB_DATA_TYPE_BINARY; - schema[2].bytes = 20; - strcpy(schema[2].name, "c"); - - schema[3].type = TSDB_DATA_TYPE_BIGINT; - schema[3].bytes = 8; - strcpy(schema[3].name, "d"); - - schema[4].type = TSDB_DATA_TYPE_SMALLINT; - schema[4].bytes = 2; - strcpy(schema[4].name, "e"); - - schema[5].type = TSDB_DATA_TYPE_TINYINT; - schema[5].bytes = 1; - strcpy(schema[5].name, "f"); - - schema[6].type = TSDB_DATA_TYPE_FLOAT; - schema[6].bytes = 4; - strcpy(schema[6].name, "g"); - - schema[7].type = TSDB_DATA_TYPE_BOOL; - schema[7].bytes = 1; - strcpy(schema[7].name, "h"); -} - -// static void addOneNode(SSchema *pSchema, int32_t tagsLen, SSkipList *pSkipList, -// char *meterId, int32_t a, double b, char *c, int64_t d, int16_t e, int8_t f, float g, -// bool h, int32_t numOfTags) { -// STabObj *pMeter = calloc(1, sizeof(STabObj)); -// pMeter->numOfTags = numOfTags; -// pMeter->pTagData = calloc(1, tagsLen + TSDB_METER_ID_LEN); -// strcpy(pMeter->meterId, meterId); -// -// char *tags = pMeter->pTagData + TSDB_METER_ID_LEN; -// int32_t offset = 0; -// -// *(int32_t *) tags = a; -// -// offset += pSchema[0].bytes; -// *(double *) (tags + offset) = b; -// -// offset += pSchema[1].bytes; -// memcpy(tags + offset, c, 3); -// -// offset += pSchema[2].bytes; -// *(int64_t *) (tags + offset) = d; -// -// offset += pSchema[3].bytes; -// *(int16_t *) (tags + offset) = e; -// -// offset += pSchema[4].bytes; -// *(int8_t *) (tags + offset) = f; -// -// offset += pSchema[5].bytes; -// *(float *) (tags + offset) = g; -// -// offset += pSchema[6].bytes; -// *(int8_t *) (tags + offset) = h ? 1 : 0; -// -// SSkipListKey pKey = SSkipListCreateKey(pSchema[0].type, tags, pSchema[0].bytes); -// SSkipListPut(pSkipList, pMeter, &pKey, 1); -//} -// -// static void addOneNode_binary(SSchema *pSchema, int32_t tagsLen, SSkipList *pSkipList, -// char *meterId, int32_t a, double b, char *c, int64_t d, int16_t e, int8_t f, float g, -// bool h, int32_t numOfTags) { -// STabObj *pMeter = calloc(1, sizeof(STabObj)); -// pMeter->numOfTags = numOfTags; -// pMeter->pTagData = calloc(1, tagsLen + TSDB_METER_ID_LEN); -// strcpy(pMeter->meterId, meterId); -// -// char *tags = pMeter->pTagData + TSDB_METER_ID_LEN; -// int32_t offset = 0; -// memcpy(tags, c, pSchema[0].bytes); -// -// offset += pSchema[0].bytes; -// *(double *) (tags + offset) = b; -// -// offset += pSchema[1].bytes; -// *(int32_t *) (tags + offset) = a; -// -// offset += pSchema[2].bytes; -// *(int64_t *) (tags + offset) = d; -// -// offset += pSchema[3].bytes; -// *(int16_t *) (tags + offset) = e; -// -// offset += pSchema[4].bytes; -// *(int8_t *) (tags + offset) = f; -// -// offset += pSchema[5].bytes; -// *(float *) (tags + offset) = g; -// -// offset += pSchema[6].bytes; -// *(int8_t *) (tags + offset) = h ? 1 : 0; -// -// SSkipListKey pKey = SSkipListCreateKey(pSchema[0].type, tags, pSchema[0].bytes); -// SSkipListPut(pSkipList, pMeter, &pKey, 1); -// SSkipListDestroyKey(&pKey); -//} - -// static void dropMeter(SSkipList *pSkipList) { -// SSkipListNode **pRes = NULL; -// int32_t num = SSkipListIterateList(pSkipList, &pRes, NULL, NULL); -// for (int32_t i = 0; i < num; ++i) { -// SSkipListNode *pNode = pRes[i]; -// STabObj *pMeter = (STabObj *) pNode->pData; -// free(pMeter->pTagData); -// free(pMeter); -// pNode->pData = NULL; -// } -// free(pRes); -//} - -// static SSkipList *createSkipList(SSchema *pSchema, int32_t numOfTags) { -// int32_t tagsLen = 0; -// for (int32_t i = 0; i < numOfTags; ++i) { -// tagsLen += pSchema[i].bytes; -// } -// -// SSkipList *pSkipList = SSkipListCreate(10, pSchema[0].type, 4); -// -// addOneNode(pSchema, tagsLen, pSkipList, "tm0\0", 0, 10.5, "abc", 1000, -10000, -20, 1.0, true, 8); -// addOneNode(pSchema, tagsLen, pSkipList, "tm1\0", 1, 20.5, "def", 1100, -10500, -30, 2.0, false, 8); -// addOneNode(pSchema, tagsLen, pSkipList, "tm2\0", 2, 30.5, "ghi", 1200, -11000, -40, 3.0, true, 8); -// addOneNode(pSchema, tagsLen, pSkipList, "tm3\0", 3, 40.5, "jkl", 1300, -11500, -50, 4.0, false, 8); -// addOneNode(pSchema, tagsLen, pSkipList, "tm4\0", 4, 50.5, "mno", 1400, -12000, -60, 5.0, true, 8); -// addOneNode(pSchema, tagsLen, pSkipList, "tm5\0", 5, 60.5, "pqr", 1500, -12500, -70, 6.0, false, 8); -// addOneNode(pSchema, tagsLen, pSkipList, "tm6\0", 6, 70.5, "stu", 1600, -13000, -80, 7.0, true, 8); -// -// return pSkipList; -//} -// -// static SSkipList *createSkipList_binary(SSchema *pSchema, int32_t numOfTags) { -// int32_t tagsLen = 0; -// for (int32_t i = 0; i < numOfTags; ++i) { -// tagsLen += pSchema[i].bytes; -// } -// -// SSkipList *pSkipList = SSkipListCreate(10, pSchema[0].type, 4); -// -// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm0\0", 0, 10.5, "abc", 1000, -10000, -20, 1.0, true, 8); -// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm1\0", 1, 20.5, "def", 1100, -10500, -30, 2.0, false, 8); -// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm2\0", 2, 30.5, "ghi", 1200, -11000, -40, 3.0, true, 8); -// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm3\0", 3, 40.5, "jkl", 1300, -11500, -50, 4.0, false, 8); -// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm4\0", 4, 50.5, "mno", 1400, -12000, -60, 5.0, true, 8); -// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm5\0", 5, 60.5, "pqr", 1500, -12500, -70, 6.0, false, 8); -// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm6\0", 6, 70.5, "stu", 1600, -13000, -80, 7.0, true, 8); -// -// return pSkipList; -//} - -//static void testQueryStr(SSchema *schema, int32_t numOfCols, char *sql, SSkipList *pSkipList, ResultObj *pResult) { -// tExprNode *pExpr = NULL; -// tSQLBinaryExprFromString(&pExpr, schema, numOfCols, sql, strlen(sql)); -// -// char str[512] = {0}; -// int32_t len = 0; -// if (pExpr == NULL) { -// printf("-----error in parse syntax:%s\n\n", sql); -// assert(pResult == NULL); -// return; -// } -// -// tSQLBinaryExprToString(pExpr, str, &len); -// printf("expr is: %s\n", str); -// -// SArray *result = NULL; -// // tExprTreeTraverse(pExpr, pSkipList, result, SSkipListNodeFilterCallback, &result); -// // printf("the result is:%lld\n", result.num); -// // -// // bool findResult = false; -// // for (int32_t i = 0; i < result.num; ++i) { -// // STabObj *pm = (STabObj *)result.pRes[i]; -// // printf("meterid:%s,\t", pm->meterId); -// // -// // for (int32_t j = 0; j < pResult->numOfResult; ++j) { -// // if (strcmp(pm->meterId, pResult->resultName[j]) == 0) { -// // findResult = true; -// // break; -// // } -// // } -// // assert(findResult == true); -// // findResult = false; -// // } -// -// printf("\n\n"); -// tExprTreeDestroy(&pExpr, NULL); -//} - -#if 0 -static void Left2RightTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) { - char str[256] = {0}; - - char *t0[1] = {"tm0"}; - char *t1[1] = {"tm1"}; - char *sql = NULL; - - ResultObj res = {1, {"tm1"}}; - testQueryStr(schema, numOfCols, "a=1", pSkipList, &res); - - char *tt[1] = {"tm6"}; - setValue(&res, 1, tt); - testQueryStr(schema, numOfCols, "a>=6", pSkipList, &res); - - setValue(&res, 1, t0); - testQueryStr(schema, numOfCols, "b<=10.6", pSkipList, &res); - - strcpy(str, "c<>'pqr'"); - char *t2[6] = {"tm0", "tm1", "tm2", "tm3", "tm4", "tm6"}; - setValue(&res, 6, t2); - testQueryStr(schema, numOfCols, str, pSkipList, &res); - - strcpy(str, "c='abc'"); - setValue(&res, 1, t0); - testQueryStr(schema, numOfCols, str, pSkipList, &res); - - char *t3[6] = {"tm1", "tm2", "tm3", "tm4", "tm5", "tm6"}; - setValue(&res, 6, t3); - testQueryStr(schema, numOfCols, "d>1050", pSkipList, &res); - - char *t4[3] = {"tm4", "tm5", "tm6"}; - setValue(&res, 3, t4); - testQueryStr(schema, numOfCols, "g>4.5980765", pSkipList, &res); - - char *t5[4] = {"tm0", "tm2", "tm4", "tm6"}; - setValue(&res, 4, t5); - testQueryStr(schema, numOfCols, "h=true", pSkipList, &res); - - char *t6[3] = {"tm1", "tm3", "tm5"}; - setValue(&res, 3, t6); - testQueryStr(schema, numOfCols, "h=0", pSkipList, &res); - - sql = "(((b<40)))\0"; - char *t7[3] = {"tm0", "tm1", "tm2"}; - setValue(&res, 3, t7); - testQueryStr(schema, numOfCols, sql, pSkipList, &res); - - sql = "((a=1) or (a=10)) or ((b=12))"; - setValue(&res, 1, t1); - testQueryStr(schema, numOfCols, sql, pSkipList, &res); - - sql = "((((((a>0 and a<2))) or a=6) or a=3) or (b=50.5)) and h=0"; - char *t8[2] = {"tm1", "tm3"}; - setValue(&res, 2, t8); - testQueryStr(schema, numOfCols, sql, pSkipList, &res); - - char *tf[1] = {"tm6"}; - setValue(&res, 1, tf); - testQueryStr(schema, numOfCols, "e = -13000", pSkipList, &res); - - char *ft[5] = {"tm0", "tm1", "tm2", "tm3", "tm4"}; - setValue(&res, 5, ft); - testQueryStr(schema, numOfCols, "f > -65", pSkipList, &res); -} - -void Right2LeftTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) { - ResultObj res = {1, {"tm1"}}; - testQueryStr(schema, numOfCols, "((1=a))", pSkipList, &res); - - char *t9[2] = {"tm0", "tm1"}; - setValue(&res, 2, t9); - testQueryStr(schema, numOfCols, "1>=a", pSkipList, &res); - - char *t0[1] = {"tm0"}; - setValue(&res, 1, t0); - testQueryStr(schema, numOfCols, "10.6>=b", pSkipList, &res); - - char *t10[3] = {"tm1", "tm3", "tm5"}; - setValue(&res, 3, t10); - testQueryStr(schema, numOfCols, "0=h", pSkipList, &res); -} - -static void IllegalExprTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) { - testQueryStr(schema, numOfCols, "h=", pSkipList, NULL); - testQueryStr(schema, numOfCols, "h<", pSkipList, NULL); - testQueryStr(schema, numOfCols, "a=1 and ", pSkipList, NULL); - testQueryStr(schema, numOfCols, "and or", pSkipList, NULL); - testQueryStr(schema, numOfCols, "and a = 1 or", pSkipList, NULL); - testQueryStr(schema, numOfCols, "(())", pSkipList, NULL); - testQueryStr(schema, numOfCols, "(", pSkipList, NULL); - testQueryStr(schema, numOfCols, "(a", pSkipList, NULL); - testQueryStr(schema, numOfCols, "(a)", pSkipList, NULL); - testQueryStr(schema, numOfCols, "())", pSkipList, NULL); - testQueryStr(schema, numOfCols, "a===1", pSkipList, NULL); - testQueryStr(schema, numOfCols, "a=1 and ", pSkipList, NULL); -} - -static void Left2RightTest_binary(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) { - char str[256] = {0}; - char *sql = NULL; - - char *t0[1] = {"tm0"}; - char *t1[1] = {"tm1"}; - - ResultObj res = {1, {"tm0"}}; - strcpy(str, "a='abc'"); - testQueryStr(schema, numOfCols, str, pSkipList, &res); - - char *tt[1] = {"tm6"}; - setValue(&res, 1, tt); - testQueryStr(schema, numOfCols, "c>=6", pSkipList, &res); - - setValue(&res, 1, t0); - testQueryStr(schema, numOfCols, "b<=10.6", pSkipList, &res); - - strcpy(str, "a<>'pqr'"); - char *t2[6] = {"tm0", "tm1", "tm2", "tm3", "tm4", "tm6"}; - setValue(&res, 6, t2); - testQueryStr(schema, numOfCols, str, pSkipList, &res); - - strcpy(str, "a='abc'"); - setValue(&res, 1, t0); - testQueryStr(schema, numOfCols, str, pSkipList, &res); - - char *t3[6] = {"tm1", "tm2", "tm3", "tm4", "tm5", "tm6"}; - setValue(&res, 6, t3); - testQueryStr(schema, numOfCols, "d>1050", pSkipList, &res); - - char *t4[3] = {"tm4", "tm5", "tm6"}; - setValue(&res, 3, t4); - testQueryStr(schema, numOfCols, "g>4.5980765", pSkipList, &res); - - char *t5[4] = {"tm0", "tm2", "tm4", "tm6"}; - setValue(&res, 4, t5); - testQueryStr(schema, numOfCols, "h=true", pSkipList, &res); - - char *t6[3] = {"tm1", "tm3", "tm5"}; - setValue(&res, 3, t6); - testQueryStr(schema, numOfCols, "h=0", pSkipList, &res); - - sql = "(((b<40)))\0"; - char *t7[3] = {"tm0", "tm1", "tm2"}; - setValue(&res, 3, t7); - testQueryStr(schema, numOfCols, sql, pSkipList, &res); - - sql = "((c=1) or (c=10)) or ((b=12))\0"; - setValue(&res, 1, t1); - testQueryStr(schema, numOfCols, sql, pSkipList, &res); - - sql = "((((((c>0 and c<2))) or c=6) or c=3) or (b=50.5)) and h=false\0"; - char *t8[2] = {"tm1", "tm3"}; - setValue(&res, 2, t8); - testQueryStr(schema, numOfCols, sql, pSkipList, &res); -} - -static void Right2LeftTest_binary(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) { - char str[256] = {0}; - char *sql = NULL; - - char *t0[1] = {"tm0"}; - char *t1[1] = {"tm1"}; - - ResultObj res = {1, {"tm0"}}; - strcpy(str, "'abc'=a"); - testQueryStr(schema, numOfCols, str, pSkipList, &res); - - char *tt[1] = {"tm6"}; - setValue(&res, 1, tt); - testQueryStr(schema, numOfCols, "6<=c", pSkipList, &res); - - setValue(&res, 1, t0); - testQueryStr(schema, numOfCols, "10.6>=b", pSkipList, &res); - - strcpy(str, "'pqr'<>a"); - char *t2[6] = {"tm0", "tm1", "tm2", "tm3", "tm4", "tm6"}; - setValue(&res, 6, t2); - testQueryStr(schema, numOfCols, str, pSkipList, &res); -} - -namespace { -// two level expression tree -tExprNode *createExpr1() { - auto *pLeft = (tExprNode*) calloc(1, sizeof(tExprNode)); - pLeft->nodeType = TEXPR_COL_NODE; - pLeft->pSchema = (SSchema*) calloc(1, sizeof(SSchema)); - - strcpy(pLeft->pSchema->name, "col_a"); - pLeft->pSchema->type = TSDB_DATA_TYPE_INT; - pLeft->pSchema->bytes = sizeof(int32_t); - pLeft->pSchema->colId = 1; - - auto *pRight = (tExprNode*) calloc(1, sizeof(tExprNode)); - pRight->nodeType = TEXPR_VALUE_NODE; - pRight->pVal = (tVariant*) calloc(1, sizeof(tVariant)); - - pRight->pVal->nType = TSDB_DATA_TYPE_INT; - pRight->pVal->i64 = 12; - - auto *pRoot = (tExprNode*) calloc(1, sizeof(tExprNode)); - pRoot->nodeType = TEXPR_NODE_EXPR; - - pRoot->_node.optr = TSDB_RELATION_EQUAL; - pRoot->_node.pLeft = pLeft; - pRoot->_node.pRight = pRight; - pRoot->_node.hasPK = true; - - return pRoot; -} - -// thress level expression tree -tExprNode* createExpr2() { - auto *pLeft2 = (tExprNode*) calloc(1, sizeof(tExprNode)); - pLeft2->nodeType = TEXPR_COL_NODE; - pLeft2->pSchema = (SSchema*) calloc(1, sizeof(SSchema)); - - strcpy(pLeft2->pSchema->name, "col_a"); - pLeft2->pSchema->type = TSDB_DATA_TYPE_BINARY; - pLeft2->pSchema->bytes = 20; - pLeft2->pSchema->colId = 1; - - auto *pRight2 = (tExprNode*) calloc(1, sizeof(tExprNode)); - pRight2->nodeType = TEXPR_VALUE_NODE; - pRight2->pVal = (tVariant*) calloc(1, sizeof(tVariant)); - - pRight2->pVal->nType = TSDB_DATA_TYPE_BINARY; - const char* v = "hello world!"; - pRight2->pVal->pz = strdup(v); - pRight2->pVal->nLen = strlen(v); - - auto *p1 = (tExprNode*) calloc(1, sizeof(tExprNode)); - p1->nodeType = TEXPR_NODE_EXPR; - - p1->_node.optr = TSDB_RELATION_LIKE; - p1->_node.pLeft = pLeft2; - p1->_node.pRight = pRight2; - p1->_node.hasPK = false; - - auto *pLeft1 = (tExprNode*) calloc(1, sizeof(tExprNode)); - pLeft1->nodeType = TEXPR_COL_NODE; - pLeft1->pSchema = (SSchema*) calloc(1, sizeof(SSchema)); - - strcpy(pLeft1->pSchema->name, "col_b"); - pLeft1->pSchema->type = TSDB_DATA_TYPE_DOUBLE; - pLeft1->pSchema->bytes = 8; - pLeft1->pSchema->colId = 99; - - auto *pRight1 = (tExprNode*) calloc(1, sizeof(tExprNode)); - pRight1->nodeType = TEXPR_VALUE_NODE; - pRight1->pVal = (tVariant*) calloc(1, sizeof(tVariant)); - - pRight1->pVal->nType = TSDB_DATA_TYPE_DOUBLE; - pRight1->pVal->dKey = 91.99; - - auto *p2 = (tExprNode*) calloc(1, sizeof(tExprNode)); - p2->nodeType = TEXPR_NODE_EXPR; - - p2->_node.optr = TSDB_RELATION_GREATER_EQUAL; - p2->_node.pLeft = pLeft1; - p2->_node.pRight = pRight1; - p2->_node.hasPK = false; - - auto *pRoot = (tExprNode*) calloc(1, sizeof(tExprNode)); - pRoot->nodeType = TEXPR_NODE_EXPR; - - pRoot->_node.optr = TSDB_RELATION_OR; - pRoot->_node.pLeft = p1; - pRoot->_node.pRight = p2; - pRoot->_node.hasPK = true; - return pRoot; -} - -void exprSerializeTest1() { - tExprNode* p1 = createExpr1(); - SBufferWriter bw = tbufInitWriter(NULL, false); - exprTreeToBinary(&bw, p1); - - size_t size = tbufTell(&bw); - ASSERT_TRUE(size > 0); - char* b = tbufGetData(&bw, false); - - tExprNode* p2 = exprTreeFromBinary(b, size); - ASSERT_EQ(p1->nodeType, p2->nodeType); - - ASSERT_EQ(p2->_node.optr, p1->_node.optr); - ASSERT_EQ(p2->_node.pLeft->nodeType, p1->_node.pLeft->nodeType); - ASSERT_EQ(p2->_node.pRight->nodeType, p1->_node.pRight->nodeType); - - SSchema* s1 = p1->_node.pLeft->pSchema; - SSchema* s2 = p2->_node.pLeft->pSchema; - - ASSERT_EQ(s2->colId, s1->colId); - ASSERT_EQ(s2->type, s1->type); - ASSERT_EQ(s2->bytes, s1->bytes); - ASSERT_STRCASEEQ(s2->name, s1->name); - - tVariant* v1 = p1->_node.pRight->pVal; - tVariant* v2 = p2->_node.pRight->pVal; - - ASSERT_EQ(v1->nType, v2->nType); - ASSERT_EQ(v1->i64, v2->i64); - ASSERT_EQ(p1->_node.hasPK, p2->_node.hasPK); - - tExprTreeDestroy(&p1, nullptr); - tExprTreeDestroy(&p2, nullptr); - - // tbufClose(&bw); -} - -void exprSerializeTest2() { - tExprNode* p1 = createExpr2(); - SBufferWriter bw = tbufInitWriter(NULL, false); - exprTreeToBinary(&bw, p1); - - size_t size = tbufTell(&bw); - ASSERT_TRUE(size > 0); - char* b = tbufGetData(&bw, false); - - tExprNode* p2 = exprTreeFromBinary(b, size); - ASSERT_EQ(p1->nodeType, p2->nodeType); - - ASSERT_EQ(p2->_node.optr, p1->_node.optr); - ASSERT_EQ(p2->_node.pLeft->nodeType, p1->_node.pLeft->nodeType); - ASSERT_EQ(p2->_node.pRight->nodeType, p1->_node.pRight->nodeType); - - tExprNode* c1Left = p1->_node.pLeft; - tExprNode* c2Left = p2->_node.pLeft; - - ASSERT_EQ(c1Left->nodeType, c2Left->nodeType); - - ASSERT_EQ(c2Left->nodeType, TEXPR_NODE_EXPR); - ASSERT_EQ(c2Left->_node.optr, TSDB_RELATION_LIKE); - - ASSERT_STRCASEEQ(c2Left->_node.pLeft->pSchema->name, "col_a"); - ASSERT_EQ(c2Left->_node.pRight->nodeType, TEXPR_VALUE_NODE); - - ASSERT_STRCASEEQ(c2Left->_node.pRight->pVal->pz, "hello world!"); - - tExprNode* c1Right = p1->_node.pRight; - tExprNode* c2Right = p2->_node.pRight; - - ASSERT_EQ(c1Right->nodeType, c2Right->nodeType); - ASSERT_EQ(c2Right->nodeType, TEXPR_NODE_EXPR); - ASSERT_EQ(c2Right->_node.optr, TSDB_RELATION_GREATER_EQUAL); - ASSERT_EQ(c2Right->_node.pRight->pVal->dKey, 91.99); - - ASSERT_EQ(p2->_node.hasPK, true); - - tExprTreeDestroy(&p1, nullptr); - tExprTreeDestroy(&p2, nullptr); - - // tbufClose(&bw); -} -} // namespace -TEST(testCase, astTest) { -// exprSerializeTest2(); -} -#endif - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/cSortTest.cpp b/2.0/src/query/tests/cSortTest.cpp deleted file mode 100644 index 64e3b41b49244b9170670fedf86e9e87a9500752..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/cSortTest.cpp +++ /dev/null @@ -1,127 +0,0 @@ -#include -#include - -#include "taos.h" -#include "tsdb.h" -#include "qExtbuffer.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wsign-compare" - -namespace { - int32_t comp(const void* p1, const void* p2) { - int32_t* x1 = (int32_t*) p1; - int32_t* x2 = (int32_t*) p2; - - if (*x1 == *x2) { - return 0; - } else { - return (*x1 > *x2)? 1:-1; - } - } - - int32_t comp1(const void* p1, const void* p2) { - int32_t ret = strncmp((char*) p1, (char*) p2, 20); - - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1:-1; - } - } -} - -TEST(testCase, colunmnwise_sort_test) { - // void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn) - void* pCols[2] = {0}; - - SSchema s[2] = {{0}}; - s[0].type = TSDB_DATA_TYPE_INT; - s[0].bytes = 4; - s[0].colId = 0; - strcpy(s[0].name, "col1"); - - s[1].type = TSDB_DATA_TYPE_BINARY; - s[1].bytes = 20; - s[1].colId = 1; - strcpy(s[1].name, "col2"); - - int32_t* p = (int32_t*) calloc(5, sizeof(int32_t)); - p[0] = 12; - p[1] = 8; - p[2] = 99; - p[3] = 7; - p[4] = 1; - - char* t1 = (char*) calloc(5, 20); - strcpy(t1, "abc"); - strcpy(t1 + 20, "def"); - strcpy(t1 + 40, "xyz"); - strcpy(t1 + 60, "klm"); - strcpy(t1 + 80, "hij"); - - pCols[0] = (char*) p; - pCols[1] = (char*) t1; - taoscQSort(reinterpret_cast(pCols), s, 2, 5, 0, comp); - - int32_t* px = (int32_t*) pCols[0]; - ASSERT_EQ(px[0], 1); - ASSERT_EQ(px[1], 7); - ASSERT_EQ(px[2], 8); - ASSERT_EQ(px[3], 12); - ASSERT_EQ(px[4], 99); - - char* px1 = (char*) pCols[1]; - ASSERT_STRCASEEQ(px1 + 20 * 0, "hij"); - ASSERT_STRCASEEQ(px1 + 20 * 1, "klm"); - ASSERT_STRCASEEQ(px1 + 20 * 2, "def"); - ASSERT_STRCASEEQ(px1 + 20 * 3, "abc"); - ASSERT_STRCASEEQ(px1 + 20 * 4, "xyz"); - - taoscQSort(pCols, s, 2, 5, 1, comp1); - px = (int32_t*) pCols[0]; - ASSERT_EQ(px[0], 12); - ASSERT_EQ(px[1], 8); - ASSERT_EQ(px[2], 1); - ASSERT_EQ(px[3], 7); - ASSERT_EQ(px[4], 99); - - px1 = (char*) pCols[1]; - ASSERT_STRCASEEQ(px1 + 20 * 0, "abc"); - ASSERT_STRCASEEQ(px1 + 20 * 1, "def"); - ASSERT_STRCASEEQ(px1 + 20 * 2, "hij"); - ASSERT_STRCASEEQ(px1 + 20 * 3, "klm"); - ASSERT_STRCASEEQ(px1 + 20 * 4, "xyz"); -} - -TEST(testCase, columnsort_test) { - SSchema field[1] = { - {TSDB_DATA_TYPE_INT, "k", sizeof(int32_t)}, - }; - - const int32_t num = 2000; - - int32_t *d = (int32_t *)malloc(sizeof(int32_t) * num); - for (int32_t i = 0; i < num; ++i) { - d[i] = i % 4; - } - - const int32_t numOfOrderCols = 1; - int32_t orderColIdx = 0; - SColumnModel *pModel = createColumnModel(field, 1, 1000); - tOrderDescriptor *pDesc = tOrderDesCreate(&orderColIdx, numOfOrderCols, pModel, 1); - - tColDataQSort(pDesc, num, 0, num - 1, (char *)d, 1); - - for (int32_t i = 0; i < num; ++i) { - printf("%d\t", d[i]); - } - printf("\n"); - - destroyColumnModel(pModel); -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/histogramTest.cpp b/2.0/src/query/tests/histogramTest.cpp deleted file mode 100644 index ff5ac08cdc065aae9bfa60e627dffa9a0cac10fb..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/histogramTest.cpp +++ /dev/null @@ -1,145 +0,0 @@ -#include -#include -#include -#include - -#include "taos.h" -#include "qHistogram.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" - -namespace { -void doHistogramAddTest() { - SHistogramInfo* pHisto = NULL; - - /** - * use arrayList, elapsed time is: - * before: - * 10,000,000 45sec, bin:1000 (-O0) / 17sec. bin:1000, (-O3) - * - * after: - * - */ - struct timeval systemTime; - gettimeofday(&systemTime, NULL); - int64_t st = - (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; - for (int32_t i = 0; i < 10000; ++i) { - tHistogramAdd(&pHisto, i); - // tHistogramPrint(pHisto); - } - // - gettimeofday(&systemTime, NULL); - int64_t et = - (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; - printf("total elapsed time: %ld\n", et - st); - - printf("elements: %d, slot:%d \n", pHisto->numOfElems, pHisto->numOfEntries); - tHistogramPrint(pHisto); - - printf("%ld\n", tHistogramSum(pHisto, 1.5)); - printf("%ld\n", tHistogramSum(pHisto, 2)); - printf("%ld\n", tHistogramSum(pHisto, 3)); - printf("%ld\n", tHistogramSum(pHisto, 4)); - printf("%ld\n", tHistogramSum(pHisto, 5)); - printf("%ld\n", tHistogramSum(pHisto, 6)); - - for (int32_t i = 399; i < 400; ++i) { - printf("val:%d, %ld\n", i, tHistogramSum(pHisto, i)); - } - - double ratio[] = {0 / 100, 20.0 / 100, 88.0 / 100, 100 / 100}; - double* res = tHistogramUniform(pHisto, ratio, 4); - for (int32_t i = 0; i < 4; ++i) { - printf("%f\n", res[i]); - } - - SHistogramInfo* pHisto1 = NULL; - for (int32_t i = (90000 - 1); i >= 80000; --i) { - tHistogramAdd(&pHisto1, i); - } - tHistogramPrint(pHisto1); - - SHistogramInfo* pRes = tHistogramMerge(pHisto1, pHisto, MAX_HISTOGRAM_BIN); - assert(pRes->numOfElems == pHisto->numOfElems + pHisto1->numOfElems); - tHistogramPrint(pRes); - - tHistogramDestroy(&pHisto); - tHistogramDestroy(&pHisto1); - tHistogramDestroy(&pRes); - free(res); -} -void doHistogramRepeatTest() { - SHistogramInfo* pHisto = NULL; - struct timeval systemTime; - - gettimeofday(&systemTime, NULL); - int64_t st = - (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; - - for (int32_t i = 0; i < 1000; ++i) { - tHistogramAdd(&pHisto, -24 + i); - // tHistogramPrint(pHisto); - } - - tHistogramDestroy(&pHisto); - -} -} - -/* test validate the names for table/database */ -TEST(testCase, histogram_binary_search) { - SHistogramInfo* pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN); - - pHisto->numOfEntries = 10; - for (int32_t i = 0; i < 10; ++i) { - pHisto->elems[i].num = 1; - pHisto->elems[i].val = i; - } - - int32_t idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 1); - assert(idx == 1); - - idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 9); - assert(idx == 9); - - idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 20); - assert(idx == 10); - - idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, -1); - assert(idx == 0); - - idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9); - assert(idx == 4); - - free(pHisto); -} - -TEST(testCase, histogram_add) { - doHistogramAddTest(); - doHistogramRepeatTest(); -} - -TEST(testCase, heapsort) { - // int32_t num = 20; - // - // SHeapEntry* pEntry = tHeapCreate(num); - // - // for(int32_t i=0; i -#include -#include -#include - -#include "qAggMain.h" -#include "tcompare.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" - -TEST(testCase, patternMatchTest) { - SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - - const char* str = "abcdef"; - int32_t ret = patternMatch("a%b%", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "tm01"; - ret = patternMatch("tm__", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "tkm1"; - ret = patternMatch("t%m1", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "tkm1"; - ret = patternMatch("%m1", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = ""; - ret = patternMatch("%_", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); - - str = "1"; - ret = patternMatch("%__", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); - - str = ""; - ret = patternMatch("%", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = " "; - ret = patternMatch("_", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "!"; - ret = patternMatch("%_", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "abcdefg"; - ret = patternMatch("abc%fg", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "abcdefgabcdeju"; - ret = patternMatch("abc%fg", str, 7, &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "abcdefgabcdeju"; - ret = patternMatch("abc%f_", str, 6, &info); - EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); - - str = "abcdefgabcdeju"; - ret = patternMatch("abc%f_", str, 1, &info); // pattern string is longe than the size - EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); - - str = "abcdefgabcdeju"; - ret = patternMatch("ab", str, 2, &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "abcdefgabcdeju"; - ret = patternMatch("a%", str, 2, &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "abcdefgabcdeju"; - ret = patternMatch("a__", str, 2, &info); - EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); - - str = "carzero"; - ret = patternMatch("%o", str, strlen(str), &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); - - str = "19"; - ret = patternMatch("%9", str, 2, &info); - EXPECT_EQ(ret, TSDB_PATTERN_MATCH); -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/percentileTest.cpp b/2.0/src/query/tests/percentileTest.cpp deleted file mode 100644 index 7c4d85b3aed5d897dca554f8091df749f228d86a..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/percentileTest.cpp +++ /dev/null @@ -1,260 +0,0 @@ -#include -#include - -#include "qResultbuf.h" -#include "taos.h" -#include "taosdef.h" - -#include "qPercentile.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" - -namespace { -tMemBucket *createBigIntDataBucket(int32_t start, int32_t end) { - tMemBucket *pBucket = tMemBucketCreate(sizeof(int64_t), TSDB_DATA_TYPE_BIGINT, start, end); - for (int32_t i = start; i <= end; ++i) { - int64_t val = i; - tMemBucketPut(pBucket, &val, 1); - } - - return pBucket; -} - -tMemBucket *createIntDataBucket(int32_t start, int32_t end) { - tMemBucket *pBucket = tMemBucketCreate(sizeof(int32_t), TSDB_DATA_TYPE_INT, start, end); - - for (int32_t i = start; i <= end; ++i) { - int32_t val = i; - tMemBucketPut(pBucket, &val, 1); - } - - return pBucket; -} - -tMemBucket *createDoubleDataBucket(int32_t start, int32_t end) { - tMemBucket *pBucket = tMemBucketCreate(sizeof(double), TSDB_DATA_TYPE_DOUBLE, start, end); - for (int32_t i = start; i <= end; ++i) { - double val = i; - int32_t ret = tMemBucketPut(pBucket, &val, 1); - if (ret != 0) { - printf("value out of range:%f", val); - } - } - - return pBucket; -} - -tMemBucket *createUnsignedDataBucket(int32_t start, int32_t end, int32_t type) { - tMemBucket *pBucket = tMemBucketCreate(tDataTypes[type].bytes, type, start, end); - for (int32_t i = start; i <= end; ++i) { - uint64_t k = i; - int32_t ret = tMemBucketPut(pBucket, &k, 1); - if (ret != 0) { - printf("value out of range:%" PRId64, k); - } - } - - return pBucket; -} - -void intDataTest() { - printf("running %s\n", __FUNCTION__); - - tMemBucket *pBucket = NULL; - double result = 0.; - - pBucket = createIntDataBucket(0, 0); - result = getPercentile(pBucket, 0); - ASSERT_DOUBLE_EQ(result, 0); - tMemBucketDestroy(pBucket); - - pBucket = createIntDataBucket(0, 1); - result = getPercentile(pBucket, 100); - ASSERT_DOUBLE_EQ(result, 1); - - result = getPercentile(pBucket, 0); - ASSERT_DOUBLE_EQ(result, 0); - tMemBucketDestroy(pBucket); - - pBucket = createIntDataBucket(-1, 1); - - result = getPercentile(pBucket, 50); - ASSERT_DOUBLE_EQ(result, 0); - - result = getPercentile(pBucket, 0); - ASSERT_DOUBLE_EQ(result, -1); - - result = getPercentile(pBucket, 75); - ASSERT_DOUBLE_EQ(result, 0.5); - - result = getPercentile(pBucket, 100); - ASSERT_DOUBLE_EQ(result, 1); - tMemBucketDestroy(pBucket); - - pBucket = createIntDataBucket(0, 99999); - result = getPercentile(pBucket, 50); - ASSERT_DOUBLE_EQ(result, 49999.5); - - tMemBucketDestroy(pBucket); -} - -void bigintDataTest() { - printf("running %s\n", __FUNCTION__); - - tMemBucket *pBucket = NULL; - double result = 0.0; - - pBucket = createBigIntDataBucket(-1000, 1000); - result = getPercentile(pBucket, 50); - ASSERT_DOUBLE_EQ(result, 0.); - tMemBucketDestroy(pBucket); - - pBucket = createBigIntDataBucket(-10000, 10000); - result = getPercentile(pBucket, 100); - ASSERT_DOUBLE_EQ(result, 10000.0); - tMemBucketDestroy(pBucket); - - pBucket = createBigIntDataBucket(-10000, 10000); - result = getPercentile(pBucket, 75); - ASSERT_DOUBLE_EQ(result, 5000.0); - - tMemBucketDestroy(pBucket); -} - -void doubleDataTest() { - printf("running %s\n", __FUNCTION__); - - tMemBucket *pBucket = NULL; - double result = 0; - - pBucket = createDoubleDataBucket(-10, 10); - result = getPercentile(pBucket, 0); - ASSERT_DOUBLE_EQ(result, -10.0); - - printf("result is: %lf\n", result); - tMemBucketDestroy(pBucket); - - pBucket = createDoubleDataBucket(-100000, 100000); - result = getPercentile(pBucket, 25); - ASSERT_DOUBLE_EQ(result, -50000); - - printf("result is: %lf\n", result); - - tMemBucketDestroy(pBucket); - - pBucket = createDoubleDataBucket(-100000, 100000); - result = getPercentile(pBucket, 50); - ASSERT_DOUBLE_EQ(result, 0); - - tMemBucketDestroy(pBucket); - - pBucket = createDoubleDataBucket(-100000, 100000); - result = getPercentile(pBucket, 75); - ASSERT_DOUBLE_EQ(result, 50000); - tMemBucketDestroy(pBucket); - - pBucket = createDoubleDataBucket(-100000, 100000); - - result = getPercentile(pBucket, 100); - ASSERT_DOUBLE_EQ(result, 100000.0); - - printf("result is: %lf\n", result); - tMemBucketDestroy(pBucket); -} - -/* - * large data test, we employ 0.1billion double data to calculated the percentile - * which is 800MB data - */ -void largeDataTest() { - printf("running : %s\n", __FUNCTION__); - - tMemBucket *pBucket = NULL; - double result = 0; - - struct timeval tv; - gettimeofday(&tv, NULL); - - int64_t start = tv.tv_sec; - printf("start time: %" PRId64 "\n", tv.tv_sec); - pBucket = createDoubleDataBucket(0, 100000000); - result = getPercentile(pBucket, 50); - ASSERT_DOUBLE_EQ(result, 50000000); - - gettimeofday(&tv, NULL); - - printf("total elapsed time: %" PRId64 " sec.", -start + tv.tv_sec); - printf("the result of %d is: %lf\n", 50, result); - tMemBucketDestroy(pBucket); -} - -void qsortTest() { - printf("running : %s\n", __FUNCTION__); - - SSchema field[1] = { - {TSDB_DATA_TYPE_INT, "k", sizeof(int32_t)}, - }; - - const int32_t num = 2000; - - int32_t *d = (int32_t *)malloc(sizeof(int32_t) * num); - for (int32_t i = 0; i < num; ++i) { - d[i] = i % 4; - } - - const int32_t numOfOrderCols = 1; - int32_t orderColIdx = 0; - SColumnModel * pModel = createColumnModel(field, 1, 1000); - tOrderDescriptor *pDesc = tOrderDesCreate(&orderColIdx, numOfOrderCols, pModel, 1); - - tColDataQSort(pDesc, num, 0, num - 1, (char *)d, 1); - - for (int32_t i = 0; i < num; ++i) { - printf("%d\t", d[i]); - } - printf("\n"); - - destroyColumnModel(pModel); -} - -void unsignedDataTest() { - printf("running %s\n", __FUNCTION__); - - tMemBucket *pBucket = NULL; - double result = 0.0; - - pBucket = createUnsignedDataBucket(0, 1000, TSDB_DATA_TYPE_UINT); - result = getPercentile(pBucket, 50); - ASSERT_DOUBLE_EQ(result, 500.0); - tMemBucketDestroy(pBucket); - - pBucket = createUnsignedDataBucket(0, 10000, TSDB_DATA_TYPE_UBIGINT); - result = getPercentile(pBucket, 100); - ASSERT_DOUBLE_EQ(result, 10000.0); - - result = getPercentile(pBucket, 0); - ASSERT_DOUBLE_EQ(result, 0.0); - - result = getPercentile(pBucket, 50); - ASSERT_DOUBLE_EQ(result, 5000); - - result = getPercentile(pBucket, 75); - ASSERT_DOUBLE_EQ(result, 7500); - tMemBucketDestroy(pBucket); - -} - -} // namespace - -TEST(testCase, percentileTest) { -// qsortTest(); - intDataTest(); - bigintDataTest(); - doubleDataTest(); - unsignedDataTest(); - largeDataTest(); -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/rangeMergeTest.cpp b/2.0/src/query/tests/rangeMergeTest.cpp deleted file mode 100644 index 9bbf47e56d1f0148682a83088dc3580e0c470b73..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/rangeMergeTest.cpp +++ /dev/null @@ -1,370 +0,0 @@ -#include -#include - -#include "qResultbuf.h" -#include "taos.h" -#include "taosdef.h" - -#include "qFilter.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" - -extern "C" { - extern void* filterInitRangeCtx(int32_t type, int32_t options); - extern int32_t filterGetRangeNum(void* h, int32_t* num); - extern int32_t filterGetRangeRes(void* h, SFilterRange *ra); - extern int32_t filterFreeRangeCtx(void* h); - extern int32_t filterAddRange(void* h, SFilterRange* ra, int32_t optr); -} - -namespace { - - -void intDataTest() { - printf("running %s\n", __FUNCTION__); - int32_t asize = 0; - SFilterRange ra[10] = {0}; - int64_t *s =NULL; - int64_t *e =NULL; - int64_t s0[3] = {-100, 1, 3}; - int64_t e0[3] = {0 , 2, 4}; - int64_t s1[3] = {INT64_MIN, 0 , 3}; - int64_t e1[3] = {100 , 50, 4}; - int64_t s2[5] = {1 , 3 , 10,30,70}; - int64_t e2[5] = {10, 100, 20,50,120}; - int64_t s3[3] = {1 , 20 , 5}; - int64_t e3[3] = {10, 100, 25}; - int64_t s4[2] = {10, 0}; - int64_t e4[2] = {20, 5}; - int64_t s5[3] = {0, 6 ,7}; - int64_t e5[3] = {4, 10,20}; - - int64_t rs[10]; - int64_t re[10]; - - int32_t num = 0; - void *h = NULL; - - s = s0; - e = e0; - asize = sizeof(s0)/sizeof(s[0]); - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - filterAddRange(h, ra, TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 0); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 3); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, -100); - ASSERT_EQ(ra[0].e, 0); - ASSERT_EQ(ra[1].s, 1); - ASSERT_EQ(ra[1].e, 2); - ASSERT_EQ(ra[2].s, 3); - ASSERT_EQ(ra[2].e, 4); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, FI_OPTION_TIMESTAMP); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, -100); - ASSERT_EQ(ra[0].e, 4); - filterFreeRangeCtx(h); - - - s = s1; - e = e1; - asize = sizeof(s1)/sizeof(s[0]); - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, 3); - ASSERT_EQ(ra[0].e, 4); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, INT64_MIN); - ASSERT_EQ(ra[0].e, 100); - filterFreeRangeCtx(h); - - - - s = s2; - e = e2; - asize = sizeof(s2)/sizeof(s[0]); - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 0); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, 1); - ASSERT_EQ(ra[0].e, 120); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, i % 2 ? TSDB_RELATION_OR : TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 0); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, i % 2 ? TSDB_RELATION_AND : TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, 70); - ASSERT_EQ(ra[0].e, 120); - filterFreeRangeCtx(h); - - - s = s3; - e = e3; - asize = sizeof(s3)/sizeof(s[0]); - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 0); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, 1); - ASSERT_EQ(ra[0].e, 100); - filterFreeRangeCtx(h); - - - - - s = s4; - e = e4; - asize = sizeof(s4)/sizeof(s[0]); - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 0); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 2); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, 0); - ASSERT_EQ(ra[0].e, 5); - ASSERT_EQ(ra[1].s, 10); - ASSERT_EQ(ra[1].e, 20); - filterFreeRangeCtx(h); - - - s = s5; - e = e5; - asize = sizeof(s5)/sizeof(s[0]); - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 0); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 2); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, 0); - ASSERT_EQ(ra[0].e, 4); - ASSERT_EQ(ra[1].s, 6); - ASSERT_EQ(ra[1].e, 20); - filterFreeRangeCtx(h); - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].s = s[i]; - ra[0].e = e[i]; - - filterAddRange(h, ra, (i == (asize -1)) ? TSDB_RELATION_AND : TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterGetRangeRes(h, ra); - ASSERT_EQ(ra[0].s, 7); - ASSERT_EQ(ra[0].e, 10); - filterFreeRangeCtx(h); - - - - int64_t s6[2] = {0, 4}; - int64_t e6[2] = {4, 6}; - s = s6; - e = e6; - asize = sizeof(s6)/sizeof(s[0]); - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].eflag = 1; - ra[1].sflag = 4; - - ra[i].s = s[i]; - ra[i].e = e[i]; - - filterAddRange(h, ra + i, TSDB_RELATION_AND); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 1); - filterFreeRangeCtx(h); - - - - memset(ra, 0, sizeof(ra)); - h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); - for (int32_t i = 0; i < asize; ++i) { - ra[0].eflag = 1; - ra[1].sflag = 1; - - ra[i].s = s[i]; - ra[i].e = e[i]; - - filterAddRange(h, ra + i, TSDB_RELATION_OR); - } - filterGetRangeNum(h, &num); - ASSERT_EQ(num, 2); - ASSERT_EQ(ra[0].s, 0); - ASSERT_EQ(ra[0].e, 4); - ASSERT_EQ(ra[0].eflag, 1); - ASSERT_EQ(ra[1].s, 4); - ASSERT_EQ(ra[1].e, 6); - ASSERT_EQ(ra[1].sflag, 1); - filterFreeRangeCtx(h); - -} - - -} // namespace - -TEST(testCase, rangeMergeTest) { - intDataTest(); - -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/resultBufferTest.cpp b/2.0/src/query/tests/resultBufferTest.cpp deleted file mode 100644 index b7173ece46c4d8098df8e45ba34e6f95e5f8a218..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/resultBufferTest.cpp +++ /dev/null @@ -1,166 +0,0 @@ -#include -#include -#include - -#include "qResultbuf.h" -#include "taos.h" -#include "tsdb.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" - -namespace { -// simple test -void simpleTest() { - SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4096, 1); - - int32_t pageId = 0; - int32_t groupId = 0; - - tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); - ASSERT_TRUE(pBufPage != NULL); - - ASSERT_EQ(getTotalBufSize(pResultBuf), 1024); - - SIDList list = getDataBufPagesIdList(pResultBuf, groupId); - ASSERT_EQ(taosArrayGetSize(list), 1); - ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); - - releaseBufPage(pResultBuf, pBufPage); - - tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); - - tFilePage* t = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t == pBufPage1); - - tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t1 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t1 == pBufPage2); - - tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t2 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t2 == pBufPage3); - - tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t3 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t3 == pBufPage4); - - tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t4 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t4 == pBufPage5); - - destroyResultBuf(pResultBuf); -} - -void writeDownTest() { - SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, 1); - - int32_t pageId = 0; - int32_t writePageId = 0; - int32_t groupId = 0; - int32_t nx = 12345; - - tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); - ASSERT_TRUE(pBufPage != NULL); - - *(int32_t*)(pBufPage->data) = nx; - writePageId = pageId; - releaseBufPage(pResultBuf, pBufPage); - - tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t1 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t1 == pBufPage1); - ASSERT_TRUE(pageId == 1); - - tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t2 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t2 == pBufPage2); - ASSERT_TRUE(pageId == 2); - - tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t3 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t3 == pBufPage3); - ASSERT_TRUE(pageId == 3); - - tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t4 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t4 == pBufPage4); - ASSERT_TRUE(pageId == 4); - releaseBufPage(pResultBuf, t4); - - // flush the written page to disk, and read it out again - tFilePage* pBufPagex = getBufPage(pResultBuf, writePageId); - ASSERT_EQ(*(int32_t*)pBufPagex->data, nx); - - SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); - ASSERT_EQ(taosArrayGetSize(pa), 5); - - destroyResultBuf(pResultBuf); -} - -void recyclePageTest() { - SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, 1); - - int32_t pageId = 0; - int32_t writePageId = 0; - int32_t groupId = 0; - int32_t nx = 12345; - - tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); - ASSERT_TRUE(pBufPage != NULL); - releaseBufPage(pResultBuf, pBufPage); - - tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t1 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t1 == pBufPage1); - ASSERT_TRUE(pageId == 1); - - tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t2 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t2 == pBufPage2); - ASSERT_TRUE(pageId == 2); - - tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t3 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t3 == pBufPage3); - ASSERT_TRUE(pageId == 3); - - tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t4 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t4 == pBufPage4); - ASSERT_TRUE(pageId == 4); - releaseBufPage(pResultBuf, t4); - - tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t5 = getBufPage(pResultBuf, pageId); - ASSERT_TRUE(t5 == pBufPage5); - ASSERT_TRUE(pageId == 5); - - // flush the written page to disk, and read it out again - tFilePage* pBufPagex = getBufPage(pResultBuf, writePageId); - *(int32_t*)(pBufPagex->data) = nx; - writePageId = pageId; // update the data - releaseBufPage(pResultBuf, pBufPagex); - - tFilePage* pBufPagex1 = getBufPage(pResultBuf, 1); - - SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); - ASSERT_EQ(taosArrayGetSize(pa), 6); - - destroyResultBuf(pResultBuf); -} -} // namespace - - -TEST(testCase, resultBufferTest) { - srand(time(NULL)); - simpleTest(); - writeDownTest(); - recyclePageTest(); -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/tsBufTest.cpp b/2.0/src/query/tests/tsBufTest.cpp deleted file mode 100644 index 62ffa785c65a3f9e2a47383bd88519e170bf4989..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/tsBufTest.cpp +++ /dev/null @@ -1,518 +0,0 @@ -#include "os.h" -#include -#include -#include - -#include "qTsbuf.h" -#include "taos.h" -#include "tsdb.h" -#include "ttoken.h" -#include "tutil.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" - -namespace { -/** - * - * @param num total number - * @param step gap between two consecutive ts - * @return - */ -int64_t* createTsList(int32_t num, int64_t start, int32_t step) { - int64_t* pList = (int64_t*)malloc(num * sizeof(int64_t)); - - for (int64_t i = 0; i < num; ++i) { - pList[i] = start + i * step; - } - - return pList; -} - -// simple test -void simpleTest() { - STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); - - // write 10 ts points - int32_t num = 10; - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - t.i64 = 1; - - int64_t* list = createTsList(10, 10000000, 30); - tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); - - EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); - EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); - EXPECT_EQ(pTSBuf->numOfGroups, 1); - - tsBufFlush(pTSBuf); - EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.numOfElem, num); - - tsBufDestroy(pTSBuf); - - free(list); -} - -// one large list of ts, the ts list need to be split into several small blocks -void largeTSTest() { - STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); - - // write 10 ts points - int32_t num = 1000000; - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - t.i64 = 1; - - int64_t* list = createTsList(num, 10000000, 30); - tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); - - // the data has been flush to disk, no data in cache - EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); - EXPECT_EQ(pTSBuf->numOfGroups, 1); - EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); - - tsBufFlush(pTSBuf); - EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.numOfElem, num); - - tsBufDestroy(pTSBuf); - free(list); -} - -void multiTagsTest() { - STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); - - int32_t num = 10000; - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - - int64_t start = 10000000; - int32_t numOfTags = 50; - int32_t step = 30; - - for (int32_t i = 0; i < numOfTags; ++i) { - int64_t* list = createTsList(num, start, step); - t.i64 = i; - - tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); - free(list); - - start += step * num; - } - - EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); - EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - - EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); - EXPECT_EQ(pTSBuf->numOfGroups, 1); - - tsBufFlush(pTSBuf); - EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.numOfElem, num); - - tsBufDestroy(pTSBuf); -} - -void multiVnodeTagsTest() { - STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); - - int32_t num = 10000; - int64_t start = 10000000; - int32_t numOfTags = 50; - int32_t step = 30; - - // 2000 vnodes - for (int32_t j = 0; j < 20; ++j) { - // vnodeId:0 - start = 10000000; - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - - for (int32_t i = 0; i < numOfTags; ++i) { - int64_t* list = createTsList(num, start, step); - t.i64 = i; - - tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); - free(list); - - start += step * num; - } - - EXPECT_EQ(pTSBuf->numOfGroups, j + 1); - } - - EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); - EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); - - EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - - EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); - - tsBufFlush(pTSBuf); - EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.numOfElem, num); - - tsBufDestroy(pTSBuf); -} - -void loadDataTest() { - STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); - - int32_t num = 10000; - int64_t oldStart = 10000000; - int32_t numOfTags = 50; - int32_t step = 30; - int32_t numOfVnode = 200; - - // 10000 vnodes - for (int32_t j = 0; j < numOfVnode; ++j) { - // vnodeId:0 - int64_t start = 10000000; - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - - for (int32_t i = 0; i < numOfTags; ++i) { - int64_t* list = createTsList(num, start, step); - t.i64 = i; - - tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); - printf("%d - %" PRIu64 "\n", i, list[0]); - - free(list); - start += step * num; - } - - EXPECT_EQ(pTSBuf->numOfGroups, j + 1); - } - - EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); - - EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); - - EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - - EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1); - - tsBufFlush(pTSBuf); - EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.numOfElem, num); - - // create from exists file - STSBuf* pNewBuf = tsBufCreateFromFile(pTSBuf->path, false); - EXPECT_EQ(pNewBuf->tsOrder, pTSBuf->tsOrder); - EXPECT_EQ(pNewBuf->numOfGroups, numOfVnode); - EXPECT_EQ(pNewBuf->fileSize, pTSBuf->fileSize); - - EXPECT_EQ(pNewBuf->pData[0].info.offset, pTSBuf->pData[0].info.offset); - EXPECT_EQ(pNewBuf->pData[0].info.numOfBlocks, pTSBuf->pData[0].info.numOfBlocks); - EXPECT_EQ(pNewBuf->pData[0].info.compLen, pTSBuf->pData[0].info.compLen); - - EXPECT_STREQ(pNewBuf->path, pTSBuf->path); - - tsBufResetPos(pNewBuf); - - int64_t s = taosGetTimestampUs(); - printf("start:%" PRIu64 "\n", s); - - int32_t x = 0; - while (tsBufNextPos(pNewBuf)) { - STSElem elem = tsBufGetElem(pNewBuf); - if (++x == 100000000) { - break; - } - - // printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); - } - - int64_t e = taosGetTimestampUs(); - printf("end:%" PRIu64 ", elapsed:%" PRIu64 ", total obj:%d\n", e, e - s, x); - tsBufDestroy(pTSBuf); - tsBufDestroy(pNewBuf); -} - -void randomIncTsTest() {} - -void TSTraverse() { - // 10000 vnodes - int32_t num = 200000; - int64_t oldStart = 10000000; - int32_t numOfTags = 3; - int32_t step = 30; - int32_t numOfVnode = 2; - - STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); - - for (int32_t j = 0; j < numOfVnode; ++j) { - // vnodeId:0 - int64_t start = 10000000; - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - - for (int32_t i = 0; i < numOfTags; ++i) { - int64_t* list = createTsList(num, start, step); - t.i64 = i; - - tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); - printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); - - free(list); - start += step * num; - - list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); - printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); - free(list); - - start += step * num; - } - - EXPECT_EQ(pTSBuf->numOfGroups, j + 1); - } - - tsBufResetPos(pTSBuf); - - //////////////////////////////////////////////////////////////////////////////////////// - // reverse traverse - int64_t s = taosGetTimestampUs(); - printf("start:%" PRIu64 "\n", s); - - pTSBuf->cur.order = TSDB_ORDER_DESC; - - // complete reverse traverse - int32_t x = 0; - while (tsBufNextPos(pTSBuf)) { - STSElem elem = tsBufGetElem(pTSBuf); - // printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); - } - - // specify the data block with vnode and tags value - tsBufResetPos(pTSBuf); - pTSBuf->cur.order = TSDB_ORDER_DESC; - - int32_t startVnode = 1; - int32_t startTag = 2; - - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - t.i64 = startTag; - - tsBufGetElemStartPos(pTSBuf, startVnode, &t); - - int32_t totalOutput = 10; - while (1) { - STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64, elem.ts); - - if (!tsBufNextPos(pTSBuf)) { - break; - } - - if (--totalOutput <= 0) { - totalOutput = 10; - - startTag -= 1; - t.i64 = startTag; - tsBufGetElemStartPos(pTSBuf, startVnode, &t); - - if (startTag == 0) { - startVnode -= 1; - startTag = 3; - } - - if (startVnode < 0) { - break; - } - } - } - - ///////////////////////////////////////////////////////////////////////////////// - // traverse - pTSBuf->cur.order = TSDB_ORDER_ASC; - tsBufResetPos(pTSBuf); - - // complete forwards traverse - while (tsBufNextPos(pTSBuf)) { - STSElem elem = tsBufGetElem(pTSBuf); - // printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); - } - - // specify the data block with vnode and tags value - tsBufResetPos(pTSBuf); - pTSBuf->cur.order = TSDB_ORDER_ASC; - - startVnode = 1; - startTag = 2; - t.i64 = startTag; - - tsBufGetElemStartPos(pTSBuf, startVnode, &t); - - totalOutput = 10; - while (1) { - STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64, elem.ts); - - if (!tsBufNextPos(pTSBuf)) { - break; - } - - if (--totalOutput <= 0) { - totalOutput = 10; - - startTag -= 1; - t.i64 = startTag; - tsBufGetElemStartPos(pTSBuf, startVnode, &t); - - if (startTag < 0) { - startVnode -= 1; - startTag = 3; - } - - if (startVnode < 0) { - break; - } - } - } - - tsBufDestroy(pTSBuf); -} - -void performanceTest() {} - -void emptyTagTest() {} - -void invalidFileTest() { - const char* cmd = "touch /tmp/test"; - - // create empty file - system(cmd); - - STSBuf* pNewBuf = tsBufCreateFromFile("/tmp/test", true); - EXPECT_TRUE(pNewBuf == NULL); - tsBufDestroy(pNewBuf); - - pNewBuf = tsBufCreateFromFile("/tmp/911", true); - EXPECT_TRUE(pNewBuf == NULL); - - tsBufDestroy(pNewBuf); -} - -void mergeDiffVnodeBufferTest() { - STSBuf* pTSBuf1 = tsBufCreate(true, TSDB_ORDER_ASC); - STSBuf* pTSBuf2 = tsBufCreate(true, TSDB_ORDER_ASC); - - int32_t step = 30; - int32_t num = 1000; - int32_t numOfTags = 10; - - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - - // vnodeId:0 - int64_t start = 10000000; - for (int32_t i = 0; i < numOfTags; ++i) { - int64_t* list = createTsList(num, start, step); - t.i64 = i; - - tsBufAppend(pTSBuf1, 1, &t, (const char*)list, num * sizeof(int64_t)); - tsBufAppend(pTSBuf2, 9, &t, (const char*)list, num * sizeof(int64_t)); - - free(list); - - start += step * num; - } - - tsBufFlush(pTSBuf2); - - tsBufMerge(pTSBuf1, pTSBuf2); - EXPECT_EQ(pTSBuf1->numOfGroups, 2); - EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); - - tsBufDisplay(pTSBuf1); - - tsBufDestroy(pTSBuf2); - tsBufDestroy(pTSBuf1); -} - -void mergeIdenticalVnodeBufferTest() { - STSBuf* pTSBuf1 = tsBufCreate(true, TSDB_ORDER_ASC); - STSBuf* pTSBuf2 = tsBufCreate(true, TSDB_ORDER_ASC); - - tVariant t = {0}; - t.nType = TSDB_DATA_TYPE_BIGINT; - - int32_t step = 30; - int32_t num = 1000; - int32_t numOfTags = 10; - - // vnodeId:0 - int64_t start = 10000000; - for (int32_t i = 0; i < numOfTags; ++i) { - int64_t* list = createTsList(num, start, step); - t.i64 = i; - - tsBufAppend(pTSBuf1, 12, &t, (const char*)list, num * sizeof(int64_t)); - free(list); - - start += step * num; - } - - for (int32_t i = numOfTags; i < numOfTags * 2; ++i) { - int64_t* list = createTsList(num, start, step); - - t.i64 = i; - tsBufAppend(pTSBuf2, 77, &t, (const char*)list, num * sizeof(int64_t)); - free(list); - - start += step * num; - } - - tsBufFlush(pTSBuf2); - - tsBufMerge(pTSBuf1, pTSBuf2); - EXPECT_EQ(pTSBuf1->numOfGroups, 2); - EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); - - tsBufResetPos(pTSBuf1); - - int32_t count = 0; - while (tsBufNextPos(pTSBuf1)) { - STSElem elem = tsBufGetElem(pTSBuf1); - - if (count++ < numOfTags * num) { - EXPECT_EQ(elem.id, 12); - } else { - EXPECT_EQ(elem.id, 77); - } - - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64, elem.ts); - } - - tsBufDestroy(pTSBuf1); - tsBufDestroy(pTSBuf2); -} -} // namespace - - -//TODO add binary tag value test case -TEST(testCase, tsBufTest) { - simpleTest(); - largeTSTest(); - multiTagsTest(); - multiVnodeTagsTest(); - loadDataTest(); - invalidFileTest(); -// randomIncTsTest(); - TSTraverse(); - mergeDiffVnodeBufferTest(); - mergeIdenticalVnodeBufferTest(); -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/unitTest.cpp b/2.0/src/query/tests/unitTest.cpp deleted file mode 100644 index 57f66ef466503fca7e9398eecd0af89d67e81375..0000000000000000000000000000000000000000 --- a/2.0/src/query/tests/unitTest.cpp +++ /dev/null @@ -1,914 +0,0 @@ -#include -#include - -#include "taos.h" -#include "tsdb.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#pragma GCC diagnostic ignored "-Wsign-compare" - -#include "../../client/inc/tscUtil.h" -#include "tutil.h" -#include "tvariant.h" -#include "ttokendef.h" - -namespace { -int32_t testValidateName(char* name) { - SStrToken token = {0}; - token.z = name; - token.n = strlen(name); - token.type = 0; - - tGetToken(name, &token.type); - return tscValidateName(&token); -} -} - -static void _init_tvariant_bool(tVariant* t) { - t->i64 = TSDB_FALSE; - t->nType = TSDB_DATA_TYPE_BOOL; -} - -static void _init_tvariant_tinyint(tVariant* t) { - t->i64 = -27; - t->nType = TSDB_DATA_TYPE_TINYINT; -} - -static void _init_tvariant_int(tVariant* t) { - t->i64 = -23997659; - t->nType = TSDB_DATA_TYPE_INT; -} - -static void _init_tvariant_bigint(tVariant* t) { - t->i64 = -3333333333333; - t->nType = TSDB_DATA_TYPE_BIGINT; -} - -static void _init_tvariant_float(tVariant* t) { - t->dKey = -8991212199.8987878776; - t->nType = TSDB_DATA_TYPE_FLOAT; -} - -static void _init_tvariant_binary(tVariant* t) { - tVariantDestroy(t); - - t->pz = (char*)calloc(1, 20); //"2e3"); - t->nType = TSDB_DATA_TYPE_BINARY; - strcpy(t->pz, "2e5"); - t->nLen = strlen(t->pz); -} - -static void _init_tvariant_nchar(tVariant* t) { - tVariantDestroy(t); - - t->wpz = (wchar_t*)calloc(1, 20 * TSDB_NCHAR_SIZE); - t->nType = TSDB_DATA_TYPE_NCHAR; - wcscpy(t->wpz, L"-2000000.8765"); - t->nLen = twcslen(t->wpz); -} - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -/* test validate the names for table/database */ -TEST(testCase, db_table_name) { - - char t01[] = "abc"; - EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS); - - char t02[] = "'abc'"; - EXPECT_EQ(testValidateName(t02), TSDB_CODE_SUCCESS); - - char t1[] = "abc.def"; - EXPECT_EQ(testValidateName(t1), TSDB_CODE_SUCCESS); - printf("%s\n", t1); - - char t2[] = "'abc.def'"; - EXPECT_EQ(testValidateName(t2), TSDB_CODE_SUCCESS); - printf("%s\n", t2); - - char t3[] = "'abc'.def"; - EXPECT_EQ(testValidateName(t3), TSDB_CODE_SUCCESS); - printf("%s\n", t3); - - char t4[] = "'abc'.'def'"; - EXPECT_EQ(testValidateName(t4), TSDB_CODE_SUCCESS); - - char t5[] = "table.'def'"; - EXPECT_EQ(testValidateName(t5), TSDB_CODE_TSC_INVALID_OPERATION); - - char t6[] = "'table'.'def'"; - EXPECT_EQ(testValidateName(t6), TSDB_CODE_TSC_INVALID_OPERATION); - - char t7[] = "'_ab1234'.'def'"; - EXPECT_EQ(testValidateName(t7), TSDB_CODE_SUCCESS); - printf("%s\n", t7); - - char t8[] = "'_ab&^%1234'.'def'"; - EXPECT_EQ(testValidateName(t8), TSDB_CODE_TSC_INVALID_OPERATION); - - char t9[] = "'_123'.'gtest中文'"; - EXPECT_EQ(testValidateName(t9), TSDB_CODE_TSC_INVALID_OPERATION); - - char t10[] = "abc.'gtest中文'"; - EXPECT_EQ(testValidateName(t10), TSDB_CODE_TSC_INVALID_OPERATION); - - char t10_1[] = "abc.'中文gtest'"; - EXPECT_EQ(testValidateName(t10_1), TSDB_CODE_TSC_INVALID_OPERATION); - - char t11[] = "'192.168.0.1'.abc"; - EXPECT_EQ(testValidateName(t11), TSDB_CODE_TSC_INVALID_OPERATION); - - char t12[] = "192.168.0.1.abc"; - EXPECT_EQ(testValidateName(t12), TSDB_CODE_TSC_INVALID_OPERATION); - - char t13[] = "abc."; - EXPECT_EQ(testValidateName(t13), TSDB_CODE_TSC_INVALID_OPERATION); - - char t14[] = ".abc"; - EXPECT_EQ(testValidateName(t14), TSDB_CODE_TSC_INVALID_OPERATION); - - char t15[] = ".'abc'"; - EXPECT_EQ(testValidateName(t15), TSDB_CODE_TSC_INVALID_OPERATION); - - char t16[] = ".abc'"; - EXPECT_EQ(testValidateName(t16), TSDB_CODE_TSC_INVALID_OPERATION); - - char t17[] = "123a.\"abc\""; - EXPECT_EQ(testValidateName(t17), TSDB_CODE_TSC_INVALID_OPERATION); - printf("%s\n", t17); - - char t18[] = "a.\"abc\""; - EXPECT_EQ(testValidateName(t18), TSDB_CODE_SUCCESS); - printf("%s\n", t18); - - char t19[] = "'_ab1234'.'def'.'ab123'"; - EXPECT_EQ(testValidateName(t19), TSDB_CODE_TSC_INVALID_OPERATION); - - char t20[] = "'_ab1234*&^'"; - EXPECT_EQ(testValidateName(t20), TSDB_CODE_TSC_INVALID_OPERATION); - - char t21[] = "'1234_abc'"; - EXPECT_EQ(testValidateName(t21), TSDB_CODE_TSC_INVALID_OPERATION); - - - // =======Containing capital letters================= - char t30[] = "ABC"; - EXPECT_EQ(testValidateName(t30), TSDB_CODE_SUCCESS); - - char t31[] = "'ABC'"; - EXPECT_EQ(testValidateName(t31), TSDB_CODE_SUCCESS); - - char t32[] = "ABC.def"; - EXPECT_EQ(testValidateName(t32), TSDB_CODE_SUCCESS); - - char t33[] = "'ABC.def"; - EXPECT_EQ(testValidateName(t33), TSDB_CODE_TSC_INVALID_OPERATION); - - char t33_0[] = "abc.DEF'"; - EXPECT_EQ(testValidateName(t33_0), TSDB_CODE_TSC_INVALID_OPERATION); - - char t34[] = "'ABC.def'"; - //int32_t tmp0 = testValidateName(t34); - EXPECT_EQ(testValidateName(t34), TSDB_CODE_SUCCESS); - - char t35[] = "'ABC'.def"; - EXPECT_EQ(testValidateName(t35), TSDB_CODE_SUCCESS); - - char t36[] = "'ABC'.'DEF'"; - EXPECT_EQ(testValidateName(t36), TSDB_CODE_SUCCESS); - - char t37[] = "abc.'DEF'"; - EXPECT_EQ(testValidateName(t37), TSDB_CODE_SUCCESS); - - char t37_1[] = "abc.'_123DEF'"; - EXPECT_EQ(testValidateName(t37_1), TSDB_CODE_SUCCESS); - - char t38[] = "'abc'.'DEF'"; - EXPECT_EQ(testValidateName(t38), TSDB_CODE_SUCCESS); - - // do not use key words - char t39[] = "table.'DEF'"; - EXPECT_EQ(testValidateName(t39), TSDB_CODE_TSC_INVALID_OPERATION); - - char t40[] = "'table'.'DEF'"; - EXPECT_EQ(testValidateName(t40), TSDB_CODE_TSC_INVALID_OPERATION); - - char t41[] = "'_abXYZ1234'.'deFF'"; - EXPECT_EQ(testValidateName(t41), TSDB_CODE_SUCCESS); - - char t42[] = "'_abDEF&^%1234'.'DIef'"; - EXPECT_EQ(testValidateName(t42), TSDB_CODE_TSC_INVALID_OPERATION); - - char t43[] = "'_123'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t43), TSDB_CODE_TSC_INVALID_OPERATION); - - char t44[] = "'aABC'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t44), TSDB_CODE_TSC_INVALID_OPERATION); - - char t45[] = "'ABC'."; - EXPECT_EQ(testValidateName(t45), TSDB_CODE_TSC_INVALID_OPERATION); - - char t46[] = ".'ABC'"; - EXPECT_EQ(testValidateName(t46), TSDB_CODE_TSC_INVALID_OPERATION); - - char t47[] = "a.\"aTWc\""; - EXPECT_EQ(testValidateName(t47), TSDB_CODE_SUCCESS); - - // ================has space ================= - char t60[] = " ABC "; - EXPECT_EQ(testValidateName(t60), TSDB_CODE_TSC_INVALID_OPERATION); - - char t60_1[] = " ABC "; - EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_TSC_INVALID_OPERATION); - - char t61[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61), TSDB_CODE_TSC_INVALID_OPERATION); - - char t61_1[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_TSC_INVALID_OPERATION); - - char t62[] = " ABC . def "; - EXPECT_EQ(testValidateName(t62), TSDB_CODE_TSC_INVALID_OPERATION); - - char t63[] = "' ABC . def "; - EXPECT_EQ(testValidateName(t63), TSDB_CODE_TSC_INVALID_OPERATION); - - char t63_0[] = " abc . DEF ' "; - EXPECT_EQ(testValidateName(t63_0), TSDB_CODE_TSC_INVALID_OPERATION); - - char t64[] = " ' ABC . def ' "; - //int32_t tmp1 = testValidateName(t64); - EXPECT_EQ(testValidateName(t64), TSDB_CODE_TSC_INVALID_OPERATION); - - char t65[] = " ' ABC '. def "; - EXPECT_EQ(testValidateName(t65), TSDB_CODE_TSC_INVALID_OPERATION); - - char t66[] = "' ABC '.' DEF '"; - EXPECT_EQ(testValidateName(t66), TSDB_CODE_TSC_INVALID_OPERATION); - - char t67[] = "abc . ' DEF '"; - EXPECT_EQ(testValidateName(t67), TSDB_CODE_TSC_INVALID_OPERATION); - - char t68[] = "' abc '.' DEF '"; - EXPECT_EQ(testValidateName(t68), TSDB_CODE_TSC_INVALID_OPERATION); - - // do not use key words - char t69[] = "table.'DEF'"; - EXPECT_EQ(testValidateName(t69), TSDB_CODE_TSC_INVALID_OPERATION); - - char t70[] = "'table'.'DEF'"; - EXPECT_EQ(testValidateName(t70), TSDB_CODE_TSC_INVALID_OPERATION); - - char t71[] = "'_abXYZ1234 '.' deFF '"; - EXPECT_EQ(testValidateName(t71), TSDB_CODE_TSC_INVALID_OPERATION); - - char t72[] = "'_abDEF&^%1234'.' DIef'"; - EXPECT_EQ(testValidateName(t72), TSDB_CODE_TSC_INVALID_OPERATION); - - char t73[] = "'_123'.' Gtest中文'"; - EXPECT_EQ(testValidateName(t73), TSDB_CODE_TSC_INVALID_OPERATION); - - char t74[] = "' aABC'.'Gtest中文'"; - EXPECT_EQ(testValidateName(t74), TSDB_CODE_TSC_INVALID_OPERATION); - - char t75[] = "' ABC '."; - EXPECT_EQ(testValidateName(t75), TSDB_CODE_TSC_INVALID_OPERATION); - - char t76[] = ".' ABC'"; - EXPECT_EQ(testValidateName(t76), TSDB_CODE_TSC_INVALID_OPERATION); - - char t77[] = " a . \"aTWc\" "; - EXPECT_EQ(testValidateName(t77), TSDB_CODE_TSC_INVALID_OPERATION); - - char t78[] = " a.\"aTWc \""; - EXPECT_EQ(testValidateName(t78), TSDB_CODE_TSC_INVALID_OPERATION); - - - // ===============muti string by space =================== - // There's no such case. - //char t160[] = "A BC"; - //EXPECT_EQ(testValidateName(t160), TSDB_CODE_TSC_INVALID_OPERATION); - //printf("end:%s\n", t160); - - // There's no such case. - //char t161[] = "' A BC '"; - //EXPECT_EQ(testValidateName(t161), TSDB_CODE_TSC_INVALID_OPERATION); - - char t162[] = " AB C . de f "; - EXPECT_EQ(testValidateName(t162), TSDB_CODE_TSC_INVALID_OPERATION); - - char t163[] = "' AB C . de f "; - EXPECT_EQ(testValidateName(t163), TSDB_CODE_TSC_INVALID_OPERATION); - - char t163_0[] = " ab c . DE F ' "; - EXPECT_EQ(testValidateName(t163_0), TSDB_CODE_TSC_INVALID_OPERATION); - - char t164[] = " ' AB C . de f ' "; - //int32_t tmp2 = testValidateName(t164); - EXPECT_EQ(testValidateName(t164), TSDB_CODE_TSC_INVALID_OPERATION); - - char t165[] = " ' A BC '. de f "; - EXPECT_EQ(testValidateName(t165), TSDB_CODE_TSC_INVALID_OPERATION); - - char t166[] = "' AB C '.' DE F '"; - EXPECT_EQ(testValidateName(t166), TSDB_CODE_TSC_INVALID_OPERATION); - - char t167[] = "ab c . ' D EF '"; - EXPECT_EQ(testValidateName(t167), TSDB_CODE_TSC_INVALID_OPERATION); - - char t168[] = "' a bc '.' DE F '"; - EXPECT_EQ(testValidateName(t168), TSDB_CODE_TSC_INVALID_OPERATION); - -} - -/* test parse time function */ -TEST(testCase, parse_time) { - taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); - char t1[] = "2018-1-1 1:1:1.952798"; - char t13[] = "1970-1-1 0:0:0"; - - int64_t time = 0, time1 = 0; - - taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 1514739661952); - - taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND); - - char t2[] = "2018-1-1T1:1:1.952Z"; - taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI, 0); - - EXPECT_EQ(time, 1514739661952 + 28800000); - - char t3[] = "2018-1-1 1:01:01.952"; - taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 1514739661952); - - char t4[] = "2018-1-1 1:01:01.9"; - char t5[] = "2018-1-1 1:01:1.900"; - char t6[] = "2018-01-01 1:1:1.90"; - char t7[] = "2018-01-01 01:01:01.9"; - char t8[] = "2018-01-01 01:01:01.9007865"; - - taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t9[] = "2017-4-3 1:1:2.980"; - char t10[] = "2017-4-3T2:1:2.98+9:00"; - taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t11[] = "2017-4-3T2:1:2.98+09:00"; - taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t12[] = "2017-4-3T2:1:2.98+0900"; - taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - taos_options(TSDB_OPTION_TIMEZONE, "UTC"); - taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 0); - - taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); - char t14[] = "1970-1-1T0:0:0Z"; - taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 0); - - char t40[] = "1970-1-1 0:0:0.999999999"; - taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND); - - char t41[] = "1997-1-1 0:0:0.999999999"; - taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 852048000999); - - char t42[] = "1997-1-1T0:0:0.999999999Z"; - taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND); - - // "%Y-%m-%d %H:%M:%S" format with TimeZone appendix is also treated as legal - // and TimeZone will be processed - char t60[] = "2017-4-3 1:1:2.980"; - char t61[] = "2017-4-3 2:1:2.98+9:00"; - taosParseTime(t60, &time, strlen(t60), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t61, &time1, strlen(t61), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t62[] = "2017-4-3 2:1:2.98+09:00"; - taosParseTime(t62, &time, strlen(t62), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t61, &time1, strlen(t61), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t63[] = "2017-4-3 2:1:2.98+0900"; - taosParseTime(t63, &time, strlen(t63), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t62, &time1, strlen(t62), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t64[] = "2017-4-2 17:1:2.98Z"; - taosParseTime(t63, &time, strlen(t63), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t64, &time1, strlen(t64), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - // "%Y-%m-%d%H:%M:%S" format with TimeZone appendix is also treated as legal - // and TimeZone will be processed - char t80[] = "2017-4-51:1:2.980"; - char t81[] = "2017-4-52:1:2.98+9:00"; - taosParseTime(t80, &time, strlen(t80), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t81, &time1, strlen(t81), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t82[] = "2017-4-52:1:2.98+09:00"; - taosParseTime(t82, &time, strlen(t82), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t81, &time1, strlen(t81), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t83[] = "2017-4-52:1:2.98+0900"; - taosParseTime(t83, &time, strlen(t83), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t82, &time1, strlen(t82), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - char t84[] = "2017-4-417:1:2.98Z"; - taosParseTime(t83, &time, strlen(t83), TSDB_TIME_PRECISION_MILLI, 0); - taosParseTime(t84, &time1, strlen(t84), TSDB_TIME_PRECISION_MILLI, 0); - EXPECT_EQ(time, time1); - - //////////////////////////////////////////////////////////////////// - // illegal timestamp format - char t15[] = "2017-12-33 0:0:0"; - EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t16[] = "2017-12-31 99:0:0"; - EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t17[] = "2017-12-31T9:0:0"; - EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t18[] = "2017-12-31T9:0:0.Z"; - EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t19[] = "2017-12-31 9:0:0.-1"; - EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t20[] = "2017-12-31 9:0:0.1+12:99"; - EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t21[] = "2017-12-31T9:0:0.1+12:99"; - EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t22[] = "2017-12-31 9:0:0.1+13:1"; - EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI, 0), 0); - - char t23[] = "2017-12-31T9:0:0.1+13:1"; - EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI, 0), 0); - - char t24[] = "2017-12-31T9:0:0.1+13:001"; - EXPECT_EQ(taosParseTime(t24, &time, strlen(t24), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t25[] = "2017-12-31T9:0:0.1+13:00abc"; - EXPECT_EQ(taosParseTime(t25, &time, strlen(t25), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t26[] = "2017-12-31T9:0:0.1+13001"; - EXPECT_EQ(taosParseTime(t26, &time, strlen(t26), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t27[] = "2017-12-31T9:0:0.1+1300abc"; - EXPECT_EQ(taosParseTime(t27, &time, strlen(t27), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t28[] = "2017-12-31T9:0:0Z+12:00"; - EXPECT_EQ(taosParseTime(t28, &time, strlen(t28), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t29[] = "2017-12-31T9:0:0.123Z+12:00"; - EXPECT_EQ(taosParseTime(t29, &time, strlen(t29), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t65[] = "2017-12-31 9:0:0.1+13:001"; - EXPECT_EQ(taosParseTime(t65, &time, strlen(t65), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t66[] = "2017-12-31 9:0:0.1+13:00abc"; - EXPECT_EQ(taosParseTime(t66, &time, strlen(t66), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t67[] = "2017-12-31 9:0:0.1+13001"; - EXPECT_EQ(taosParseTime(t67, &time, strlen(t67), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t68[] = "2017-12-31 9:0:0.1+1300abc"; - EXPECT_EQ(taosParseTime(t68, &time, strlen(t68), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t69[] = "2017-12-31 9:0:0Z+12:00"; - EXPECT_EQ(taosParseTime(t69, &time, strlen(t69), TSDB_TIME_PRECISION_MILLI, 0), -1); - - char t70[] = "2017-12-31 9:0:0.123Z+12:00"; - EXPECT_EQ(taosParseTime(t70, &time, strlen(t70), TSDB_TIME_PRECISION_MILLI, 0), -1); - -} - -/* test parse time profiling */ -TEST(testCase, parse_time_profile) { - taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); - char t1[] = "2018-1-8 1:1:1.952"; - char t2[] = "2018-1-8T1:1:1.952+0800"; - char t3[] = "2018-1-8 1:1:1.952+0800"; - char t4[] = "2018-1-81:1:1.952+0800"; - - char t5[] = "2018-1-8 1:1:1.952"; - char t6[] = "2018-1-8T1:1:1.952+08:00"; - char t7[] = "2018-1-8 1:1:1.952+08:00"; - char t8[] = "2018-1-81:1:1.952+08:00"; - - char t9[] = "2018-1-8 1:1:1.952"; - char t10[] = "2018-1-8T1:1:1.952Z"; - char t11[] = "2018-1-8 1:1:1.952z"; - char t12[] = "2018-1-81:1:1.952Z"; - - struct timeval start, end; - int64_t time = 0, time1 = 0; - - int32_t total_run = 100000000; - long total_time_us; - - gettimeofday(&start, NULL); - for (int i = 0; i < total_run; ++i) { - taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI, 0); - } - gettimeofday(&end, NULL); - total_time_us = ((end.tv_sec - start.tv_sec)* 1000000) + (end.tv_usec - start.tv_usec); - printf("[t1] The elapsed time is %f seconds in %d run, average:%fns\n", total_time_us/1000000.0, total_run, 1000*(float)total_time_us/(float)total_run); - - gettimeofday(&start, NULL); - for (int i = 0; i < total_run; ++i) { - taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI, 0); - } - gettimeofday(&end, NULL); - total_time_us = ((end.tv_sec - start.tv_sec)* 1000000) + (end.tv_usec - start.tv_usec); - printf("[t2] The elapsed time is %f seconds in %d run, average:%fns\n", total_time_us/1000000.0, total_run, 1000*(float)total_time_us/(float)total_run); - - gettimeofday(&start, NULL); - for (int i = 0; i < total_run; ++i) { - taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI, 0); - } - gettimeofday(&end, NULL); - total_time_us = ((end.tv_sec - start.tv_sec)* 1000000) + (end.tv_usec - start.tv_usec); - printf("[t3] The elapsed time is %f seconds in %d run, average:%fns\n", total_time_us/1000000.0, total_run, 1000*(float)total_time_us/(float)total_run); - - gettimeofday(&start, NULL); - for (int i = 0; i < total_run; ++i) { - taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0); - } - gettimeofday(&end, NULL); - total_time_us = ((end.tv_sec - start.tv_sec)* 1000000) + (end.tv_usec - start.tv_usec); - printf("[t4] The elapsed time is %f seconds in %d run, average:%fns\n", total_time_us/1000000.0, total_run, 1000*(float)total_time_us/(float)total_run); -} - - -TEST(testCase, tvariant_convert) { - // 1. bool data to all other data types - tVariant t = {0}; - _init_tvariant_bool(&t); - - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - EXPECT_EQ(t.i64, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - EXPECT_EQ(t.i64, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.dKey, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.dKey, 0); - - _init_tvariant_bool(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "FALSE"); - tVariantDestroy(&t); - - _init_tvariant_bool(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"FALSE"); - tVariantDestroy(&t); - - // 2. tinyint to other data types - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64, 1); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - EXPECT_EQ(t.i64, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - EXPECT_EQ(t.i64, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); - EXPECT_EQ(t.i64, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.dKey, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.dKey, -27); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-27"); - tVariantDestroy(&t); - - _init_tvariant_tinyint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-27"); - tVariantDestroy(&t); - - // 3. int to other data - // types////////////////////////////////////////////////////////////////// - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64, 1); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); - EXPECT_EQ(t.i64, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.dKey, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.dKey, -23997659); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-23997659"); - tVariantDestroy(&t); - - _init_tvariant_int(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-23997659"); - tVariantDestroy(&t); - - // 4. bigint to other data - // type////////////////////////////////////////////////////////////////////////////// - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64, 1); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64, -3333333333333); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_EQ(t.dKey, -3333333333333); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_EQ(t.dKey, -3333333333333); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-3333333333333"); - tVariantDestroy(&t); - - _init_tvariant_bigint(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-3333333333333"); - tVariantDestroy(&t); - - // 5. float to other data - // types//////////////////////////////////////////////////////////////////////// - _init_tvariant_float(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64, 1); - - _init_tvariant_float(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64, -8991212199); - - _init_tvariant_float(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885); - - _init_tvariant_float(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885); - - _init_tvariant_float(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-8991212199.898788"); - tVariantDestroy(&t); - - _init_tvariant_float(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-8991212199.898788"); - tVariantDestroy(&t); - - // 6. binary to other data types - // ////////////////////////////////////////////////////////////////// - t.pz = "true"; - t.nLen = strlen(t.pz); - t.nType = TSDB_DATA_TYPE_BINARY; - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64, 1); - - _init_tvariant_binary(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), -1); - - _init_tvariant_binary(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64, 200000); - - _init_tvariant_binary(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_DOUBLE_EQ(t.dKey, 200000); - - _init_tvariant_binary(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_DOUBLE_EQ(t.dKey, 200000); - - _init_tvariant_binary(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "2e5"); - tVariantDestroy(&t); - - _init_tvariant_binary(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"2e5"); - tVariantDestroy(&t); - - // 7. nchar to other data types - // ////////////////////////////////////////////////////////////////// - t.wpz = L"FALSE"; - t.nLen = wcslen(t.wpz); - t.nType = TSDB_DATA_TYPE_NCHAR; - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - EXPECT_EQ(t.i64, 0); - - _init_tvariant_nchar(&t); - EXPECT_LE(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); - - _init_tvariant_nchar(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); - EXPECT_EQ(t.i64, -2000000); - - _init_tvariant_nchar(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); - EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765); - - _init_tvariant_nchar(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); - EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765); - - _init_tvariant_nchar(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); - EXPECT_STREQ(t.pz, "-2000000.8765"); - tVariantDestroy(&t); - - _init_tvariant_nchar(&t); - EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); - EXPECT_STREQ(t.wpz, L"-2000000.8765"); - tVariantDestroy(&t); -} - -TEST(testCase, tGetToken_Test) { - char* s = ".123 "; - uint32_t type = 0; - - int32_t len = tGetToken(s, &type); - EXPECT_EQ(type, TK_FLOAT); - EXPECT_EQ(len, strlen(s) - 1); - - char s1[] = "1.123e10 "; - len = tGetToken(s1, &type); - EXPECT_EQ(type, TK_FLOAT); - EXPECT_EQ(len, strlen(s1) - 1); - - char s4[] = "0xff "; - len = tGetToken(s4, &type); - EXPECT_EQ(type, TK_HEX); - EXPECT_EQ(len, strlen(s4) - 1); - - // invalid data type - char s2[] = "e10 "; - len = tGetToken(s2, &type); - EXPECT_FALSE(type == TK_FLOAT); - - char s3[] = "1.1.1.1"; - len = tGetToken(s3, &type); - EXPECT_EQ(type, TK_IPTOKEN); - EXPECT_EQ(len, strlen(s3)); - - char s5[] = "0x "; - len = tGetToken(s5, &type); - EXPECT_FALSE(type == TK_HEX); -} - -static SStrToken createStrToken(char* s) { - SStrToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; - t.type = TK_STRING; - t.z = s; - t.n = strlen(s); - - return t; -} - -TEST(testCase, isValidNumber_test) { - SStrToken t1 = createStrToken("123abc"); - - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createStrToken("0xabc"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_HEX); - - t1 = createStrToken("0b11101"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_BIN); - - t1 = createStrToken(".134abc"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createStrToken("1e1 "); - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createStrToken("1+2"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL); - - t1 = createStrToken("-0x123"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_HEX); - - t1 = createStrToken("-1"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_INTEGER); - - t1 = createStrToken("-0b1110"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_BIN); - - t1 = createStrToken("-.234"); - EXPECT_EQ(tGetNumericStringType(&t1), TK_FLOAT); -} - -TEST(testCase, getTempFilePath_test) { - char path[4096] = {0}; - memset(path, 1, 4096); - - taosGetTmpfilePath("new_tmp", path); - printf("%s\n", path); -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/cmake/cmake.define b/cmake/cmake.define index c985ba1cc705b3482a8defacdf6715c84bea4b2a..c9a188600a16867ba141b88bc82b5251f48efab7 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION 3.16) +set(CMAKE_VERBOSE_MAKEFILE ON) + #set output directory SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin) @@ -43,11 +45,14 @@ ENDIF () IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) - SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd4999 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") + SET(COMMON_FLAGS "/W3 /D_WIN32") - IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) - SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") - ENDIF () + # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) + # SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") + # ENDIF () + + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}") ELSE () IF (${SANITIZER} MATCHES "true") diff --git a/cmake/cmake.options b/cmake/cmake.options index b51f096185a7d42802af2078fcdefd04d35940bc..f32c5acdd1526616a13c93bf62b0a97e6b3434a8 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -25,9 +25,28 @@ IF(${TD_WINDOWS}) ON ) -ENDIF () + MESSAGE("build msvcregex Win32") + option( + BUILD_MSVCREGEX + "If build msvcregex on Windows" + ON + ) -IF(${TD_LINUX} MATCHES TRUE) + MESSAGE("build wcwidth Win32") + option( + BUILD_WCWIDTH + "If build wcwidth on Windows" + ON + ) + + MESSAGE("build wingetopt Win32") + option( + BUILD_WINGETOPT + "If build wingetopt on Windows" + ON + ) + +ENDIF () option( BUILD_TEST @@ -35,8 +54,6 @@ option( ON ) -ENDIF () - option( BUILD_WITH_LEVELDB "If build with leveldb" diff --git a/cmake/msvcregex_CMakeLists.txt.in b/cmake/msvcregex_CMakeLists.txt.in new file mode 100644 index 0000000000000000000000000000000000000000..686ac4cb5d8baa43c7dae4c099c6065714d90b19 --- /dev/null +++ b/cmake/msvcregex_CMakeLists.txt.in @@ -0,0 +1,13 @@ + +# msvcregex +ExternalProject_Add(msvcregex + GIT_REPOSITORY https://gitee.com/l0km/libgnurx-msvc.git + GIT_TAG master + SOURCE_DIR "${TD_CONTRIB_DIR}/msvcregex" + BINARY_DIR "" + #BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) \ No newline at end of file diff --git a/cmake/pthread_CMakeLists.txt.in b/cmake/pthread_CMakeLists.txt.in index 2553df65001eb9728ec734ebb8b02fdbc722c3c2..774f5dd972a5acad673e24c57435a5e2ff3d2b87 100644 --- a/cmake/pthread_CMakeLists.txt.in +++ b/cmake/pthread_CMakeLists.txt.in @@ -3,7 +3,7 @@ ExternalProject_Add(pthread GIT_REPOSITORY https://github.com/GerHobbelt/pthread-win32 GIT_TAG v3.0.3.1 - SOURCE_DIR "${TD_CONTRIB_DIR}/pthread-win32" + SOURCE_DIR "${TD_CONTRIB_DIR}/pthread" BINARY_DIR "" #BUILD_IN_SOURCE TRUE CONFIGURE_COMMAND "" diff --git a/cmake/wcwidth_CMakeLists.txt.in b/cmake/wcwidth_CMakeLists.txt.in new file mode 100644 index 0000000000000000000000000000000000000000..234ccbf915f3fe279266ed823e90c3ad4d7b2744 --- /dev/null +++ b/cmake/wcwidth_CMakeLists.txt.in @@ -0,0 +1,13 @@ + +# wcwidth +ExternalProject_Add(wcwidth + GIT_REPOSITORY https://github.com/fumiyas/wcwidth-cjk.git + GIT_TAG master + SOURCE_DIR "${TD_CONTRIB_DIR}/wcwidth" + BINARY_DIR "" + #BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) \ No newline at end of file diff --git a/cmake/wingetopt_CMakeLists.txt.in b/cmake/wingetopt_CMakeLists.txt.in new file mode 100644 index 0000000000000000000000000000000000000000..b93925c170ff4a276ce951d96b858d9779573928 --- /dev/null +++ b/cmake/wingetopt_CMakeLists.txt.in @@ -0,0 +1,13 @@ + +# wingetopt +ExternalProject_Add(wingetopt + GIT_REPOSITORY https://github.com/alex85k/wingetopt.git + GIT_TAG master + SOURCE_DIR "${TD_CONTRIB_DIR}/wingetopt" + BINARY_DIR "" + #BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) \ No newline at end of file diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1ddc765c5c5a93dd4b8a6001ebede7c3bde22a59..19923a5ad63bc6ec584324116956ea272b3df52b 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -14,9 +14,24 @@ if(${BUILD_PTHREAD}) cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() -# gnu regex -if(${BUILD_GNUREGEX}) - cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +# iconv +if(${BUILD_WITH_ICONV}) + cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + +# msvc regex +if(${BUILD_MSVCREGEX}) + cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + +# wcwidth +if(${BUILD_WCWIDTH}) + cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + +# wingetopt +if(${BUILD_WINGETOPT}) + cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() # googletest @@ -99,8 +114,27 @@ if(${BUILD_TEST}) target_include_directories( gtest PUBLIC $ - PUBLIC $ ) + if(${TD_WINDOWS}) + target_include_directories( + gtest + PUBLIC $ + ) + endif(${TD_WINDOWS}) + if(${TD_LINUX}) + target_include_directories( + gtest + PUBLIC $ + ) + endif(${TD_LINUX}) + if(${TD_DARWIN}) + target_include_directories( + gtest + PUBLIC $ + ) + endif(${TD_DARWIN}) + + endif(${BUILD_TEST}) # cJson @@ -182,6 +216,53 @@ if(${BUILD_WITH_NURAFT}) add_subdirectory(nuraft) endif(${BUILD_WITH_NURAFT}) +# pthread +if(${BUILD_PTHREAD}) + set(CMAKE_BUILD_TYPE release) + add_definitions(-DPTW32_STATIC_LIB) + add_subdirectory(pthread) + set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread) + add_library(pthread STATIC IMPORTED GLOBAL) + SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib) +endif() + +# iconv +if(${BUILD_WITH_ICONV}) + add_subdirectory(iconv) +endif(${BUILD_WITH_ICONV}) + +# wingetopt +if(${BUILD_WINGETOPT}) + add_subdirectory(wingetopt) +endif(${BUILD_WINGETOPT}) + +# msvcregex +if(${BUILD_MSVCREGEX}) + add_library(msvcregex STATIC "") + target_sources(msvcregex + PRIVATE "msvcregex/regex.c" + ) + target_include_directories(msvcregex + PRIVATE "msvcregex" + ) + target_link_libraries(msvcregex + INTERFACE Shell32 + ) + SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex) +endif(${BUILD_MSVCREGEX}) + +# msvcregex +if(${BUILD_WCWIDTH}) + add_library(wcwidth STATIC "") + target_sources(wcwidth + PRIVATE "wcwidth/wcwidth.c" + ) + target_include_directories(wcwidth + PRIVATE "wcwidth" + ) + SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth) +endif(${BUILD_WCWIDTH}) + # CRAFT if(${BUILD_WITH_CRAFT}) add_library(craft STATIC IMPORTED GLOBAL) @@ -238,8 +319,12 @@ if(${BUILD_WITH_SQLITE}) target_link_libraries(sqlite INTERFACE m INTERFACE pthread - INTERFACE dl ) + if(NOT TD_WINDOWS) + target_link_libraries(sqlite + INTERFACE dl + ) + endif(NOT TD_WINDOWS) endif(${BUILD_WITH_SQLITE}) # pthread diff --git a/contrib/test/craft/CMakeLists.txt b/contrib/test/craft/CMakeLists.txt index e0f6ae64bd3e9c998db15c1006cebf1b15702f5b..ec8b44b67395a481ed2970f19cb5d0e1290f6d6b 100644 --- a/contrib/test/craft/CMakeLists.txt +++ b/contrib/test/craft/CMakeLists.txt @@ -1,2 +1,9 @@ add_executable(simulate_vnode "simulate_vnode.c") -target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) \ No newline at end of file +target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) +if(${BUILD_WINGETOPT}) + target_link_libraries(simulate_vnode PUBLIC wingetopt) + target_include_directories( + simulate_vnode + PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" + ) +endif() \ No newline at end of file diff --git a/contrib/test/tdev/src/main.c b/contrib/test/tdev/src/main.c index 5e1de83e88a2a76d2381726ded0c28fe00ab67a6..e40040ce9762e39ca3b23380f4470352a96c921b 100644 --- a/contrib/test/tdev/src/main.c +++ b/contrib/test/tdev/src/main.c @@ -6,43 +6,39 @@ #define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s))) #define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa)) -#define tPutA(buf, val) \ - ({ \ - memcpy(buf, &val, sizeof(val)); \ - POINTER_SHIFT(buf, sizeof(val)); \ - }) +static inline void tPutA(void **buf, uint64_t val) { + memcpy(buf, &val, sizeof(val)); + *buf = POINTER_SHIFT(buf, sizeof(val)); +} -#define tPutB(buf, val) \ - ({ \ - ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \ - ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \ - ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \ - ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \ - ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \ - ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \ - ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; \ - ((uint8_t *)buf)[0] = (val)&0xff; \ - POINTER_SHIFT(buf, sizeof(val)); \ - }) +static inline void tPutB(void **buf, uint64_t val) { + ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; + ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; + ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; + ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; + ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; + ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; + ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; + ((uint8_t *)buf)[0] = (val)&0xff; + *buf = POINTER_SHIFT(buf, sizeof(val)); +} -#define tPutC(buf, val) \ - ({ \ - if (buf) { \ - ((uint64_t *)buf)[0] = (val); \ - POINTER_SHIFT(buf, sizeof(val)); \ - } \ - NULL; \ - }) +static inline void tPutC(void **buf, uint64_t val) { + if (buf) { + ((uint64_t *)buf)[0] = (val); + POINTER_SHIFT(buf, sizeof(val)); + } + *buf = NULL; +} -#define tPutD(buf, val) \ - ({ \ - uint64_t tmp = val; \ - for (size_t i = 0; i < sizeof(val); i++) { \ - ((uint8_t *)buf)[i] = tmp & 0xff; \ - tmp >>= 8; \ - } \ - POINTER_SHIFT(buf, sizeof(val)); \ - }) +static inline void tPutD(void **buf, uint64_t val) { + uint64_t tmp = val; + for (size_t i = 0; i < sizeof(val); i++) { + ((uint8_t *)buf)[i] = tmp & 0xff; + tmp >>= 8; + } + *buf = POINTER_SHIFT(buf, sizeof(val)); +} static inline void tPutE(void **buf, uint64_t val) { if (buf) { @@ -61,7 +57,7 @@ static void func(T t) { switch (t) { case A: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - pBuf = tPutA(pBuf, val); + tPutA(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -69,7 +65,7 @@ static void func(T t) { break; case B: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - pBuf = tPutB(pBuf, val); + tPutB(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -77,7 +73,7 @@ static void func(T t) { break; case C: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - pBuf = tPutC(pBuf, val); + tPutC(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -85,7 +81,7 @@ static void func(T t) { break; case D: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - pBuf = tPutD(pBuf, val); + tPutD(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index fe5f8cf7edcb6a317247414a772a49d56c7e6dbd..80059dde4d1017b8fa1c71b3dd74271f4927187d 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -39,6 +39,7 @@ target_include_directories(tstream target_include_directories(demoapi PUBLIC "${TD_SOURCE_DIR}/include/client" + PUBLIC "${TD_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) diff --git a/example/src/demoapi.c b/example/src/demoapi.c index 45da88fa1b44b37cfe7e10a29a36d4093fe370e0..c3a6d267766dcb5621c126fb1dc3d411f9f3929a 100644 --- a/example/src/demoapi.c +++ b/example/src/demoapi.c @@ -4,10 +4,12 @@ #include #include #include -#include +// #include #include +#ifndef WINDOWS #include - +#endif +#include "osSleep.h" #include "taos.h" #define debugPrint(fmt, ...) \ @@ -32,6 +34,7 @@ int64_t g_num_of_tb = 2; int64_t g_num_of_rec = 3; +#ifndef WINDOWS static struct argp_option options[] = { {"tables", 't', "NUMBER", 0, "Number of child tables, default is 10000."}, {"records", 'n', "NUMBER", 0, @@ -61,16 +64,16 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { } static struct argp argp = {options, parse_opt, "", ""}; - +#endif static void prepare_data(TAOS* taos) { TAOS_RES *res; res = taos_query(taos, "drop database if exists test;"); taos_free_result(res); - usleep(100000); + taosMsleep(100); res = taos_query(taos, "create database test;"); taos_free_result(res); - usleep(100000); + taosMsleep(100); taos_select_db(taos, "test"); char command[1024] = {0}; @@ -87,18 +90,17 @@ static void prepare_data(TAOS* taos) { taos_free_result(res); for (int64_t i = 0; i < g_num_of_tb; i ++) { + // sprintf(command, "create table t%"PRId64" using meters " + // "tags(%"PRId64", '%s', '%s', '%s');", + // i, i, (i%2)?"beijing":"shanghai", + // (i%2)?"朝阳区":"黄浦区", + // (i%2)?"长安街":"中山路"); sprintf(command, "create table t%"PRId64" using meters " - "tags(%"PRId64", '%s', '%s', '%s');", - i, i, (i%2)?"beijing":"shanghai", - (i%2)?"朝阳区":"黄浦区", - (i%2)?"长安街":"中山路"); -/* sprintf(command, "create table t%"PRId64" using meters " "tags(%"PRId64", '%s', '%s', '%s');", i, i, (i%2)?"beijing":"shanghai", (i%2)?"chaoyang":"huangpu", (i%2?"changan street":"jianguo rd")); - */ res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { okPrint("t%" PRId64 " created\n", i); @@ -117,7 +119,8 @@ static void prepare_data(TAOS* taos) { "'%c%d', '%s%c%d', '%c%d')", i, 1650000000000+j, (float)j, j, 'a'+(int)j%25, rand(), - "涛思", 'z' - (int)j%25, rand(), + // "涛思", 'z' - (int)j%25, rand(), + "TAOS", 'z' - (int)j%25, rand(), 'b' - (int)j%25, rand() ); res = taos_query(taos, command); @@ -176,7 +179,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { } if (block) { - warnPrint("%s", "call taos_fetch_block()\n"); + warnPrint("%s", "call taos_fetch_block(), don't call taos_fetch_lengths()\n"); int rows = 0; while ((rows = taos_fetch_block(res, &row))) { int *lengths = taos_fetch_lengths(res); @@ -196,7 +199,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %"PRId64"\n", f, c, - *(int64_t*)(row[f]+c*sizeof(int64_t))); + *(int64_t*)((char*)(row[f])+c*sizeof(int64_t))); } break; @@ -208,7 +211,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %d\n", f, c, - *(int32_t*)(row[f]+c*sizeof(int32_t))); + *(int32_t*)((char*)(row[f])+c*sizeof(int32_t))); } break; @@ -220,7 +223,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %f\n", f, c, - *(float*)(row[f]+c*sizeof(float))); + *(float*)((char*)(row[f])+c*sizeof(float))); } break; @@ -235,9 +238,9 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { if (offsets) { for (int c = 0; c < rows; c++) { if (offsets[c] != -1) { - int length = *(int16_t*)(row[f] + offsets[c]); + int length = *(int16_t*)((char*)(row[f]) + offsets[c]); char *buf = calloc(1, length + 1); - strncpy(buf, (char *)(row[f] + offsets[c] + 2), length); + strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length); printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n", c, f, offsets[c], length, buf); free(buf); @@ -314,8 +317,9 @@ int main(int argc, char *argv[]) { const char* host = "127.0.0.1"; const char* user = "root"; const char* passwd = "taosdata"; - +#ifndef WINDOWS argp_parse(&argp, argc, argv, 0, 0, NULL); +#endif TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos)); diff --git a/example/src/tmq.c b/example/src/tmq.c index fdd26bc95db32f04823887e006acca138f36bfaa..46f57799e7499dc77d1a9e1a5cb0f7a75d1e6309 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -22,6 +22,7 @@ static int running = 1; static void msg_process(TAOS_RES* msg) { char buf[1024]; + memset(buf, 0, 1024); printf("topic: %s\n", tmq_get_topic_name(msg)); printf("vg:%d\n", tmq_get_vgroup_id(msg)); while (1) { @@ -102,6 +103,7 @@ int32_t create_topic() { /*const char* sql = "select * from tu1";*/ /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/ + /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/ pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); @@ -140,7 +142,7 @@ int32_t create_topic() { return 0; } -void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { +void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets) { printf("commit %d\n", resp); } @@ -162,7 +164,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "td.connect.db", "abc1"); tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); - tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); return tmq; } @@ -188,7 +190,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { cnt++; /*printf("get data\n");*/ /*msg_process(tmqmessage);*/ - tmq_message_destroy(tmqmessage); + taos_free_result(tmqmessage); /*} else {*/ /*break;*/ } @@ -218,9 +220,9 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000); if (tmqmessage) { msg_process(tmqmessage); - tmq_message_destroy(tmqmessage); + taos_free_result(tmqmessage); - if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0); + /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/ } } @@ -248,7 +250,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) { batchCnt++; /*skipLogNum += tmqGetSkipLogNum(tmqmessage);*/ /*msg_process(tmqmessage);*/ - tmq_message_destroy(tmqmessage); + taos_free_result(tmqmessage); } else { break; } diff --git a/include/client/taos.h b/include/client/taos.h index 218090363305fb7bbcb1cc2fa1627e4afb89bf12..2ace01f95ebe2845a47e35b2d9f6508b98bf3128 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -92,38 +92,14 @@ typedef struct taosField { typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); -typedef struct TAOS_BIND { - int buffer_type; - void *buffer; - uintptr_t buffer_length; // unused - uintptr_t *length; - int *is_null; - - int is_unsigned; // unused - int *error; // unused - union { - int64_t ts; - int8_t b; - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - unsigned char *bin; - char *nchar; - } u; - unsigned int allocated; -} TAOS_BIND; - -typedef struct TAOS_MULTI_BIND { +typedef struct TAOS_BIND_v2 { int buffer_type; void *buffer; - uintptr_t buffer_length; + int32_t buffer_length; int32_t *length; char *is_null; int num; -} TAOS_MULTI_BIND; +} TAOS_BIND_v2; typedef enum { SET_CONF_RET_SUCC = 0, @@ -152,34 +128,35 @@ DLL_EXPORT void taos_close(TAOS *taos); const char *taos_data_type(int type); -DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); -DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); -DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags); -DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); -DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); - -DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); -DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); -DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); -DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind); -DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); -DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); -DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); -DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); -DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); - -DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); -DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); - -DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); -DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result -DLL_EXPORT void taos_free_result(TAOS_RES *res); -DLL_EXPORT int taos_field_count(TAOS_RES *res); -DLL_EXPORT int taos_num_fields(TAOS_RES *res); -DLL_EXPORT int taos_affected_rows(TAOS_RES *res); +DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); +DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); +DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags); +DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); +DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); + +DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); +DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); +DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); +DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind); +DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind); +DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx); +DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); +DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); +DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); + +DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); +DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); + +DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); +DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result +DLL_EXPORT void taos_free_result(TAOS_RES *res); +DLL_EXPORT int taos_field_count(TAOS_RES *res); +DLL_EXPORT int taos_num_fields(TAOS_RES *res); +DLL_EXPORT int taos_affected_rows(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); @@ -239,19 +216,18 @@ typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t; typedef struct tmq_conf_t tmq_conf_t; typedef struct tmq_list_t tmq_list_t; -// typedef struct tmq_message_t tmq_message_t; -typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param)); +typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *)); DLL_EXPORT tmq_list_t *tmq_list_new(); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); DLL_EXPORT void tmq_list_destroy(tmq_list_t *); -#if 1 +#if 0 DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); #endif -DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen); +DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen); DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); @@ -285,24 +261,19 @@ DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb); -#if 0 -// temporary used function for demo only -void tmqShowMsg(tmq_message_t *tmq_message); -int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message); -#endif - /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */ DLL_EXPORT char *tmq_get_topic_name(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); +// TODO +#if 0 +DLL_EXPORT char *tmq_get_block_table_name(TAOS_RES *res); +#endif + #if 0 -DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message); DLL_EXPORT int64_t tmq_get_request_offset(tmq_message_t *message); DLL_EXPORT int64_t tmq_get_response_offset(tmq_message_t *message); -DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic); -DLL_EXPORT int32_t tmq_field_count(tmq_t *tmq, const char *topic); #endif -DLL_EXPORT void tmq_message_destroy(TAOS_RES *res); /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */ #if 0 DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); @@ -315,6 +286,16 @@ DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); #endif +typedef enum { + TSDB_SRV_STATUS_UNAVAILABLE = 0, + TSDB_SRV_STATUS_NETWORK_OK = 1, + TSDB_SRV_STATUS_SERVICE_OK = 2, + TSDB_SRV_STATUS_SERVICE_DEGRADED = 3, + TSDB_SRV_STATUS_EXTING = 4, +} TSDB_SERVER_STATUS; + +DLL_EXPORT TSDB_SERVER_STATUS taos_check_server_status(const char *fqdn, int port, char *details, int maxlen); + #ifdef __cplusplus } #endif diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 6e344dfdb4422456b745c66e85a4f5f27800b4e0..af8bda2593fdcd84533414176ee7084343f982aa 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -78,13 +78,6 @@ typedef enum { TSDB_SMA_TYPE_ROLLUP = 2, // Rollup SMA } ETsdbSmaType; -typedef enum { - TSDB_BSMA_TYPE_NONE = 0, // no block-wise SMA - TSDB_BSMA_TYPE_I = 1, // sum/min/max(default) -} ETsdbBSmaType; - -#define TSDB_BSMA_TYPE_LATEST TSDB_BSMA_TYPE_I - extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 043f2d1d129b238fc0fa0f933bf8b2adb2b2619e..fa203e231a938b1a9be04207adec90ed74442154 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -54,12 +54,34 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet); BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) +#define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1) +#define colDataSetNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] = -1) + +#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT) + +#define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) + +#define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes)) +// SColumnInfoData, rowNumber +#define colDataGetData(p1_, r_) \ + ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) \ + : colDataGetNumData(p1_, r_)) + static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { + if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON){ + if(colDataIsNull_var(pColumnInfoData, row)){ + return true; + } + char *data = colDataGetVarData(pColumnInfoData, row); + return (*data == TSDB_DATA_TYPE_NULL); + } + if (!pColumnInfoData->hasNull) { return false; } - if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - return pColumnInfoData->varmeta.offset[row] == -1; + + if (pColumnInfoData->info.type== TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) { + return colDataIsNull_var(pColumnInfoData, row); } else { if (pColumnInfoData->nullbitmap == NULL) { return false; @@ -86,7 +108,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - return pColumnInfoData->varmeta.offset[row] == -1; + return colDataIsNull_var(pColumnInfoData, row); } else { if (pColumnInfoData->nullbitmap == NULL) { return false; @@ -96,17 +118,10 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u } } -#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT) - -// SColumnInfoData, rowNumber -#define colDataGetData(p1_, r_) \ - ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \ - : ((p1_)->pData + ((r_) * (p1_)->info.bytes))) - static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) { // There is a placehold for each NULL value of binary or nchar type. if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type. + colDataSetNull_var(pColumnInfoData, currentRow); // it is a null value of VAR type. } else { colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow); } @@ -117,7 +132,7 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { for (int32_t i = start; i < start + nRows; ++i) { - pColumnInfoData->varmeta.offset[i] = -1; // it is a null value of VAR type. + colDataSetNull_var(pColumnInfoData,i); // it is a null value of VAR type. } } else { for (int32_t i = start; i < start + nRows; ++i) { @@ -265,3 +280,4 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da #endif #endif /*_TD_COMMON_EP_H_*/ + diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 8a87a1bd2eb3670e34ecdb24c711c2b0c904e9b1..a0f4351bbf725c23b8d9e9291fc6a04b5dfb0460 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -502,7 +502,7 @@ typedef struct { #define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) -#define kvRowLen(r) (*(TDRowLenT *)(r)) +#define kvRowLen(r) (*(uint16_t *)(r)) #define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t))) #define kvRowSetLen(r, len) kvRowLen(r) = (len) #define kvRowSetNCols(r, n) kvRowNCols(r) = (n) @@ -608,7 +608,7 @@ void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); -static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, int8_t type, const void *value) { +static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, const void *value, int32_t tlen) { if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); @@ -621,7 +621,6 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co pBuilder->nCols++; - int32_t tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; if (tlen > pBuilder->alloc - pBuilder->size) { while (tlen > pBuilder->alloc - pBuilder->size) { pBuilder->alloc *= 2; diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 5022cb7be80773460f42844af10cc17290de658b..88ce0cd970ec3528f75221a4ab2768550e6fd358 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -32,7 +32,6 @@ extern char tsLocalEp[]; extern uint16_t tsServerPort; extern int32_t tsVersion; extern int32_t tsStatusInterval; -extern bool tsEnableTelemetryReporting; // common extern int32_t tsRpcTimer; @@ -82,6 +81,12 @@ extern uint16_t tsMonitorPort; extern int32_t tsMonitorMaxLogs; extern bool tsMonitorComp; +// telem +extern bool tsEnableTelem; +extern int32_t tsTelemInterval; +extern char tsTelemServer[]; +extern uint16_t tsTelemPort; + // query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node @@ -130,7 +135,6 @@ void taosCfgDynamicOptions(const char *option, const char *value); void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary); struct SConfig *taosGetCfg(); -int32_t taosAddClientLogCfg(SConfig *pCfg); #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 32d59c69293cd47fdf59ed24402a4f0d2fa5a907..5ea53b5510a27e7b2aa9fed70b2bd041066ed3e8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -269,6 +269,8 @@ typedef struct SSchema { #define SSCHMEA_BYTES(s) ((s)->bytes) #define SSCHMEA_NAME(s) ((s)->name) +STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols); + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; @@ -280,10 +282,14 @@ typedef struct { int32_t numOfTags; int32_t numOfSmas; int32_t commentLen; + int32_t ast1Len; + int32_t ast2Len; SArray* pColumns; // array of SField SArray* pTags; // array of SField SArray* pSmas; // array of SField char* comment; + char* pAst1; + char* pAst2; } SMCreateStbReq; int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq); @@ -318,13 +324,22 @@ typedef struct SEpSet { int32_t tEncodeSEpSet(SCoder* pEncoder, const SEpSet* pEp); int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp); int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp); -void* taosDecodeSEpSet(void* buf, SEpSet* pEp); +void* taosDecodeSEpSet(const void* buf, SEpSet* pEp); + +typedef struct { + SEpSet epSet; +} SMEpSet; + +int32_t tSerializeSMEpSet(void* buf, int32_t bufLen, SMEpSet* pReq); +int32_t tDeserializeSMEpSet(void* buf, int32_t buflen, SMEpSet* pReq); typedef struct { int8_t connType; int32_t pid; char app[TSDB_APP_NAME_LEN]; char db[TSDB_DB_NAME_LEN]; + char user[TSDB_USER_LEN]; + char passwd[TSDB_PASSWORD_LEN]; int64_t startTime; } SConnectReq; @@ -476,7 +491,7 @@ typedef struct { char intervalUnit; char slidingUnit; char - offsetUnit; // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration. + offsetUnit; // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration. int8_t precision; int64_t interval; int64_t sliding; @@ -509,7 +524,7 @@ typedef struct { int8_t precision; // time resolution int8_t compression; int8_t replications; - int8_t quorum; + int8_t strict; int8_t update; int8_t cacheLastRow; int8_t ignoreExist; @@ -531,7 +546,7 @@ typedef struct { int32_t daysToKeep2; int32_t fsyncPeriod; int8_t walLevel; - int8_t quorum; + int8_t strict; int8_t cacheLastRow; int8_t replications; } SAlterDbReq; @@ -604,11 +619,13 @@ typedef struct { int8_t precision; int8_t compression; int8_t replications; - int8_t quorum; + int8_t strict; int8_t update; int8_t cacheLastRow; int8_t streamMode; int8_t singleSTable; + int32_t numOfRetensions; + SArray* pRetensions; } SDbCfgRsp; int32_t tSerializeSDbCfgRsp(void* buf, int32_t bufLen, const SDbCfgRsp* pRsp); @@ -639,10 +656,10 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); typedef struct { char db[TSDB_DB_FNAME_LEN]; -} SSyncDbReq, SCompactDbReq; +} SCompactDbReq; -int32_t tSerializeSSyncDbReq(void* buf, int32_t bufLen, SSyncDbReq* pReq); -int32_t tDeserializeSSyncDbReq(void* buf, int32_t bufLen, SSyncDbReq* pReq); +int32_t tSerializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq); +int32_t tDeserializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq); typedef struct { char name[TSDB_FUNC_NAME_LEN]; @@ -652,6 +669,7 @@ typedef struct { int8_t outputType; int32_t outputLen; int32_t bufSize; + int32_t codeLen; int64_t signature; char* pComment; char* pCode; @@ -671,6 +689,7 @@ int32_t tDeserializeSDropFuncReq(void* buf, int32_t bufLen, SDropFuncReq* pReq); typedef struct { int32_t numOfFuncs; + bool ignoreCodeComment; SArray* pFuncNames; } SRetrieveFuncReq; @@ -699,6 +718,7 @@ typedef struct { int32_t tSerializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp); int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp); +void tFreeSFuncInfo(SFuncInfo *pInfo); void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp* pRsp); typedef struct { @@ -723,7 +743,7 @@ typedef struct { typedef struct { int32_t vgId; - int8_t role; + int32_t syncState; int64_t numOfTables; int64_t numOfTimeSeries; int64_t totalStorage; @@ -736,6 +756,10 @@ typedef struct { int64_t numOfBatchInsertSuccessReqs; } SVnodeLoad; +typedef struct { + int32_t syncState; +} SMnodeLoad; + typedef struct { int32_t sver; // software version int64_t dnodeVer; // dnode table version in sdb @@ -810,7 +834,7 @@ typedef struct { int8_t walLevel; int8_t precision; int8_t compression; - int8_t quorum; + int8_t strict; int8_t update; int8_t cacheLastRow; int8_t replica; @@ -819,7 +843,7 @@ typedef struct { SReplica replicas[TSDB_MAX_REPLICA]; int32_t numOfRetensions; SArray* pRetensions; // SRetention -} SCreateVnodeReq, SAlterVnodeReq; +} SCreateVnodeReq; int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); @@ -830,11 +854,36 @@ typedef struct { int32_t dnodeId; int64_t dbUid; char db[TSDB_DB_FNAME_LEN]; -} SDropVnodeReq, SSyncVnodeReq, SCompactVnodeReq; +} SDropVnodeReq; int32_t tSerializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq); int32_t tDeserializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq); +typedef struct { + int64_t dbUid; + char db[TSDB_DB_FNAME_LEN]; +} SCompactVnodeReq; + +int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq); +int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq); + +typedef struct { + int32_t vgVersion; + int32_t totalBlocks; + int32_t daysToKeep0; + int32_t daysToKeep1; + int32_t daysToKeep2; + int8_t walLevel; + int8_t strict; + int8_t cacheLastRow; + int8_t replica; + int8_t selfIndex; + SReplica replicas[TSDB_MAX_REPLICA]; +} SAlterVnodeReq; + +int32_t tSerializeSAlterVnodeReq(void* buf, int32_t bufLen, SAlterVnodeReq* pReq); +int32_t tDeserializeSAlterVnodeReq(void* buf, int32_t bufLen, SAlterVnodeReq* pReq); + typedef struct { SMsgHead header; char dbFName[TSDB_DB_FNAME_LEN]; @@ -942,7 +991,6 @@ int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); void tFreeSShowRsp(SShowRsp* pRsp); typedef struct { - int32_t type; char db[TSDB_DB_FNAME_LEN]; char tb[TSDB_TABLE_NAME_LEN]; int64_t showId; @@ -1071,10 +1119,12 @@ int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); typedef struct { - int8_t finished; - char name[TSDB_STEP_NAME_LEN]; - char desc[TSDB_STEP_DESC_LEN]; -} SStartupReq; + int32_t statusCode; + char details[1024]; +} SServerStatusRsp; + +int32_t tSerializeSServerStatusRsp(void* buf, int32_t bufLen, SServerStatusRsp* pRsp); +int32_t tDeserializeSServerStatusRsp(void* buf, int32_t bufLen, SServerStatusRsp* pRsp); /** * The layout of the query message payload is as following: @@ -1233,11 +1283,16 @@ typedef struct { } SMVCreateStreamRsp, SMSCreateStreamRsp; typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - int8_t igExists; - char* sql; - char* ast; - char subscribeDbName[TSDB_DB_NAME_LEN]; + char name[TSDB_TOPIC_FNAME_LEN]; + int8_t igExists; + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + int8_t withTagSchema; + char* sql; + char* ast; + int64_t subDbUid; + char subscribeDbName[TSDB_DB_NAME_LEN]; } SCMCreateTopicReq; int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq); @@ -1251,10 +1306,14 @@ typedef struct { int32_t tSerializeSCMCreateTopicRsp(void* buf, int32_t bufLen, const SCMCreateTopicRsp* pRsp); int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicRsp* pRsp); +typedef struct { + int64_t consumerId; +} SMqConsumerLostMsg; + typedef struct { int32_t topicNum; int64_t consumerId; - char* consumerGroup; + char cgroup[TSDB_CGROUP_LEN]; SArray* topicNames; // SArray } SCMSubscribeReq; @@ -1262,7 +1321,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pReq->topicNum); tlen += taosEncodeFixedI64(buf, pReq->consumerId); - tlen += taosEncodeString(buf, pReq->consumerGroup); + tlen += taosEncodeString(buf, pReq->cgroup); for (int32_t i = 0; i < pReq->topicNum; i++) { tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i)); @@ -1273,7 +1332,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) { buf = taosDecodeFixedI32(buf, &pReq->topicNum); buf = taosDecodeFixedI64(buf, &pReq->consumerId); - buf = taosDecodeString(buf, &pReq->consumerGroup); + buf = taosDecodeStringTo(buf, pReq->cgroup); pReq->topicNames = taosArrayInit(pReq->topicNum, sizeof(void*)); for (int32_t i = 0; i < pReq->topicNum; i++) { char* name; @@ -1349,10 +1408,10 @@ static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq } typedef struct { - const char* key; - SArray* lostConsumers; // SArray - SArray* removedConsumers; // SArray - SArray* newConsumers; // SArray + char key[TSDB_SUBSCRIBE_KEY_LEN]; + SArray* lostConsumers; // SArray + SArray* removedConsumers; // SArray + SArray* newConsumers; // SArray } SMqRebSubscribe; static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) { @@ -1360,7 +1419,7 @@ static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) { if (pRebSub == NULL) { goto _err; } - pRebSub->key = strdup(key); + strcpy(pRebSub->key, key); pRebSub->lostConsumers = taosArrayInit(0, sizeof(int64_t)); if (pRebSub->lostConsumers == NULL) { goto _err; @@ -1385,6 +1444,7 @@ _err: // this message is sent from mnode to mnode(read thread to write thread), so there is no need for serialization or // deserialization typedef struct { + int8_t* mqInReb; SHashObj* rebSubHash; // SHashObj } SMqDoRebalanceMsg; @@ -1426,8 +1486,12 @@ typedef struct { typedef struct { float xFilesFactor; int32_t delay; - int8_t nFuncIds; + int32_t qmsg1Len; + int32_t qmsg2Len; func_id_t* pFuncIds; + char* qmsg1; // not null: pAst1:qmsg1:SRetention1 => trigger aggr task1 + char* qmsg2; // not null: pAst2:qmsg2:SRetention2 => trigger aggr task2 + int8_t nFuncIds; } SRSmaParam; typedef struct SVCreateTbReq { @@ -1887,6 +1951,64 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { return buf; } +enum { + TOPIC_SUB_TYPE__DB = 1, + TOPIC_SUB_TYPE__TABLE, +}; + +typedef struct { + int64_t leftForVer; + int32_t vgId; + int64_t oldConsumerId; + int64_t newConsumerId; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + int8_t subType; + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + int8_t withTagSchema; + char* qmsg; +} SMqRebVgReq; + +static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pReq) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pReq->leftForVer); + tlen += taosEncodeFixedI32(buf, pReq->vgId); + tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); + tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); + tlen += taosEncodeString(buf, pReq->subKey); + tlen += taosEncodeFixedI8(buf, pReq->subType); + tlen += taosEncodeFixedI8(buf, pReq->withTbName); + tlen += taosEncodeFixedI8(buf, pReq->withSchema); + tlen += taosEncodeFixedI8(buf, pReq->withTag); + tlen += taosEncodeFixedI8(buf, pReq->withTagSchema); + if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { + tlen += taosEncodeString(buf, pReq->qmsg); + } + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) { + buf = taosDecodeFixedI64(buf, &pReq->leftForVer); + buf = taosDecodeFixedI32(buf, &pReq->vgId); + buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); + buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); + buf = taosDecodeStringTo(buf, pReq->subKey); + buf = taosDecodeFixedI8(buf, &pReq->subType); + buf = taosDecodeFixedI8(buf, &pReq->withTbName); + buf = taosDecodeFixedI8(buf, &pReq->withSchema); + buf = taosDecodeFixedI8(buf, &pReq->withTag); + buf = taosDecodeFixedI8(buf, &pReq->withTagSchema); + if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { + buf = taosDecodeString(buf, &pReq->qmsg); + } + return (void*)buf; +} + +typedef struct { + int8_t reserved; +} SMqRebVgRsp; + typedef struct { int64_t leftForVer; int32_t vgId; @@ -2219,9 +2341,10 @@ static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) { } } -static FORCE_INLINE void tdFreeTSmaWrapper(STSmaWrapper* pSW) { +static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW) { tdDestroyTSmaWrapper(pSW); - taosMemoryFreeClear(pSW); + taosMemoryFree(pSW); + return NULL; } static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) { @@ -2357,6 +2480,7 @@ typedef struct { int64_t consumerId; } SMqRspHead; +#if 0 typedef struct { SMsgHead head; @@ -2370,6 +2494,17 @@ typedef struct { uint64_t reqId; char topic[TSDB_TOPIC_FNAME_LEN]; } SMqPollReq; +#endif + +typedef struct { + SMsgHead head; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + int32_t epoch; + uint64_t reqId; + int64_t consumerId; + int64_t blockingTime; + int64_t currentOffset; +} SMqPollReqV2; typedef struct { int32_t vgId; @@ -2443,13 +2578,81 @@ static FORCE_INLINE void* tDecodeSMqPollRspV2(const void* buf, SMqPollRspV2* pRs return (void*)buf; } +typedef struct { + SMqRspHead head; + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; + int32_t blockNum; + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + int8_t withTagSchema; + SArray* blockDataLen; // SArray + SArray* blockData; // SArray + SArray* blockTbName; // SArray + SArray* blockSchema; // SArray + SArray* blockTags; // SArray + SArray* blockTagSchema; // SArray +} SMqDataBlkRsp; + +static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); + tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); + tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); + tlen += taosEncodeFixedI32(buf, pRsp->blockNum); + if (pRsp->blockNum != 0) { + tlen += taosEncodeFixedI8(buf, pRsp->withTbName); + tlen += taosEncodeFixedI8(buf, pRsp->withSchema); + tlen += taosEncodeFixedI8(buf, pRsp->withTag); + tlen += taosEncodeFixedI8(buf, pRsp->withTagSchema); + + for (int32_t i = 0; i < pRsp->blockNum; i++) { + int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i); + void* data = taosArrayGetP(pRsp->blockData, i); + tlen += taosEncodeFixedI32(buf, bLen); + tlen += taosEncodeBinary(buf, data, bLen); + } + } + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* pRsp) { + buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); + buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); + buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); + buf = taosDecodeFixedI32(buf, &pRsp->blockNum); + pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*)); + if (pRsp->blockNum != 0) { + buf = taosDecodeFixedI8(buf, &pRsp->withTbName); + buf = taosDecodeFixedI8(buf, &pRsp->withSchema); + buf = taosDecodeFixedI8(buf, &pRsp->withTag); + buf = taosDecodeFixedI8(buf, &pRsp->withTagSchema); + + for (int32_t i = 0; i < pRsp->blockNum; i++) { + int32_t bLen = 0; + void* data = NULL; + buf = taosDecodeFixedI32(buf, &bLen); + buf = taosDecodeBinary(buf, &data, bLen); + taosArrayPush(pRsp->blockDataLen, &bLen); + taosArrayPush(pRsp->blockData, &data); + } + } + return (void*)buf; +} + typedef struct { SMqRspHead head; char cgroup[TSDB_CGROUP_LEN]; SArray* topics; // SArray } SMqCMGetSubEpRsp; -static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); } +static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { + // taosMemoryFree(pSubTopicEp->schema.pSchema); + taosArrayDestroy(pSubTopicEp->vgs); +} static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { int32_t tlen = 0; @@ -2529,6 +2732,7 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p } return buf; } + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 6c3671a8d6068376b9bb63f3bd14fd56919183d9..02d8b76b9b50528cb782ae05036f115f15374600 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -44,6 +44,7 @@ typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq); typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp); typedef void (*RegisterBrokenLinkArgFp)(SMgmtWrapper* pWrapper, SRpcMsg* pMsg); typedef void (*ReleaseHandleFp)(SMgmtWrapper* pWrapper, void* handle, int8_t type); +typedef void (*ReportStartup)(SMgmtWrapper* pWrapper, const char* name, const char* desc); typedef struct { SMgmtWrapper* pWrapper; @@ -53,6 +54,7 @@ typedef struct { SendRspFp sendRspFp; RegisterBrokenLinkArgFp registerBrokenLinkArgFp; ReleaseHandleFp releaseHandleFp; + ReportStartup reportStartupFp; } SMsgCb; void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb); @@ -62,6 +64,7 @@ int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq); void tmsgSendRsp(const SRpcMsg* pRsp); void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg); void tmsgReleaseHandle(void* handle, int8_t type); +void tmsgReportStartup(const char* name, const char* desc); #ifdef __cplusplus } diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index b3c47353b8302bf4ded505a85f5f60b3af088584..d0f88a660c503fc7d5b992ec8f280c84a8aacaf2 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -82,12 +82,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_DND_ALTER_BNODE, "dnode-alter-bnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_BNODE, "dnode-drop-bnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CREATE_VNODE, "dnode-create-vnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE, "dnode-alter-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_SYNC_VNODE, "dnode-sync-vnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_COMPACT_VNODE, "dnode-compact-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_NETWORK_TEST, "dnode-nettest", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "dnode-server-status", NULL, NULL) // Requests handled by MNODE TD_NEW_MSG_SEG(TDMT_MND_MSG) @@ -149,6 +146,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp) @@ -180,6 +178,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp) + TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL) @@ -220,6 +219,10 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SYNC_UNKNOWN, "vnode-sync-unknown", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL) + + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_COMPACT_VNODE, "vnode-compact-vnode", NULL, NULL) // Requests handled by QNODE TD_NEW_MSG_SEG(TDMT_QND_MSG) @@ -243,7 +246,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_MON_SM_INFO, "monitor-sinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_BM_INFO, "monitor-binfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_VM_LOAD, "monitor-vload", NULL, NULL) - + TD_DEF_MSG_TYPE(TDMT_MON_MM_LOAD, "monitor-mload", NULL, NULL) + #if defined(TD_MSG_NUMBER_) TDMT_MAX #endif diff --git a/include/common/trow.h b/include/common/trow.h index 963542fb31f2641ab36c5d2d9f6c9aa7e77f1c27..bc5413f9cea8c81d8a2067d82ff6c1ed56e06808 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -219,7 +219,7 @@ static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return #define TD_ROW_OFFSET(p) ((p)->toffset); // During ParseInsert when without STSchema, how to get the offset for STpRow? -void tdMergeBitmap(uint8_t *srcBitmap, int32_t srcLen, uint8_t *dstBitmap); +void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap); static FORCE_INLINE void tdRowCopy(void *dst, STSRow *row) { memcpy(dst, row, TD_ROW_LEN(row)); } static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType); static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType); @@ -651,6 +651,8 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { TD_ROW_SET_INFO(pBuilder->pBuf, 0); TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); + TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); + uint32_t len = 0; switch (pBuilder->rowType) { case TD_ROW_TP: @@ -682,6 +684,43 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { return TSDB_CODE_SUCCESS; } +/** + * @brief The invoker is responsible for memory alloc/dealloc. + * + * @param pBuilder + * @param pBuf Output buffer of STSRow + */ +static int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { + pBuilder->pBuf = (STSRow *)pBuf; + if (!pBuilder->pBuf) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); + + uint32_t len = 0; + switch (pBuilder->rowType) { + case TD_ROW_TP: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); +#endif + break; + case TD_ROW_KV: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); +#endif + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + + /** * @brief 由调用方管理存储空间的分配及释放，一次输入多个参数 * @@ -1105,11 +1144,11 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col if (TD_IS_TP_ROW(pIter->pRow)) { STColumn *pCol = NULL; STSchema *pSchema = pIter->pSchema; - while (pIter->colIdx <= pSchema->numOfCols) { + while (pIter->colIdx < pSchema->numOfCols) { pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key if (colId == pCol->colId) { break; - } else if (colId < pCol->colId) { + } else if (pCol->colId < colId) { ++pIter->colIdx; continue; } else { @@ -1165,6 +1204,18 @@ static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, in return TSDB_CODE_SUCCESS; } +/** + * @brief + * + * @param pRow + * @param colId + * @param colType + * @param flen + * @param offset + * @param colIdx start from 0 + * @param pVal + * @return FORCE_INLINE + */ static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { @@ -1172,10 +1223,20 @@ static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t return true; } void *pBitmap = tdGetBitmapAddrTp(pRow, flen); - tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx - 1); + tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx); return true; } +/** + * @brief + * + * @param pRow + * @param colId + * @param offset + * @param colIdx start from 0 + * @param pVal + * @return FORCE_INLINE + */ static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { @@ -1183,7 +1244,7 @@ static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t o return true; } void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); - tdGetKvRowValOfCol(pVal, pRow, pBitmap, offset, colIdx - 1); + tdGetKvRowValOfCol(pVal, pRow, pBitmap, offset, colIdx); return true; } @@ -1213,6 +1274,101 @@ static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, return result; } +static void tdSCellValPrint(SCellVal *pVal, int8_t colType) { + if (tdValTypeIsNull(pVal->valType)) { + printf("NULL "); + return; + } else if (tdValTypeIsNone(pVal->valType)) { + printf("NONE "); + return; + } + switch (colType) { + case TSDB_DATA_TYPE_BOOL: + printf("%s ", (*(int8_t *)pVal->val) == 0 ? "false" : "true"); + break; + case TSDB_DATA_TYPE_TINYINT: + printf("%" PRIi8 " ", *(int8_t *)pVal->val); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf("%" PRIi16 " ", *(int16_t *)pVal->val); + break; + case TSDB_DATA_TYPE_INT: + printf("%" PRIi32 " ", *(int32_t *)pVal->val); + break; + case TSDB_DATA_TYPE_BIGINT: + printf("%" PRIi64 " ", *(int64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_FLOAT: + printf("%f ", *(float *)pVal->val); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf("%lf ", *(double *)pVal->val); + break; + case TSDB_DATA_TYPE_VARCHAR: + printf("VARCHAR "); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + printf("%" PRIi64 " ", *(int64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_NCHAR: + printf("NCHAR "); + break; + case TSDB_DATA_TYPE_UTINYINT: + printf("%" PRIu8 " ", *(uint8_t *)pVal->val); + break; + case TSDB_DATA_TYPE_USMALLINT: + printf("%" PRIu16 " ", *(uint16_t *)pVal->val); + break; + case TSDB_DATA_TYPE_UINT: + printf("%" PRIu32 " ", *(uint32_t *)pVal->val); + break; + case TSDB_DATA_TYPE_UBIGINT: + printf("%" PRIu64 " ", *(uint64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_JSON: + printf("JSON "); + break; + case TSDB_DATA_TYPE_VARBINARY: + printf("VARBIN "); + break; + case TSDB_DATA_TYPE_DECIMAL: + printf("DECIMAL "); + break; + case TSDB_DATA_TYPE_BLOB: + printf("BLOB "); + break; + case TSDB_DATA_TYPE_MEDIUMBLOB: + printf("MedBLOB "); + break; + // case TSDB_DATA_TYPE_BINARY: + // printf("BINARY "); + // break; + case TSDB_DATA_TYPE_MAX: + printf("UNDEF "); + break; + default: + printf("UNDEF "); + break; + } +} + +static void tdSRowPrint(STSRow *row, STSchema *pSchema) { + STSRowIter iter = {0}; + tdSTSRowIterInit(&iter, pSchema); + tdSTSRowIterReset(&iter, row); + printf(">>>"); + for (int i = 0; i < pSchema->numOfCols; ++i) { + STColumn *stCol = pSchema->columns + i; + SCellVal sVal = { 255, NULL}; + if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { + break; + } + ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2); + tdSCellValPrint(&sVal, stCol->type); + } + printf("\n"); +} + #ifdef TROW_ORIGIN_HZ typedef struct { uint32_t nRows; diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index c98e5bf99d3345f7612c2095bc21000f25dbe0d3..39858cbbdeb4a6218cfdf0d3a712c5107fa93716 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -86,155 +86,157 @@ #define TK_SINGLE_STABLE 68 #define TK_STREAM_MODE 69 #define TK_RETENTIONS 70 -#define TK_NK_COMMA 71 -#define TK_NK_COLON 72 -#define TK_TABLE 73 -#define TK_NK_LP 74 -#define TK_NK_RP 75 -#define TK_STABLE 76 -#define TK_ADD 77 -#define TK_COLUMN 78 -#define TK_MODIFY 79 -#define TK_RENAME 80 -#define TK_TAG 81 -#define TK_SET 82 -#define TK_NK_EQ 83 -#define TK_USING 84 -#define TK_TAGS 85 -#define TK_NK_DOT 86 -#define TK_COMMENT 87 -#define TK_BOOL 88 -#define TK_TINYINT 89 -#define TK_SMALLINT 90 -#define TK_INT 91 -#define TK_INTEGER 92 -#define TK_BIGINT 93 -#define TK_FLOAT 94 -#define TK_DOUBLE 95 -#define TK_BINARY 96 -#define TK_TIMESTAMP 97 -#define TK_NCHAR 98 -#define TK_UNSIGNED 99 -#define TK_JSON 100 -#define TK_VARCHAR 101 -#define TK_MEDIUMBLOB 102 -#define TK_BLOB 103 -#define TK_VARBINARY 104 -#define TK_DECIMAL 105 -#define TK_SMA 106 -#define TK_ROLLUP 107 -#define TK_FILE_FACTOR 108 -#define TK_NK_FLOAT 109 -#define TK_DELAY 110 -#define TK_SHOW 111 -#define TK_DATABASES 112 -#define TK_TABLES 113 -#define TK_STABLES 114 -#define TK_MNODES 115 -#define TK_MODULES 116 -#define TK_QNODES 117 -#define TK_FUNCTIONS 118 -#define TK_INDEXES 119 -#define TK_FROM 120 -#define TK_ACCOUNTS 121 -#define TK_APPS 122 -#define TK_CONNECTIONS 123 -#define TK_LICENCE 124 -#define TK_GRANTS 125 -#define TK_QUERIES 126 -#define TK_SCORES 127 -#define TK_TOPICS 128 -#define TK_VARIABLES 129 -#define TK_BNODES 130 -#define TK_SNODES 131 -#define TK_LIKE 132 -#define TK_INDEX 133 -#define TK_FULLTEXT 134 -#define TK_FUNCTION 135 -#define TK_INTERVAL 136 -#define TK_TOPIC 137 -#define TK_AS 138 -#define TK_DESC 139 -#define TK_DESCRIBE 140 -#define TK_RESET 141 -#define TK_QUERY 142 -#define TK_EXPLAIN 143 -#define TK_ANALYZE 144 -#define TK_VERBOSE 145 -#define TK_NK_BOOL 146 -#define TK_RATIO 147 -#define TK_COMPACT 148 -#define TK_VNODES 149 -#define TK_IN 150 -#define TK_OUTPUTTYPE 151 -#define TK_AGGREGATE 152 -#define TK_BUFSIZE 153 -#define TK_STREAM 154 -#define TK_INTO 155 -#define TK_TRIGGER 156 -#define TK_AT_ONCE 157 -#define TK_WINDOW_CLOSE 158 -#define TK_WATERMARK 159 -#define TK_KILL 160 -#define TK_CONNECTION 161 -#define TK_MERGE 162 -#define TK_VGROUP 163 -#define TK_REDISTRIBUTE 164 -#define TK_SPLIT 165 -#define TK_SYNCDB 166 -#define TK_NULL 167 -#define TK_NK_QUESTION 168 -#define TK_NK_ARROW 169 -#define TK_ROWTS 170 -#define TK_TBNAME 171 -#define TK_QSTARTTS 172 -#define TK_QENDTS 173 -#define TK_WSTARTTS 174 -#define TK_WENDTS 175 -#define TK_WDURATION 176 -#define TK_CAST 177 -#define TK_NOW 178 -#define TK_TODAY 179 -#define TK_TIMEZONE 180 -#define TK_COUNT 181 -#define TK_FIRST 182 -#define TK_LAST 183 -#define TK_LAST_ROW 184 -#define TK_BETWEEN 185 -#define TK_IS 186 -#define TK_NK_LT 187 -#define TK_NK_GT 188 -#define TK_NK_LE 189 -#define TK_NK_GE 190 -#define TK_NK_NE 191 -#define TK_MATCH 192 -#define TK_NMATCH 193 -#define TK_CONTAINS 194 -#define TK_JOIN 195 -#define TK_INNER 196 -#define TK_SELECT 197 -#define TK_DISTINCT 198 -#define TK_WHERE 199 -#define TK_PARTITION 200 -#define TK_BY 201 -#define TK_SESSION 202 -#define TK_STATE_WINDOW 203 -#define TK_SLIDING 204 -#define TK_FILL 205 -#define TK_VALUE 206 -#define TK_NONE 207 -#define TK_PREV 208 -#define TK_LINEAR 209 -#define TK_NEXT 210 -#define TK_GROUP 211 -#define TK_HAVING 212 -#define TK_ORDER 213 -#define TK_SLIMIT 214 -#define TK_SOFFSET 215 -#define TK_LIMIT 216 -#define TK_OFFSET 217 -#define TK_ASC 218 -#define TK_NULLS 219 +#define TK_STRICT 71 +#define TK_NK_COMMA 72 +#define TK_NK_COLON 73 +#define TK_TABLE 74 +#define TK_NK_LP 75 +#define TK_NK_RP 76 +#define TK_STABLE 77 +#define TK_ADD 78 +#define TK_COLUMN 79 +#define TK_MODIFY 80 +#define TK_RENAME 81 +#define TK_TAG 82 +#define TK_SET 83 +#define TK_NK_EQ 84 +#define TK_USING 85 +#define TK_TAGS 86 +#define TK_NK_DOT 87 +#define TK_COMMENT 88 +#define TK_BOOL 89 +#define TK_TINYINT 90 +#define TK_SMALLINT 91 +#define TK_INT 92 +#define TK_INTEGER 93 +#define TK_BIGINT 94 +#define TK_FLOAT 95 +#define TK_DOUBLE 96 +#define TK_BINARY 97 +#define TK_TIMESTAMP 98 +#define TK_NCHAR 99 +#define TK_UNSIGNED 100 +#define TK_JSON 101 +#define TK_VARCHAR 102 +#define TK_MEDIUMBLOB 103 +#define TK_BLOB 104 +#define TK_VARBINARY 105 +#define TK_DECIMAL 106 +#define TK_SMA 107 +#define TK_ROLLUP 108 +#define TK_FILE_FACTOR 109 +#define TK_NK_FLOAT 110 +#define TK_DELAY 111 +#define TK_SHOW 112 +#define TK_DATABASES 113 +#define TK_TABLES 114 +#define TK_STABLES 115 +#define TK_MNODES 116 +#define TK_MODULES 117 +#define TK_QNODES 118 +#define TK_FUNCTIONS 119 +#define TK_INDEXES 120 +#define TK_FROM 121 +#define TK_ACCOUNTS 122 +#define TK_APPS 123 +#define TK_CONNECTIONS 124 +#define TK_LICENCE 125 +#define TK_GRANTS 126 +#define TK_QUERIES 127 +#define TK_SCORES 128 +#define TK_TOPICS 129 +#define TK_VARIABLES 130 +#define TK_BNODES 131 +#define TK_SNODES 132 +#define TK_CLUSTER 133 +#define TK_LIKE 134 +#define TK_INDEX 135 +#define TK_FULLTEXT 136 +#define TK_FUNCTION 137 +#define TK_INTERVAL 138 +#define TK_TOPIC 139 +#define TK_AS 140 +#define TK_DESC 141 +#define TK_DESCRIBE 142 +#define TK_RESET 143 +#define TK_QUERY 144 +#define TK_EXPLAIN 145 +#define TK_ANALYZE 146 +#define TK_VERBOSE 147 +#define TK_NK_BOOL 148 +#define TK_RATIO 149 +#define TK_COMPACT 150 +#define TK_VNODES 151 +#define TK_IN 152 +#define TK_OUTPUTTYPE 153 +#define TK_AGGREGATE 154 +#define TK_BUFSIZE 155 +#define TK_STREAM 156 +#define TK_INTO 157 +#define TK_TRIGGER 158 +#define TK_AT_ONCE 159 +#define TK_WINDOW_CLOSE 160 +#define TK_WATERMARK 161 +#define TK_KILL 162 +#define TK_CONNECTION 163 +#define TK_MERGE 164 +#define TK_VGROUP 165 +#define TK_REDISTRIBUTE 166 +#define TK_SPLIT 167 +#define TK_SYNCDB 168 +#define TK_NULL 169 +#define TK_NK_QUESTION 170 +#define TK_NK_ARROW 171 +#define TK_ROWTS 172 +#define TK_TBNAME 173 +#define TK_QSTARTTS 174 +#define TK_QENDTS 175 +#define TK_WSTARTTS 176 +#define TK_WENDTS 177 +#define TK_WDURATION 178 +#define TK_CAST 179 +#define TK_NOW 180 +#define TK_TODAY 181 +#define TK_TIMEZONE 182 +#define TK_COUNT 183 +#define TK_FIRST 184 +#define TK_LAST 185 +#define TK_LAST_ROW 186 +#define TK_BETWEEN 187 +#define TK_IS 188 +#define TK_NK_LT 189 +#define TK_NK_GT 190 +#define TK_NK_LE 191 +#define TK_NK_GE 192 +#define TK_NK_NE 193 +#define TK_MATCH 194 +#define TK_NMATCH 195 +#define TK_CONTAINS 196 +#define TK_JOIN 197 +#define TK_INNER 198 +#define TK_SELECT 199 +#define TK_DISTINCT 200 +#define TK_WHERE 201 +#define TK_PARTITION 202 +#define TK_BY 203 +#define TK_SESSION 204 +#define TK_STATE_WINDOW 205 +#define TK_SLIDING 206 +#define TK_FILL 207 +#define TK_VALUE 208 +#define TK_NONE 209 +#define TK_PREV 210 +#define TK_LINEAR 211 +#define TK_NEXT 212 +#define TK_GROUP 213 +#define TK_HAVING 214 +#define TK_ORDER 215 +#define TK_SLIMIT 216 +#define TK_SOFFSET 217 +#define TK_LIMIT 218 +#define TK_OFFSET 219 +#define TK_ASC 220 +#define TK_NULLS 221 #define TK_NK_SPACE 300 #define TK_NK_COMMENT 301 diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 7f2e59ea852e2792e369849a348f6d7e7fd9cf83..405b20c521befb905d59fa71813db72966f3ea7d 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -49,7 +49,7 @@ typedef struct { #define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v)) #define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) -#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) +#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON)) #define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) @@ -252,7 +252,7 @@ typedef struct tDataTypeDescriptor { int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull); } tDataTypeDescriptor; -extern tDataTypeDescriptor tDataTypes[15]; +extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX]; bool isValidDataType(int32_t type); diff --git a/include/dnode/bnode/bnode.h b/include/dnode/bnode/bnode.h index 528cea8828a8cb63356ed04cb6514df876feb772..d5958d432bf494b82bccd608364a461302327a2d 100644 --- a/include/dnode/bnode/bnode.h +++ b/include/dnode/bnode/bnode.h @@ -26,6 +26,9 @@ extern "C" { typedef struct SBnode SBnode; typedef struct { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } SBnodeLoad; typedef struct { diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 98481259194ba6f87e99e7c465d21c84244747ef..eed91d7561d24b1abb50e9f9b72b77d999efb7c6 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -80,6 +80,7 @@ void mndStop(SMnode *pMnode); * @return int32_t 0 for success, -1 for failure. */ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pCluster, SMonVgroupInfo *pVgroup, SMonGrantInfo *pGrant); +int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); /** * @brief Process the read, write, sync request. diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index a28c093e85bfbb7c5e1c3789e71b23e700b0549c..d7d53ad1d04f12acef867bcbf0eebd33495c5e96 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -92,7 +92,13 @@ extern "C" { typedef struct SMnode SMnode; typedef struct SSdbRaw SSdbRaw; typedef struct SSdbRow SSdbRow; -typedef enum { SDB_KEY_BINARY = 1, SDB_KEY_INT32 = 2, SDB_KEY_INT64 = 3 } EKeyType; + +typedef enum { + SDB_KEY_BINARY = 1, + SDB_KEY_INT32 = 2, + SDB_KEY_INT64 = 3, +} EKeyType; + typedef enum { SDB_STATUS_INIT = 0, SDB_STATUS_CREATING = 1, diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 5abda69aa8b7a9213a60a6f03c14593eb3314600..b88afcb39df49c518b41968eee74638ca2143ae1 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -213,7 +213,6 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, void * pTransporter, const */ int32_t catalogGetAllMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); - int32_t catalogGetQnodeList(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList); int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num); @@ -224,6 +223,8 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); +int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo); + /** * Destroy catalog and relase all resources diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 1303a1fb6accb9894f875d4a22ba1e35e8129c48..68fc3be617d9292f5a2f618af6eb4e52b2ec7aba 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -166,6 +166,7 @@ typedef struct SInputColumnInfoData { SColumnInfoData *pPTS; // primary timestamp column SColumnInfoData **pData; SColumnDataAgg **pColumnDataAgg; + uint64_t uid; // table uid } SInputColumnInfoData; // sql function runtime context @@ -191,7 +192,7 @@ typedef struct SqlFunctionCtx { int16_t functionId; // function id char * pOutput; // final result output buffer, point to sdata->data int32_t numOfParams; - SVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param + SFunctParam *param; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param int64_t *ptsList; // corresponding timestamp array list SColumnInfoData *pTsOutput; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ int32_t offset; @@ -295,19 +296,19 @@ typedef struct SPoint { void * val; } SPoint; -void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); -void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); -void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); -struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const int64_t* fillVal); -bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); - -struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, - int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - struct SFillColInfo* pFillCol, const char* id); - -void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); -int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity); -int64_t getFillInfoStart(struct SFillInfo *pFillInfo); +//void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); +//void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); +//void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); +//struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const SValueNode* val); +//bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); +// +//struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, +// SInterval* pInterval, int32_t fillType, +// struct SFillColInfo* pCol, const char* id); +// +//void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); +//int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity); +//int64_t getFillInfoStart(struct SFillInfo *pFillInfo); int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType); diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 22bbc36ee13acb0411c22074cb09eeb013bd4738..2fff819f5496327b8b5264b30a23558331a8cbed 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -110,18 +110,28 @@ typedef enum EFunctionType { FUNCTION_TYPE_QENDTS, FUNCTION_TYPE_WSTARTTS, FUNCTION_TYPE_WENDTS, - FUNCTION_TYPE_WDURATION + FUNCTION_TYPE_WDURATION, + + // user defined funcion + FUNCTION_TYPE_UDF = 10000 } EFunctionType; struct SqlFunctionCtx; struct SResultRowEntryInfo; struct STimeWindow; +struct SCatalog; + +typedef struct SFmGetFuncInfoParam { + struct SCatalog* pCtg; + void *pRpc; + const SEpSet* pMgmtEps; +} SFmGetFuncInfoParam; int32_t fmFuncMgtInit(); void fmFuncMgtDestroy(); -int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType); +int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType); int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len); @@ -138,6 +148,7 @@ bool fmIsWindowClauseFunc(int32_t funcId); bool fmIsSpecialDataRequiredFunc(int32_t funcId); bool fmIsDynamicScanOptimizedFunc(int32_t funcId); bool fmIsMultiResFunc(int32_t funcId); +bool fmIsUserDefinedFunc(int32_t funcId); typedef enum EFuncDataRequired { FUNC_DATA_REQUIRED_DATA_LOAD = 1, diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index af0580674d950077f60c37f98e66c3451e91a479..9d8cf61b0646c764cee7056152f7873caa61b14f 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -202,6 +202,14 @@ int32_t tSerializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo) int32_t tDeserializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo); void tFreeSMonVloadInfo(SMonVloadInfo *pInfo); +typedef struct { + int8_t isMnode; + SMnodeLoad load; +} SMonMloadInfo; + +int32_t tSerializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInfo); +int32_t tDeserializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInfo); + typedef struct { const char *server; uint16_t port; diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index b62555d5492cb5e187aaea4fbc7889e31935ed81..0394dfd9bd9b5ede18f5f3d8a628a4a5e17c557c 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -47,6 +47,7 @@ typedef struct SDatabaseOptions { SValueNode* pNumOfVgroups; SValueNode* pSingleStable; SValueNode* pStreamMode; + SValueNode* pStrict; SNodeList* pRetentions; } SDatabaseOptions; @@ -294,6 +295,16 @@ typedef struct SDropStreamStmt { bool ignoreNotExists; } SDropStreamStmt; +typedef struct SCreateFunctionStmt { + ENodeType type; + bool ignoreExists; + char funcName[TSDB_FUNC_NAME_LEN]; + bool isAgg; + char libraryPath[PATH_MAX]; + SDataType outputDt; + int32_t bufSize; +} SCreateFunctionStmt; + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 8db93ee5f995de1b88abf3572edfd97d2be49d0a..a47bf4caee55a96c65fd353fe0daea353c9c2aee 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -48,6 +48,9 @@ extern "C" { (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ cell1 = cell1->pNext, cell2 = cell2->pNext) +#define REPLACE_LIST1_NODE(newNode) cell1->pNode = (SNode*)(newNode) +#define REPLACE_LIST2_NODE(newNode) cell2->pNode = (SNode*)(newNode) + #define FOREACH_FOR_REWRITE(node, list) \ for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext) @@ -134,6 +137,7 @@ typedef enum ENodeType { QUERY_NODE_SHOW_QNODES_STMT, QUERY_NODE_SHOW_SNODES_STMT, QUERY_NODE_SHOW_BNODES_STMT, + QUERY_NODE_SHOW_CLUSTER_STMT, QUERY_NODE_SHOW_DATABASES_STMT, QUERY_NODE_SHOW_FUNCTIONS_STMT, QUERY_NODE_SHOW_INDEXES_STMT, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 01e03a983d6e13f07af04ec857479b00036d5b1d..b4a290cbfca7d6c4b7d26d4db9c8dea752159326 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -155,7 +155,6 @@ typedef struct SLogicSubplan { typedef struct SQueryLogicPlan { ENodeType type; - int32_t totalLevel; SNodeList* pTopSubplans; } SQueryLogicPlan; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index d2f73e40715911ff456f39dfad3aa6953ed7b45e..143224d8f5a6f5f8907b1040336c98f240bf20cf 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -229,10 +229,10 @@ typedef struct SFillNode { typedef struct SSelectStmt { ENodeType type; // QUERY_NODE_SELECT_STMT bool isDistinct; - SNodeList* pProjectionList; // SNode + SNodeList* pProjectionList; SNode* pFromTable; SNode* pWhere; - SNodeList* pPartitionByList; // SNode + SNodeList* pPartitionByList; SNode* pWindow; SNodeList* pGroupByList; // SGroupingSetNode SNode* pHaving; @@ -245,12 +245,14 @@ typedef struct SSelectStmt { } SSelectStmt; typedef enum ESetOperatorType { - SET_OP_TYPE_UNION_ALL = 1 + SET_OP_TYPE_UNION_ALL = 1, + SET_OP_TYPE_UNION } ESetOperatorType; typedef struct SSetOperator { ENodeType type; // QUERY_NODE_SET_OPERATOR ESetOperatorType opType; + SNodeList* pProjectionList; SNode* pLeft; SNode* pRight; SNodeList* pOrderByList; // SOrderByExprNode @@ -283,12 +285,12 @@ typedef struct SVgDataBlocks { } SVgDataBlocks; typedef struct SVnodeModifOpStmt { - ENodeType nodeType; - ENodeType sqlNodeType; - SArray* pDataBlocks; // data block for each vgroup, SArray. - uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert - uint32_t insertType; // insert data from [file|sql statement| bound statement] - const char* sql; // current sql statement position + ENodeType nodeType; + ENodeType sqlNodeType; + SArray* pDataBlocks; // data block for each vgroup, SArray. + uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert + uint32_t insertType; // insert data from [file|sql statement| bound statement] + const char* sql; // current sql statement position } SVnodeModifOpStmt; typedef struct SExplainOptions { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 02cd07322622ab56ccc257f8f4bec140cecb73b7..58482735ba6a5e975bef2ca44fa9afc547d42172 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -21,6 +21,15 @@ extern "C" { #endif #include "querynodes.h" +#include "query.h" + +typedef struct SStmtCallback { + TAOS_STMT* pStmt; + int32_t (*getTbNameFn)(TAOS_STMT*, char**); + int32_t (*setBindInfoFn)(TAOS_STMT*, STableMeta*, void*); + int32_t (*setExecInfoFn)(TAOS_STMT*, SHashObj*, SHashObj*); + int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**); +} SStmtCallback; typedef struct SParseContext { uint64_t requestId; @@ -34,6 +43,7 @@ typedef struct SParseContext { char *pMsg; // extended error message if exists to help identifying the problem in sql statement. int32_t msgLen; // max length of the msg struct SCatalog *pCatalog; + SStmtCallback *pStmtCb; } SParseContext; typedef struct SCmdMsgInfo { @@ -66,11 +76,27 @@ typedef struct SQuery { } SQuery; int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); +bool isInsertSql(const char* pStr, size_t length); void qDestroyQuery(SQuery* pQueryNode); int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); +int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash); +int32_t qResetStmtDataBlock(void* block, bool keepBuf); +int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); +void qFreeStmtDataBlock(void* pDataBlock); +int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc); +void qDestroyStmtDataBlock(void* pBlock); +int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); +int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum); +int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields); +int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields); +int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); +void destroyBoundColumnInfo(void* pBoundInfo); +int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen); + + #ifdef __cplusplus } #endif diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index f343295c563669756da0ae5e809b1c0832820f23..851d4d63c56728853c75b8c8425f017e98af862a 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -30,6 +30,7 @@ typedef struct SPlanContext { SNode* pAstRoot; bool topicQuery; bool streamQuery; + bool rSmaQuery; bool showRewrite; int8_t triggerType; int64_t watermark; @@ -45,7 +46,6 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo // @pSource one execution location of this group of datasource subplans int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource); -typedef TAOS_MULTI_BIND TAOS_BIND_v2; // todo remove int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_BIND_v2* pParams); // Convert to subplan to string for the scheduler to send to the executor diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index bb550e75e8c5e0f8afbd619c29c1eb165ceadcc4..2b86b61cddd2ae3204cb12bedc06bbc47452dd01 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -51,6 +51,10 @@ typedef struct STableComInfo { typedef struct SIndexMeta { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SIndexMeta; diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 7d765bf13964ca4dc26594e88d221a66cd817c8a..0c7db45c4b43479db85ab43876697f9ab3baae9f 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -76,6 +76,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp /* Time related functions */ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 6705846d05eca112a60c0f9749d7e52c1ceedef6..0be27e6d3a65909deea9bbb5b638094cf598190f 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -20,12 +20,14 @@ extern "C" { #endif +#include #include -#include +//#include #include "cJSON.h" -#include "taosdef.h" -#include "trpc.h" -#include "wal.h" +#include "tdef.h" +//#include "taosdef.h" +//#include "trpc.h" +//#include "wal.h" typedef uint64_t SyncNodeId; typedef int32_t SyncGroupId; @@ -69,6 +71,9 @@ typedef struct SFsmCbMeta { uint64_t seqNum; } SFsmCbMeta; +struct SRpcMsg; +typedef struct SRpcMsg SRpcMsg; + typedef struct SSyncFSM { void* data; @@ -84,7 +89,7 @@ typedef struct SSyncFSM { struct SSyncRaftEntry; typedef struct SSyncRaftEntry SSyncRaftEntry; -#define SYNC_INDEX_BEGIN 0 +#define SYNC_INDEX_BEGIN 0 #define SYNC_INDEX_INVALID -1 // abstract definition of log store in raft @@ -115,6 +120,12 @@ typedef struct SSyncLogStore { } SSyncLogStore; +struct SWal; +typedef struct SWal SWal; + +struct SEpSet; +typedef struct SEpSet SEpSet; + typedef struct SSyncInfo { SyncGroupId vgId; SSyncCfg syncCfg; @@ -157,6 +168,8 @@ struct SSyncBuffer; typedef struct SSyncBuffer SSyncBuffer; //----------------------------------------- +const char* syncStr(ESyncState state); + #ifdef __cplusplus } #endif diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 0438d6d43648276ef609f5da66a09fe480d80ed1..8de4c7cd103a4bfd7a918be44a12acf276ef5af6 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -20,12 +20,13 @@ extern "C" { #endif +#include #include -#include +//#include #include "cJSON.h" -#include "taosdef.h" +//#include "taosdef.h" #include "trpc.h" -#include "wal.h" +//#include "wal.h" // ------------------ ds ------------------- typedef struct SRaftId { diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 614f358ac26c98da9f06b7a116723d6a32d6f859..65d3bd017159530ffcd94c34e173e0a9d58ea68c 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -60,6 +60,7 @@ typedef struct { typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *); typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); +typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *); typedef struct SRpcInit { uint16_t localPort; // local port @@ -81,7 +82,9 @@ typedef struct SRpcInit { // call back to retrieve the client auth info, for server app only RpcAfp afp; - ; + + // user defined retry func + RpcRfp rfp; void *parent; } SRpcInit; diff --git a/include/os/os.h b/include/os/os.h index 3ea94d009437e9cd32cfa1b95ce44d4f28fe2d3b..86abcc15f56efc565cb76126ab7073d087434462 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -22,7 +22,6 @@ extern "C" { #include #include -#include #include @@ -51,6 +50,15 @@ extern "C" { #include #include #endif +#else + +#include +#ifndef TD_USE_WINSOCK +#include +#else +#include +#endif + #endif diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index 5d45ced3edfafce8797e8386588ca01935a8d66c..e2a122a0fe4f220f00576941906e74b1e8036339 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -63,22 +63,22 @@ int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val); -void* atomic_add_fetch_ptr(void *ptr, int32_t val); +void* atomic_add_fetch_ptr(void *ptr, void *val); int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val); -void* atomic_fetch_add_ptr(void *ptr, int32_t val); +void* atomic_fetch_add_ptr(void *ptr, void *val); int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val); -void* atomic_sub_fetch_ptr(void *ptr, int32_t val); +void* atomic_sub_fetch_ptr(void *ptr, void *val); int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val); -void* atomic_fetch_sub_ptr(void *ptr, int32_t val); +void* atomic_fetch_sub_ptr(void *ptr, void *val); int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val); diff --git a/include/os/osDef.h b/include/os/osDef.h index 492df1047c75ee43010f9a10712d05668d598624..8da0c2c00cc7713d2d5dcdeda1b60634ea89b236 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -191,7 +191,7 @@ extern "C" { #define threadlocal __declspec( thread ) #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define PRIzu "ld" #else #define PRIzu "zu" diff --git a/include/os/osDir.h b/include/os/osDir.h index e7da54bf548ff4da95e800e0beccdda9e13c4df7..b549acde3706bd68e4b3f18aa24fb056eb96a189 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -36,12 +36,12 @@ typedef struct TdDirEntry *TdDirEntryPtr; void taosRemoveDir(const char *dirname); -bool taosDirExist(char *dirname); +bool taosDirExist(const char *dirname); int32_t taosMkDir(const char *dirname); int32_t taosMulMkDir(const char *dirname); void taosRemoveOldFiles(const char *dirname, int32_t keepDays); int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); -int32_t taosRealPath(char *dirname, int32_t maxlen); +int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen); bool taosIsDir(const char *dirname); char* taosDirName(char *dirname); char* taosDirEntryBaseName(char *dirname); @@ -50,7 +50,7 @@ TdDirPtr taosOpenDir(const char *dirname); TdDirEntryPtr taosReadDir(TdDirPtr pDir); bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry); char* taosGetDirEntryName(TdDirEntryPtr pDirEntry); -int32_t taosCloseDir(TdDirPtr pDir); +int32_t taosCloseDir(TdDirPtr *ppDir); #ifdef __cplusplus } diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 14d50858b77d23836091eed1508e755a12cd74fe..a3f92a0b2914dca8afe5e63e9c121694039d52ac 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -34,6 +34,7 @@ extern int64_t tsOpenMax; extern int64_t tsStreamMax; extern float tsNumOfCores; extern int64_t tsTotalMemoryKB; +extern char* tsProcPath; extern char configDir[]; extern char tsDataDir[]; diff --git a/include/os/osFile.h b/include/os/osFile.h index 36ca6fb8bb507cb604aca8399011960f9a14ca1d..b364d233efdf6b467e8cd24e2685e4e50ffc5f8e 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -93,7 +93,6 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); -void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); bool taosValidFile(TdFilePtr pFile); int32_t taosGetErrorFile(TdFilePtr pFile); diff --git a/include/os/osLz4.h b/include/os/osLz4.h index 09aaa553acc66fe71bd4e34576cf6911799ecbd4..5bb33242d45dd7918c7ef2a1a53b59076e9ce4c0 100644 --- a/include/os/osLz4.h +++ b/include/os/osLz4.h @@ -20,7 +20,7 @@ extern "C" { #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS int32_t BUILDIN_CLZL(uint64_t val); int32_t BUILDIN_CLZ(uint32_t val); int32_t BUILDIN_CTZL(uint64_t val); diff --git a/include/os/osMath.h b/include/os/osMath.h index 0b5e4bd93ceeb5f97064b4d011675a5e15fdbd70..cabb821844fc7f5d4a913d247291347af5cd0638 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -23,7 +23,7 @@ extern "C" { #define TPOW2(x) ((x) * (x)) #define TABS(x) ((x) > 0 ? (x) : -(x)) -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define TSWAP(a, b, c) \ do { \ @@ -33,6 +33,7 @@ extern "C" { } while (0) #define TMAX(a, b) (((a) > (b)) ? (a) : (b)) #define TMIN(a, b) (((a) < (b)) ? (a) : (b)) + #define TRANGE(aa, bb, cc) ((aa) = TMAX((aa), (bb)),(aa) = TMIN((aa), (cc))) #else diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 7af8dd37bffb9bad5e91abe21936e272442c2889..a47239089a5d41f4af7dd5b4a6ff9d1bc9d5cac4 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -19,32 +19,53 @@ // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC - #define socket SOCKET_FUNC_TAOS_FORBID - #define bind BIND_FUNC_TAOS_FORBID - #define listen LISTEN_FUNC_TAOS_FORBID - #define accept ACCEPT_FUNC_TAOS_FORBID - #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID - #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID - #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID - #define inet_addr INET_ADDR_FUNC_TAOS_FORBID - #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID + #define socket SOCKET_FUNC_TAOS_FORBID + #define bind BIND_FUNC_TAOS_FORBID + #define listen LISTEN_FUNC_TAOS_FORBID + #define accept ACCEPT_FUNC_TAOS_FORBID + #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID + #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID + #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID + #define inet_addr INET_ADDR_FUNC_TAOS_FORBID + #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID #endif #if defined(WINDOWS) - #include "winsock2.h" - #include - #include - #include + #if BYTE_ORDER == LITTLE_ENDIAN + #include + #define htobe16(x) _byteswap_ushort(x) + #define htole16(x) (x) + #define be16toh(x) _byteswap_ushort(x) + #define le16toh(x) (x) + + #define htobe32(x) _byteswap_ulong(x) + #define htole32(x) (x) + #define be32toh(x) _byteswap_ulong(x) + #define le32toh(x) (x) + + #define htobe64(x) _byteswap_uint64(x) + #define htole64(x) (x) + #define be64toh(x) _byteswap_uint64(x) + #define le64toh(x) (x) + #else + #error byte order not supported + #endif + + #define __BYTE_ORDER BYTE_ORDER + #define __BIG_ENDIAN BIG_ENDIAN + #define __LITTLE_ENDIAN LITTLE_ENDIAN + #define __PDP_ENDIAN PDP_ENDIAN + #else + #include + #include + + #if defined(_TD_DARWIN_64) + #include + #else #include - #include - - #if defined(_TD_DARWIN_64) - #include - #else - #include - #include - #endif + #include + #endif #endif #ifdef __cplusplus @@ -52,14 +73,10 @@ extern "C" { #endif #if defined(WINDOWS) - #define htobe64 htonll -#endif - -#if defined(WINDOWS) + typedef int socklen_t; #define TAOS_EPOLL_WAIT_TIME 100 typedef SOCKET eventfd_t; #define eventfd(a, b) -1 - typedef SOCKET EpollFd; #define EpollClose(pollFd) epoll_close(pollFd) #ifndef EPOLLWAKEUP #define EPOLLWAKEUP (1u << 29) @@ -79,31 +96,29 @@ extern "C" { #if defined(_TD_DARWIN_64) // #define htobe64 htonll -# include +#include -# define htobe16(x) OSSwapHostToBigInt16(x) -# define htole16(x) OSSwapHostToLittleInt16(x) -# define be16toh(x) OSSwapBigToHostInt16(x) -# define le16toh(x) OSSwapLittleToHostInt16(x) +#define htobe16(x) OSSwapHostToBigInt16(x) +#define htole16(x) OSSwapHostToLittleInt16(x) +#define be16toh(x) OSSwapBigToHostInt16(x) +#define le16toh(x) OSSwapLittleToHostInt16(x) -# define htobe32(x) OSSwapHostToBigInt32(x) -# define htole32(x) OSSwapHostToLittleInt32(x) -# define be32toh(x) OSSwapBigToHostInt32(x) -# define le32toh(x) OSSwapLittleToHostInt32(x) +#define htobe32(x) OSSwapHostToBigInt32(x) +#define htole32(x) OSSwapHostToLittleInt32(x) +#define be32toh(x) OSSwapBigToHostInt32(x) +#define le32toh(x) OSSwapLittleToHostInt32(x) -# define htobe64(x) OSSwapHostToBigInt64(x) -# define htole64(x) OSSwapHostToLittleInt64(x) -# define be64toh(x) OSSwapBigToHostInt64(x) -# define le64toh(x) OSSwapLittleToHostInt64(x) +#define htobe64(x) OSSwapHostToBigInt64(x) +#define htole64(x) OSSwapHostToLittleInt64(x) +#define be64toh(x) OSSwapBigToHostInt64(x) +#define le64toh(x) OSSwapLittleToHostInt64(x) -# define __BYTE_ORDER BYTE_ORDER -# define __BIG_ENDIAN BIG_ENDIAN -# define __LITTLE_ENDIAN LITTLE_ENDIAN -# define __PDP_ENDIAN PDP_ENDIAN +#define __BYTE_ORDER BYTE_ORDER +#define __BIG_ENDIAN BIG_ENDIAN +#define __LITTLE_ENDIAN LITTLE_ENDIAN +#define __PDP_ENDIAN PDP_ENDIAN #endif -#define TAOS_EPOLL_WAIT_TIME 500 - typedef int32_t SocketFd; typedef SocketFd EpollFd; @@ -122,7 +137,7 @@ typedef struct TdEpoll *TdEpollPtr; int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen); int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len); int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len); -int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen); +int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, int *addrLen); int32_t taosCloseSocketNoCheck1(SocketFd fd); int32_t taosCloseSocket(TdSocketPtr *ppSocket); int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer); @@ -139,14 +154,15 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes); int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len); +void taosWinSocketInit(); TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port); int32_t taosKeepTcpAlive(TdSocketPtr pSocket); -TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, socklen_t *addrLen); +TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen); -int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, socklen_t *addrLen); +int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, int *addrLen); void taosBlockSIGPIPE(); uint32_t taosGetIpv4FromFqdn(const char *); diff --git a/include/os/osString.h b/include/os/osString.h index 66d69a849c94d6a2c6c136d771c6ef0db3e15eef..a4100652c340e5a48446c9d97fb5d9bd44acb83e 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -39,7 +39,7 @@ typedef int32_t TdUcs4; #define wchar_t WCHAR_T_TYPE_TAOS_FORBID #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define tstrdup(str) _strdup(str) #else #define tstrdup(str) strdup(str) diff --git a/include/os/osThread.h b/include/os/osThread.h index 18c1b290abd55ad7813b68317bc973149b0268a8..6252a0cb60b83a4065ba99c992ce92574006dadd 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -22,9 +22,12 @@ extern "C" { #endif +#ifndef WINDOWS #ifndef __USE_XOPEN2K +#define TD_USE_SPINLOCK_AS_MUTEX typedef pthread_mutex_t pthread_spinlock_t; #endif +#endif typedef pthread_t TdThread; typedef pthread_spinlock_t TdThreadSpinlock; @@ -41,6 +44,13 @@ typedef pthread_key_t TdThreadKey; #define taosThreadCleanupPush pthread_cleanup_push #define taosThreadCleanupPop pthread_cleanup_pop + +#ifdef WINDOWS +#define TD_PTHREAD_MUTEX_INITIALIZER (TdThreadMutex)(-1) +#else +#define TD_PTHREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER +#endif + // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC @@ -218,8 +228,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock * lock); int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock); int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock); void taosThreadTestCancel(void); -int32_t taosThreadSigMask(int32_t how, sigset_t const *set, sigset_t * oset); -int32_t taosThreadSigWait(const sigset_t * set, int32_t *sig); +void taosThreadClear(TdThread *thread); #ifdef __cplusplus } diff --git a/include/os/osTime.h b/include/os/osTime.h index fd431f6df8e65e952ce63d54a31a54c32432dd27..b9e407cbcf0f664236f87dc01e7a1279e2a39150 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -20,8 +20,6 @@ extern "C" { #endif -#include - // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC @@ -34,7 +32,7 @@ extern "C" { #define mktime MKTIME_FUNC_TAOS_FORBID #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define CLOCK_REALTIME 0 @@ -58,6 +56,8 @@ extern "C" { int32_t taosGetTimeOfDay(struct timeval *tv); +int32_t taosClockGetTime(int clock_id, struct timespec *pTS); + //@return timestamp in second int32_t taosGetTimestampSec(); @@ -78,7 +78,7 @@ static FORCE_INLINE int64_t taosGetTimestampUs() { //@return timestamp in nanosecond static FORCE_INLINE int64_t taosGetTimestampNs() { struct timespec systemTime = {0}; - clock_gettime(CLOCK_REALTIME, &systemTime); + taosClockGetTime(CLOCK_REALTIME, &systemTime); return (int64_t)systemTime.tv_sec * 1000000000L + (int64_t)systemTime.tv_nsec; } diff --git a/include/util/talgo.h b/include/util/talgo.h index 0fd44a6e91d664580a23188061cf14b9637919ab..3ce26526087bfe393234aad696601839280fc479 100644 --- a/include/util/talgo.h +++ b/include/util/talgo.h @@ -27,6 +27,11 @@ extern "C" { typedef int32_t (*__compar_fn_t)(const void *, const void *); #endif +typedef void *(*FCopy)(void *); +typedef void (*FDelete)(void *); +typedef int32_t (*FEncode)(void **buf, const void *dst); +typedef void *(*FDecode)(const void *buf, void *dst); + #define TD_EQ 0x1 #define TD_GT 0x2 #define TD_LT 0x4 @@ -77,7 +82,7 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size * @return */ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, - __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot); + __ext_compar_fn_t compar, char* buf, bool maxroot); /** * sort heap to make sure it is a max/min root heap @@ -93,7 +98,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const * @return */ void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, - const void *parswap, __ext_swap_fn_t swap, bool maxroot); + bool maxroot); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 93ab133dc14394fcbf97ac892fb4d531dd96d427..00fd658fac21775adce2383d747873a5fb4b3b7d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -132,6 +132,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) #define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) #define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) +#define TSDB_CODE_TSC_STMT_API_ERROR TAOS_DEF_ERROR_CODE(0, 0X0225) +#define TSDB_CODE_TSC_STMT_TBNAME_ERROR TAOS_DEF_ERROR_CODE(0, 0X0226) +#define TSDB_CODE_TSC_STMT_CLAUSE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0227) // mnode-common #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300) @@ -279,6 +282,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8) #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9) #define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA) +#define TSDB_CODE_MND_CONSUMER_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x03EB) // mnode-stream #define TSDB_CODE_MND_STREAM_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F0) @@ -613,6 +617,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INTER_SLIDING_UNIT TAOS_DEF_ERROR_CODE(0, 0x2630) #define TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG TAOS_DEF_ERROR_CODE(0, 0x2631) #define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x2632) +#define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG TAOS_DEF_ERROR_CODE(0, 0x2633) +#define TSDB_CODE_PAR_INCORRECT_NUM_OF_COL TAOS_DEF_ERROR_CODE(0, 0x2634) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) @@ -621,6 +627,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FUNC_FUNTION_ERROR TAOS_DEF_ERROR_CODE(0, 0x2800) #define TSDB_CODE_FUNC_FUNTION_PARA_NUM TAOS_DEF_ERROR_CODE(0, 0x2801) #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2802) +#define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803) #ifdef __cplusplus } diff --git a/include/util/tarray.h b/include/util/tarray.h index 383af8309d5798e1d445a1fda2f7abeb7b53f2ea..a41bcd934917e815f2d71d9421a67e86332dfbaf 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -41,10 +41,10 @@ extern "C" { #define TARRAY_GET_START(array) ((array)->pData) typedef struct SArray { - size_t size; + size_t size; uint32_t capacity; uint32_t elemSize; - void* pData; + void* pData; } SArray; /** @@ -199,6 +199,13 @@ SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize); */ SArray* taosArrayDup(const SArray* pSrc); +/** + * deep copy a new array + * @param pSrc + */ +SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy); + + /** * clear the array (remove all element) * @param pArray @@ -212,19 +219,9 @@ void taosArrayClear(SArray* pArray); */ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)); - -/** - * destroy array list - * @param pArray - */ void* taosArrayDestroy(SArray* pArray); - -/** - * - * @param pArray - * @param fp - */ -void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)); +void taosArrayDestroyP(SArray* pArray, FDelete fp); +void taosArrayDestroyEx(SArray* pArray, FDelete fp); /** * sort the array @@ -272,6 +269,9 @@ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param); +int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode); +void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz); + #ifdef __cplusplus } #endif diff --git a/include/util/tcompare.h b/include/util/tcompare.h index cc9e8ae4641138be528830e17467dab7897f0166..ed07ae1c9cf5b0d06a181ec4d083d5fb8f98e06e 100644 --- a/include/util/tcompare.h +++ b/include/util/tcompare.h @@ -105,6 +105,8 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight); int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight); int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight); +int32_t compareJsonContainsKey(const void *pLeft, const void *pRight); + __compar_fn_t getComparFunc(int32_t type, int32_t optr); __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); int32_t doCompare(const char *a, const char *b, int32_t type, size_t size); diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 0e1d352f9a8b2cdc4ab9bffb728868738b8b9cac..012368d9cba4ce62cb11ca4d3bacad8724e574d5 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -32,6 +32,7 @@ typedef enum { CFG_STYPE_ENV_VAR, CFG_STYPE_APOLLO_URL, CFG_STYPE_ARG_LIST, + CFG_STYPE_TAOS_OPTIONS } ECfgSrcType; typedef enum { diff --git a/include/util/tdef.h b/include/util/tdef.h index ec90dd888a3719a52035feddd91c645cd7eab419..2548df71864381b2ccc234e3d79656c47993d211 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -239,6 +239,8 @@ typedef enum ELogicConditionType { #define TSDB_FUNC_BUF_SIZE 512 #define TSDB_FUNC_TYPE_SCALAR 1 #define TSDB_FUNC_TYPE_AGGREGATE 2 +#define TSDB_FUNC_SCRIPT_BIN_LIB 0 +#define TSDB_FUNC_SCRIPT_LUA 1 #define TSDB_FUNC_MAX_RETRIEVE 1024 #define TSDB_INDEX_NAME_LEN 65 // 64 + 1 '\0' @@ -271,6 +273,8 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 +#define TSDB_MAX_JSON_TAG_LEN 16384 + #define TSDB_AUTH_LEN 16 #define TSDB_PASSWORD_LEN 32 #define TSDB_USET_PASSWORD_LEN 129 @@ -297,16 +301,6 @@ typedef enum ELogicConditionType { #define TSDB_DNODE_CONFIG_LEN 128 #define TSDB_DNODE_VALUE_LEN 256 -#define TSDB_MQTT_HOSTNAME_LEN 64 -#define TSDB_MQTT_PORT_LEN 8 -#define TSDB_MQTT_USER_LEN 24 -#define TSDB_MQTT_PASS_LEN 24 -#define TSDB_MQTT_TOPIC_LEN 64 -#define TSDB_MQTT_CLIENT_ID_LEN 32 - -#define TSDB_DB_TYPE_DEFAULT 0 -#define TSDB_DB_TYPE_TOPIC 1 - #define TSDB_DEFAULT_PKT_SIZE 65480 // same as RPC_MAX_UDP_SIZE #define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE @@ -315,9 +309,6 @@ typedef enum ELogicConditionType { #define TSDB_CQ_SQL_SIZE 1024 #define TSDB_MIN_VNODES 16 #define TSDB_MAX_VNODES 512 -#define TSDB_MIN_VNODES_PER_DB 1 -#define TSDB_MAX_VNODES_PER_DB 4096 -#define TSDB_DEFAULT_VN_PER_DB 2 #define TSDB_DNODE_ROLE_ANY 0 #define TSDB_DNODE_ROLE_MGMT 1 @@ -331,104 +322,81 @@ typedef enum ELogicConditionType { #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta +#define TSDB_MIN_VNODES_PER_DB 1 +#define TSDB_MAX_VNODES_PER_DB 4096 +#define TSDB_DEFAULT_VN_PER_DB 2 #define TSDB_MIN_CACHE_BLOCK_SIZE 1 #define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode #define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 - -#define TSDB_MIN_TOTAL_BLOCKS 3 -#define TSDB_MAX_TOTAL_BLOCKS 10000 -#define TSDB_DEFAULT_TOTAL_BLOCKS 6 - -#define TSDB_MIN_DAYS_PER_FILE 60 // unit minute -#define TSDB_MAX_DAYS_PER_FILE (3650 * 1440) -#define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440) - -#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute -#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. -#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years - -#define TSDB_MIN_MIN_ROW_FBLOCK 10 -#define TSDB_MAX_MIN_ROW_FBLOCK 1000 -#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 - -#define TSDB_MIN_MAX_ROW_FBLOCK 200 -#define TSDB_MAX_MAX_ROW_FBLOCK 10000 -#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 - -#define TSDB_MIN_COMMIT_TIME 30 -#define TSDB_MAX_COMMIT_TIME 40960 -#define TSDB_DEFAULT_COMMIT_TIME 3600 - -#define TSDB_MIN_FSYNC_PERIOD 0 -#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond -#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second - -#define TSDB_MIN_WAL_LEVEL 1 -#define TSDB_MAX_WAL_LEVEL 2 -#define TSDB_DEFAULT_WAL_LEVEL 1 - -#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI -#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO -#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI - -#define TSDB_MIN_COMP_LEVEL 0 -#define TSDB_MAX_COMP_LEVEL 2 -#define TSDB_DEFAULT_COMP_LEVEL 2 - -#define TSDB_MIN_DB_REPLICA_OPTION 1 -#define TSDB_MAX_DB_REPLICA_OPTION 3 -#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 - -#define TSDB_MIN_DB_QUORUM_OPTION 1 -#define TSDB_MAX_DB_QUORUM_OPTION 2 -#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 - -#define TSDB_MIN_DB_TTL_OPTION 1 -#define TSDB_DEFAULT_DB_TTL_OPTION 0 - -#define TSDB_MIN_DB_SINGLE_STABLE_OPTION 0 -#define TSDB_MAX_DB_SINGLE_STABLE_OPTION 1 -#define TSDB_DEFAULT_DB_SINGLE_STABLE_OPTION 0 - -#define TSDB_MIN_DB_STREAM_MODE_OPTION 0 -#define TSDB_MAX_DB_STREAM_MODE_OPTION 1 -#define TSDB_DEFAULT_DB_STREAM_MODE_OPTION 0 - -#define TSDB_MAX_JOIN_TABLE_NUM 10 -#define TSDB_MAX_UNION_CLAUSE 5 - +#define TSDB_MIN_TOTAL_BLOCKS 3 +#define TSDB_MAX_TOTAL_BLOCKS 10000 +#define TSDB_DEFAULT_TOTAL_BLOCKS 6 +#define TSDB_MIN_DAYS_PER_FILE 60 // unit minute +#define TSDB_MAX_DAYS_PER_FILE (3650 * 1440) +#define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440) +#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute +#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. +#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years +#define TSDB_MIN_MINROWS_FBLOCK 10 +#define TSDB_MAX_MINROWS_FBLOCK 1000 +#define TSDB_DEFAULT_MINROWS_FBLOCK 100 +#define TSDB_MIN_MAXROWS_FBLOCK 200 +#define TSDB_MAX_MAXROWS_FBLOCK 10000 +#define TSDB_DEFAULT_MAXROWS_FBLOCK 4096 +#define TSDB_MIN_COMMIT_TIME 30 +#define TSDB_MAX_COMMIT_TIME 40960 +#define TSDB_DEFAULT_COMMIT_TIME 3600 +#define TSDB_MIN_FSYNC_PERIOD 0 +#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond +#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second +#define TSDB_MIN_DB_TTL 1 +#define TSDB_DEFAULT_DB_TTL 1 +#define TSDB_MIN_WAL_LEVEL 1 +#define TSDB_MAX_WAL_LEVEL 2 +#define TSDB_DEFAULT_WAL_LEVEL 1 +#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI +#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO +#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI +#define TSDB_MIN_COMP_LEVEL 0 +#define TSDB_MAX_COMP_LEVEL 2 +#define TSDB_DEFAULT_COMP_LEVEL 2 +#define TSDB_MIN_DB_REPLICA 1 +#define TSDB_MAX_DB_REPLICA 3 +#define TSDB_DEFAULT_DB_REPLICA 1 +#define TSDB_DB_STRICT_OFF 0 +#define TSDB_DB_STRICT_ON 1 +#define TSDB_DEFAULT_DB_STRICT 0 #define TSDB_MIN_DB_UPDATE 0 #define TSDB_MAX_DB_UPDATE 2 -#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 - -#define TSDB_MIN_DB_CACHE_LAST_ROW 0 -#define TSDB_MAX_DB_CACHE_LAST_ROW 3 -#define TSDB_DEFAULT_CACHE_LAST_ROW 0 - -#define TSDB_MIN_DB_STREAM_MODE 0 -#define TSDB_MAX_DB_STREAM_MODE 1 -#define TSDB_DEFAULT_DB_STREAM_MODE 0 +#define TSDB_DEFAULT_DB_UPDATE 0 +#define TSDB_MIN_DB_CACHE_LAST_ROW 0 +#define TSDB_MAX_DB_CACHE_LAST_ROW 3 +#define TSDB_DEFAULT_CACHE_LAST_ROW 0 +#define TSDB_DB_STREAM_MODE_OFF 0 +#define TSDB_DB_STREAM_MODE_ON 1 +#define TSDB_DEFAULT_DB_STREAM_MODE 0 +#define TSDB_DB_SINGLE_STABLE_ON 0 +#define TSDB_DB_SINGLE_STABLE_OFF 1 +#define TSDB_DEFAULT_DB_SINGLE_STABLE 0 #define TSDB_MIN_DB_FILE_FACTOR 0 #define TSDB_MAX_DB_FILE_FACTOR 1 #define TSDB_DEFAULT_DB_FILE_FACTOR 0.1 +#define TSDB_MIN_DB_DELAY 1 +#define TSDB_MAX_DB_DELAY 10 +#define TSDB_DEFAULT_DB_DELAY 2 +#define TSDB_MIN_EXPLAIN_RATIO 0 +#define TSDB_MAX_EXPLAIN_RATIO 1 +#define TSDB_DEFAULT_EXPLAIN_RATIO 0.001 -#define TSDB_MIN_DB_DELAY 1 -#define TSDB_MAX_DB_DELAY 10 -#define TSDB_DEFAULT_DB_DELAY 2 +#define TSDB_MAX_JOIN_TABLE_NUM 10 +#define TSDB_MAX_UNION_CLAUSE 5 #define TSDB_DEFAULT_EXPLAIN_VERBOSE false -#define TSDB_MIN_EXPLAIN_RATIO 0 -#define TSDB_MAX_EXPLAIN_RATIO 1 -#define TSDB_DEFAULT_EXPLAIN_RATIO 0.001 - #define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024 #define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN" -#define TSDB_MAX_JOIN_TABLE_NUM 10 -#define TSDB_MAX_UNION_CLAUSE 5 - #define TSDB_MAX_FIELD_LEN 16384 #define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384 #define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384 diff --git a/include/util/tencode.h b/include/util/tencode.h index 7c877ae4283fc662cf5cd43d59b571e931b08a02..11357ba533fe00b25ddaefac0a80db22679ac344 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -236,7 +236,8 @@ static FORCE_INLINE int32_t tEncodeFloat(SCoder* pEncoder, float val) { union { uint32_t ui; float f; - } v = {.f = val}; + } v; + v.f = val; return tEncodeU32(pEncoder, v.ui); } @@ -245,7 +246,8 @@ static FORCE_INLINE int32_t tEncodeDouble(SCoder* pEncoder, double val) { union { uint64_t ui; double d; - } v = {.d = val}; + } v; + v.d = val; return tEncodeU64(pEncoder, v.ui); } diff --git a/include/util/tjson.h b/include/util/tjson.h index 6b2221f704c768e494ba797913ede2deb5f6a7f4..286a7e65fcd4780a653df5a61fc1d5de21253b62 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -22,6 +22,9 @@ extern "C" { #endif +#ifdef WINDOWS +#define tjsonGetNumberValue(pJson, pName, val) -1 +#else #define tjsonGetNumberValue(pJson, pName, val) \ ({ \ uint64_t _tmp = 0; \ @@ -29,6 +32,7 @@ extern "C" { val = _tmp; \ _code; \ }) +#endif typedef void SJson; @@ -76,6 +80,7 @@ char* tjsonToString(const SJson* pJson); char* tjsonToUnformattedString(const SJson* pJson); SJson* tjsonParse(const char* pStr); +bool tjsonValidateJson(const char* pJson); #ifdef __cplusplus } diff --git a/include/util/tlog.h b/include/util/tlog.h index d3ab9b0bfb76b566ee91101c2fc8fb7298ace2c4..32421a59ccb0dcbfe88a7c7192f2badb85baccee 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -59,6 +59,7 @@ extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; extern int32_t tqDebugFlag; extern int32_t fsDebugFlag; +extern int32_t fnDebugFlag; int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); diff --git a/include/util/tutil.h b/include/util/tutil.h index e0f92be76a90d5abbfa22366d8513f3e06facbbd..444a893a888abb5c28de0c70476af9beee32cc10 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -26,8 +26,6 @@ extern "C" { #endif int32_t strdequote(char *src); -int32_t strndequote(char *dst, const char *z, int32_t len); -int32_t strRmquote(char *z, int32_t len); size_t strtrim(char *src); char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote); char **strsplit(char *src, const char *delim, int32_t *num); diff --git a/packaging/install.sh b/packaging/install.sh index 9ce7fc326a9685aa40ad3a08b70f78656770c792..740d356f800bc214d4ff378c2097d98db5d16bdc 100755 --- a/packaging/install.sh +++ b/packaging/install.sh @@ -30,6 +30,15 @@ bin_dir="/usr/local/taos/bin" service_config_dir="/etc/systemd/system" +#taos-tools para +demoName="taosdemo" +benchmarkName="taosBenchmark" +dumpName="taosdump" +emailName="taosdata.com" +taosName="taos" +toolsName="taostools" + + # Color setting RED='\033[0;31m' GREEN='\033[1;32m' @@ -230,8 +239,20 @@ function install_header() { # temp install taosBenchmark function install_taosTools() { - cd ${script_dir}/taos-tools/ - tar xvf taosTools-1.4.1-Linux-x64.tar.gz && cd taosTools-1.4.1/ && ./install-taostools.sh + ${csudo} rm -f ${bin_link_dir}/${benchmarkName} || : + ${csudo} rm -f ${bin_link_dir}/${dumpName} || : + ${csudo} rm -f ${bin_link_dir}/rm${toolsName} || : + + ${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${dumpName} ${install_main_dir}/bin/${dumpName} + ${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${benchmarkName} + ${csudo} ln -sf ${install_main_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${demoName} + #Make link + [[ -x ${install_main_dir}/bin/${benchmarkName} ]] && \ + ${csudo} ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : + [[ -x ${install_main_dir}/bin/${demoName} ]] && \ + ${csudo} ln -s ${install_main_dir}/bin/${demoName} ${bin_link_dir}/${demoName} || : + [[ -x ${install_main_dir}/bin/${dumpName} ]] && \ + ${csudo} ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : } function add_newHostname_to_hosts() { diff --git a/packaging/release.sh b/packaging/release.sh index adf1195e5654bc48a021362c7e82426a2a4e8f11..a56d991bf889800ffbc5954e20fb1ebd28127b77 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -39,7 +39,7 @@ cd ${compile_dir} echo "compile_dir: ${compile_dir}" -cmake .. +cmake .. -DBUILD_TOOLS=true make -j32 release_dir="${top_dir}/release" @@ -55,7 +55,6 @@ mkdir -p ${install_dir} mkdir -p ${install_dir}/bin mkdir -p ${install_dir}/lib mkdir -p ${install_dir}/inc -mkdir -p ${install_dir}/taos-tools install_files="${script_dir}/install.sh" chmod a+x ${script_dir}/install.sh || : @@ -64,13 +63,14 @@ cp ${install_files} ${install_dir} header_files="${top_dir}/include/client/taos.h ${top_dir}/include/util/taoserror.h" cp ${header_files} ${install_dir}/inc -bin_files="${compile_dir}/build/bin/taosd ${compile_dir}/build/bin/taos ${compile_dir}/build/bin/create_table ${compile_dir}/build/bin/tmq_sim ${script_dir}/remove.sh" -cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +bin_files="${compile_dir}/source/dnode/mgmt/taosd ${compile_dir}/tools/shell/taos ${compile_dir}/tests/test/c/create_table ${compile_dir}/tests/test/c/tmq_sim ${script_dir}/remove.sh ${compile_dir}/build/bin/taosBenchmark ${compile_dir}/build/bin/taosdump" +cp -rf ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : + +cp -rf ${compile_dir}/source/client/libtaos.so ${install_dir}/lib/ +cp -rf ${compile_dir}/source/libs/tdb/libtdb.so ${install_dir}/lib/ +cp -rf ${compile_dir}/build/lib/libavro* ${install_dir}/lib/ > /dev/null || echo -e "failed to copy avro libraries" +cp -rf ${compile_dir}/build/lib/pkgconfig ${install_dir}/lib/ > /dev/null || echo -e "failed to copy pkgconfig directory" -cp ${compile_dir}/build/lib/libtaos.so ${install_dir}/lib/ -cp ${compile_dir}/build/lib/libtdb.so ${install_dir}/lib/ -taostoolfile="${top_dir}/tools/taosTools-1.4.1-Linux-x64.tar.gz" -cp ${taostoolfile} ${install_dir}/taos-tools #cp ${compile_dir}/source/dnode/mnode/impl/libmnode.so ${install_dir}/lib/ #cp ${compile_dir}/source/dnode/qnode/libqnode.so ${install_dir}/lib/ diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 772ff5e69afe566c2f7f53394e8b409c051f6ae3..814caf330ab8d938d04b7b4c261bfc4c9c71b5f8 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -44,7 +44,7 @@ extern "C" { } while (0) #define ERROR_MSG_BUF_DEFAULT_SIZE 512 -#define HEARTBEAT_INTERVAL 1500 // ms +#define HEARTBEAT_INTERVAL 1500 // ms enum { RES_TYPE__QUERY = 1, @@ -187,11 +187,13 @@ typedef struct SRequestSendRecvBody { } SRequestSendRecvBody; typedef struct { - int8_t resType; - char* topic; - SArray* res; // SArray - int32_t resIter; - int32_t vgId; + int8_t resType; + char topic[TSDB_TOPIC_FNAME_LEN]; + int32_t vgId; + SSchemaWrapper schema; + int32_t resIter; + SMqDataBlkRsp rsp; + SReqResultInfo resInfo; } SMqRspObj; typedef struct SRequestObj { @@ -203,7 +205,8 @@ typedef struct SRequestObj { char* sqlstr; // sql string int32_t sqlLen; int64_t self; - char* msgBuf; // error msg buffer + char* msgBuf; + int32_t msgBufLen; int32_t code; SArray* dbList; SArray* tableList; @@ -211,16 +214,25 @@ typedef struct SRequestObj { SRequestSendRecvBody body; } SRequestObj; +void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); +void doSetOneRowPtr(SReqResultInfo* pResultInfo); +void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); +int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4); +void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); +void doFreeReqResultInfo(SReqResultInfo* pResInfo); + static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; - int32_t resIter = msg->resIter == -1 ? 0 : msg->resIter; - return (SReqResultInfo*)taosArrayGet(msg->res, resIter); + return (SReqResultInfo*)&msg->resInfo; } -static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res) { +static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { SMqRspObj* msg = (SMqRspObj*)res; - if (++msg->resIter < taosArrayGetSize(msg->res)) { - return (SReqResultInfo*)taosArrayGet(msg->res, msg->resIter); + msg->resIter++; + if (msg->resIter < msg->rsp.blockNum) { + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter); + setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4); + return &msg->resInfo; } return NULL; } @@ -238,25 +250,25 @@ extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); -int taos_init(); +int taos_init(); -void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo); -void destroyTscObj(void* pObj); -STscObj *acquireTscObj(int64_t rid); -int32_t releaseTscObj(int64_t rid); +void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo); +void destroyTscObj(void* pObj); +STscObj* acquireTscObj(int64_t rid); +int32_t releaseTscObj(int64_t rid); uint64_t generateRequestId(); -void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); -void destroyRequest(SRequestObj* pRequest); -SRequestObj *acquireRequest(int64_t rid); -int32_t releaseRequest(int64_t rid); +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); +void destroyRequest(SRequestObj* pRequest); +SRequestObj* acquireRequest(int64_t rid); +int32_t releaseRequest(int64_t rid); char* getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); void resetConnectDB(STscObj* pTscObj); -int taos_options_imp(TSDB_OPTION option, const char* str); +int taos_options_imp(TSDB_OPTION option, const char* str); void* openTransporter(const char* user, const char* auth, int32_t numOfThreads); @@ -268,17 +280,12 @@ void initMsgHandleFp(); TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, uint16_t port, int connType); -int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery); +int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb); + int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList); int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); -void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); -void doSetOneRowPtr(SReqResultInfo* pResultInfo); -void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); -void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); -int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4); - // --- heartbeat // global, called by mgmt int hbMgrInit(); @@ -290,7 +297,7 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key); void appHbMgrCleanup(void); // conn level -int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); +int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen); @@ -298,6 +305,8 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v // --- mq void hbMgrInitMqHbRspHandle(); +SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery); + #ifdef __cplusplus } #endif diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index c29361758dda67b079ca613651550c8d3ccb5ca8..04e9c3be9a9478d4086f84721adf46332a51777c 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -19,6 +19,9 @@ #ifdef __cplusplus extern "C" { #endif +#include "catalog.h" + +typedef void STableDataBlocks; typedef enum { STMT_TYPE_INSERT = 1, @@ -26,17 +29,65 @@ typedef enum { STMT_TYPE_QUERY, } STMT_TYPE; +typedef enum { + STMT_INIT = 1, + STMT_PREPARE, + STMT_SETTBNAME, + STMT_SETTAGS, + STMT_FETCH_TAG_FIELDS, + STMT_FETCH_COL_FIELDS, + STMT_BIND, + STMT_BIND_COL, + STMT_ADD_BATCH, + STMT_EXECUTE, +} STMT_STATUS; + +typedef struct SStmtTableCache { + STableDataBlocks* pDataBlock; + void* boundTags; +} SStmtTableCache; + +typedef struct SStmtBindInfo { + bool needParse; + uint64_t tbUid; + uint64_t tbSuid; + int32_t sBindRowNum; + int32_t sBindLastIdx; + int8_t tbType; + void* boundTags; + char* tbName; + SName sname; +} SStmtBindInfo; + +typedef struct SStmtExecInfo { + int32_t affectedRows; + SRequestObj* pRequest; + SHashObj* pVgHash; + SHashObj* pBlockHash; +} SStmtExecInfo; + +typedef struct SStmtSQLInfo { + STMT_TYPE type; + STMT_STATUS status; + bool autoCreate; + uint64_t runTimes; + SHashObj* pTableCache; //SHash + SQuery* pQuery; + char* sqlStr; + int32_t sqlLen; +} SStmtSQLInfo; + typedef struct STscStmt { - STMT_TYPE type; - //int16_t last; - //STscObj* taos; - //SSqlObj* pSql; - //SMultiTbStmt mtb; - //SNormalStmt normal; - - //int numOfRows; + STscObj* taos; + SCatalog* pCatalog; + int32_t affectedRows; + + SStmtSQLInfo sql; + SStmtExecInfo exec; + SStmtBindInfo bInfo; } STscStmt; + #define STMT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define STMT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define STMT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) @@ -44,16 +95,16 @@ typedef struct STscStmt { TAOS_STMT *stmtInit(TAOS *taos); int stmtClose(TAOS_STMT *stmt); int stmtExec(TAOS_STMT *stmt); -char *stmtErrstr(TAOS_STMT *stmt); +const char *stmtErrstr(TAOS_STMT *stmt); int stmtAffectedRows(TAOS_STMT *stmt); -int stmtBind(TAOS_STMT *stmt, TAOS_BIND *bind); int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length); -int stmtSetTbNameTags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags); +int stmtSetTbName(TAOS_STMT *stmt, const char *tbName); +int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags); int stmtIsInsert(TAOS_STMT *stmt, int *insert); int stmtGetParamNum(TAOS_STMT *stmt, int *nums); int stmtAddBatch(TAOS_STMT *stmt); TAOS_RES *stmtUseResult(TAOS_STMT *stmt); -int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); +int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx); #ifdef __cplusplus diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 865c3ead27ce59c29f7815d5ba4c9694056c458b..9a34334466d34cee74115f4735d15979215772ea 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -30,15 +30,15 @@ #define TSC_VAR_RELEASED 0 SAppInfo appInfo; -int32_t clientReqRefPool = -1; +int32_t clientReqRefPool = -1; int32_t clientConnRefPool = -1; -static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; -volatile int32_t tscInitRes = 0; +static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; +volatile int32_t tscInitRes = 0; static void registerRequest(SRequestObj *pRequest) { STscObj *pTscObj = acquireTscObj(pRequest->pTscObj->id); - + assert(pTscObj != NULL); // connection has been released already, abort creating request. @@ -49,8 +49,8 @@ static void registerRequest(SRequestObj *pRequest) { if (pTscObj->pAppInfo) { SInstanceSummary *pSummary = &pTscObj->pAppInfo->summary; - int32_t total = atomic_add_fetch_64((int64_t*)&pSummary->totalRequests, 1); - int32_t currentInst = atomic_add_fetch_64((int64_t*)&pSummary->currentRequests, 1); + int32_t total = atomic_add_fetch_64((int64_t *)&pSummary->totalRequests, 1); + int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1); tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); @@ -60,16 +60,16 @@ static void registerRequest(SRequestObj *pRequest) { static void deregisterRequest(SRequestObj *pRequest) { assert(pRequest != NULL); - STscObj * pTscObj = pRequest->pTscObj; + STscObj *pTscObj = pRequest->pTscObj; SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary; - int32_t currentInst = atomic_sub_fetch_64((int64_t*)&pActivity->currentRequests, 1); + int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); int64_t duration = taosGetTimestampUs() - pRequest->metric.start; tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 " ms, current:%d, app current:%d", - pRequest->self, pTscObj->id, pRequest->requestId, duration/1000, num, currentInst); + pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst); releaseTscObj(pTscObj->id); } @@ -109,12 +109,12 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { } void closeAllRequests(SHashObj *pRequests) { - void *pIter = taosHashIterate(pRequests, NULL); + void *pIter = taosHashIterate(pRequests, NULL); while (pIter != NULL) { int64_t *rid = pIter; - releaseRequest(*rid); - + releaseRequest(*rid); + pIter = taosHashIterate(pRequests, pIter); } } @@ -144,7 +144,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } - + pObj->pAppInfo = pAppInfo; tstrncpy(pObj->user, user, sizeof(pObj->user)); memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN); @@ -160,13 +160,9 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI return pObj; } -STscObj *acquireTscObj(int64_t rid) { - return (STscObj *)taosAcquireRef(clientConnRefPool, rid); -} +STscObj *acquireTscObj(int64_t rid) { return (STscObj *)taosAcquireRef(clientConnRefPool, rid); } -int32_t releaseTscObj(int64_t rid) { - return taosReleaseRef(clientConnRefPool, rid); -} +int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, rid); } void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) { assert(pObj != NULL); @@ -186,14 +182,15 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty pRequest->pTscObj = pObj; pRequest->body.fp = fp; // not used it yet pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); + pRequest->msgBufLen = ERROR_MSG_BUF_DEFAULT_SIZE; tsem_init(&pRequest->body.rspSem, 0, 0); registerRequest(pRequest); - + return pRequest; } -static void doFreeReqResultInfo(SReqResultInfo *pResInfo) { +void doFreeReqResultInfo(SReqResultInfo *pResInfo) { taosMemoryFreeClear(pResInfo->pRspMsg); taosMemoryFreeClear(pResInfo->length); taosMemoryFreeClear(pResInfo->row); @@ -216,7 +213,7 @@ static void doDestroyRequest(void *p) { assert(RID_VALID(pRequest->self)); taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); - + taosMemoryFreeClear(pRequest->msgBuf); taosMemoryFreeClear(pRequest->sqlstr); taosMemoryFreeClear(pRequest->pDb); @@ -243,14 +240,9 @@ void destroyRequest(SRequestObj *pRequest) { taosRemoveRef(clientReqRefPool, pRequest->self); } -SRequestObj *acquireRequest(int64_t rid) { - return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid); -} - -int32_t releaseRequest(int64_t rid) { - return taosReleaseRef(clientReqRefPool, rid); -} +SRequestObj *acquireRequest(int64_t rid) { return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid); } +int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, rid); } void taos_init_imp(void) { // In the APIs of other program language, taos_cleanup is not available yet. @@ -307,164 +299,50 @@ int taos_init() { } int taos_options_imp(TSDB_OPTION option, const char *str) { -#if 0 - SGlobalCfg *cfg = NULL; + if (option != TSDB_OPTION_CONFIGDIR) { + taos_init(); // initialize global config + } else { + tstrncpy(configDir, str, PATH_MAX); + tscInfo("set cfg:%s to %s", configDir, str); + return 0; + } + + SConfig *pCfg = taosGetCfg(); + SConfigItem *pItem = NULL; switch (option) { case TSDB_OPTION_CONFIGDIR: - cfg = taosGetConfigOption("configDir"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(configDir, str, TSDB_FILENAME_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscInfo("set config file directory:%s", str); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } + pItem = cfgGetItem(pCfg, "configDir"); break; - case TSDB_OPTION_SHELL_ACTIVITY_TIMER: - cfg = taosGetConfigOption("shellActivityTimer"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tsShellActivityTimer = atoi(str); - if (tsShellActivityTimer < 1) tsShellActivityTimer = 1; - if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600; - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, - tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); - } + pItem = cfgGetItem(pCfg, "shellActivityTimer"); break; - - case TSDB_OPTION_LOCALE: { // set locale - cfg = taosGetConfigOption("locale"); - assert(cfg != NULL); - - size_t len = strlen(str); - if (len == 0 || len > TD_LOCALE_LEN) { - tscInfo("Invalid locale:%s, use default", str); - return -1; - } - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - char sep = '.'; - - if (strlen(tsLocale) == 0) { // locale does not set yet - char *defaultLocale = setlocale(LC_CTYPE, ""); - - // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. - // The launch of current system will abort soon. - if (defaultLocale == NULL) { - tscError("failed to get default locale, please set the correct locale in current OS"); - return -1; - } - - tstrncpy(tsLocale, defaultLocale, TD_LOCALE_LEN); - } - - // set the user specified locale - char *locale = setlocale(LC_CTYPE, str); - - if (locale != NULL) { // failed to set the user specified locale - tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { // set the user specified locale failed, use default LC_CTYPE as current locale - locale = setlocale(LC_CTYPE, tsLocale); - tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); - } - - tstrncpy(tsLocale, locale, TD_LOCALE_LEN); - - char *charset = strrchr(tsLocale, sep); - if (charset != NULL) { - charset += 1; - - charset = taosCharsetReplace(charset); - - if (taosValidateEncodec(charset)) { - if (strlen(tsCharset) == 0) { - tscInfo("charset set:%s", charset); - } else { - tscInfo("charset changed from %s to %s", tsCharset, charset); - } - - tstrncpy(tsCharset, charset, TD_LOCALE_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - - } else { - tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset); - } - - taosMemoryFree(charset); - } else { // it may be windows system - tscInfo("charset remains:%s", tsCharset); - } - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } + case TSDB_OPTION_LOCALE: + pItem = cfgGetItem(pCfg, "locale"); break; - } - - case TSDB_OPTION_CHARSET: { - /* set charset will override the value of charset, assigned during system locale changed */ - cfg = taosGetConfigOption("charset"); - assert(cfg != NULL); - - size_t len = strlen(str); - if (len == 0 || len > TD_LOCALE_LEN) { - tscInfo("failed to set charset:%s", str); - return -1; - } - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - if (taosValidateEncodec(str)) { - if (strlen(tsCharset) == 0) { - tscInfo("charset is set:%s", str); - } else { - tscInfo("charset changed from %s to %s", tsCharset, str); - } - - tstrncpy(tsCharset, str, TD_LOCALE_LEN); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { - tscInfo("charset:%s not valid", str); - } - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } - + case TSDB_OPTION_CHARSET: + pItem = cfgGetItem(pCfg, "charset"); break; - } - case TSDB_OPTION_TIMEZONE: - cfg = taosGetConfigOption("timezone"); - assert(cfg != NULL); - - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(tsTimezoneStr, str, TD_TIMEZONE_LEN); - tsSetTimeZone(); - cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscDebug("timezone set:%s, input:%s by taos_options", tsTimezoneStr, str); - } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, - tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); - } + pItem = cfgGetItem(pCfg, "timezone"); break; - default: - // TODO return the correct error code to client in the format for taos_errstr() - tscError("Invalid option %d", option); - return -1; + break; } -#endif - return 0; + + if (pItem == NULL) { + tscError("Invalid option %d", option); + return -1; + } + + int code = cfgSetItem(pCfg, pItem->name, str, CFG_STYPE_TAOS_OPTIONS); + if (code != 0) { + tscError("failed to set cfg:%s to %s since %s", pItem->name, str, terrstr()); + } else { + tscInfo("set cfg:%s to %s", pItem->name, str); + } + + return code; } /** @@ -493,7 +371,7 @@ uint64_t generateRequestId() { } uint64_t id = 0; - + while (true) { int64_t ts = taosGetTimestampMs(); uint64_t pid = taosGetPId(); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index a6678b2ec0881e74f1fa4c18723285258b1bf6c8..6159da9cb161e6604cbdd4f504114bf4b831b1d3 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -630,10 +630,29 @@ void appHbMgrCleanup(void) { int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { SAppHbMgr *pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i); + + void *pIter = taosHashIterate(pTarget->activeInfo, NULL); + while (pIter != NULL) { + SClientHbReq *pOneReq = pIter; + hbFreeReq(pOneReq); + taosHashCleanup(pOneReq->info); + pIter = taosHashIterate(pTarget->activeInfo, pIter); + } taosHashCleanup(pTarget->activeInfo); pTarget->activeInfo = NULL; + + + pIter = taosHashIterate(pTarget->connInfo, NULL); + while (pIter != NULL) { + SHbConnInfo *info = pIter; + taosMemoryFree(info->param); + pIter = taosHashIterate(pTarget->connInfo, pIter); + } taosHashCleanup(pTarget->connInfo); pTarget->connInfo = NULL; + + taosMemoryFree(pTarget->key); + taosMemoryFree(pTarget); } } @@ -716,12 +735,23 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in } void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { - int32_t code = 0; - code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); - code = taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); - if (code) { + SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + if (pReq) { + hbFreeReq(pReq); + taosHashCleanup(pReq->info); + taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + } + + SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); + if (info) { + taosMemoryFree(info->param); + taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); + } + + if (NULL == pReq || NULL == info) { return; } + atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 74b8e711dcc31e9dba4d934c27686d675a9b7a2d..10edb38bf164e635b7cc2571447f786ad2000a33 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -146,7 +146,8 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj* (*pRequest)->sqlstr[sqlLen] = 0; (*pRequest)->sqlLen = sqlLen; - if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, sizeof((*pRequest)->self))) { + if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, + sizeof((*pRequest)->self))) { destroyRequest(*pRequest); *pRequest = NULL; tscError("put request to request hash failed"); @@ -157,7 +158,7 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj* return TSDB_CODE_SUCCESS; } -int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) { +int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb) { STscObj* pTscObj = pRequest->pTscObj; SParseContext cxt = { @@ -170,6 +171,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) { .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pTransporter = pTscObj->pAppInfo->pTransporter, + .pStmtCb = pStmtCb, }; cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); @@ -262,7 +264,8 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t } void setResPrecision(SReqResultInfo* pResInfo, int32_t precision) { - if (precision != TSDB_TIME_PRECISION_MILLI && precision != TSDB_TIME_PRECISION_MICRO && precision != TSDB_TIME_PRECISION_NANO) { + if (precision != TSDB_TIME_PRECISION_MILLI && precision != TSDB_TIME_PRECISION_MICRO && + precision != TSDB_TIME_PRECISION_NANO) { return; } @@ -274,7 +277,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, - pRequest->metric.start, &res); + pRequest->metric.start, &res); if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); @@ -298,15 +301,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList return pRequest->code; } -SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) { - SRequestObj* pRequest = NULL; - SQuery* pQuery = NULL; - SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); - - int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); - if (TSDB_CODE_SUCCESS == code) { - code = parseSql(pRequest, false, &pQuery); - } +SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) { + SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); if (TSDB_CODE_SUCCESS == code) { switch (pQuery->execMode) { @@ -331,7 +327,10 @@ SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) { } taosArrayDestroy(pNodeList); - qDestroyQuery(pQuery); + if (!keepQuery) { + qDestroyQuery(pQuery); + } + if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; } @@ -339,6 +338,18 @@ SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) { return pRequest; } +SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { + SRequestObj* pRequest = NULL; + SQuery* pQuery = NULL; + + int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); + if (TSDB_CODE_SUCCESS == code) { + code = parseSql(pRequest, false, &pQuery, NULL); + } + + return launchQueryImpl(pRequest, pQuery, code, false); +} + int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { SCatalog* pCatalog = NULL; int32_t code = 0; @@ -383,7 +394,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { int32_t code = 0; while (retryNum++ < REQUEST_MAX_TRY_TIMES) { - pRequest = execQueryImpl(pTscObj, sql, sqlLen); + pRequest = launchQuery(pTscObj, sql, sqlLen); if (TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { break; } @@ -512,6 +523,8 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest, int8_t connType) { connectReq.pid = htonl(appInfo.pid); connectReq.startTime = htobe64(appInfo.startTime); tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app)); + tstrncpy(connectReq.user, pObj->user, sizeof(connectReq.user)); + tstrncpy(connectReq.passwd, pObj->pass, sizeof(connectReq.passwd)); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); void* pReq = taosMemoryMalloc(contLen); @@ -715,6 +728,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p)); ASSERT(len <= bytes); + ASSERT((p + len) < (pResultInfo->convertBuf[i] + colLength[i])); varDataSetLen(p, len); pCol->offset[j] = (p - pResultInfo->convertBuf[i]); @@ -725,6 +739,80 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; pResultInfo->row[i] = pResultInfo->pCol[i].pData; } + + if (type == TSDB_DATA_TYPE_JSON) { + char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pResultInfo->convertBuf[i] = p; + int32_t len = 0; + SResultColumn* pCol = &pResultInfo->pCol[i]; + for (int32_t j = 0; j < numOfRows; ++j) { + if (pCol->offset[j] != -1) { + char* pStart = pCol->offset[j] + pCol->pData; + + int32_t jsonInnerType = *pStart; + char* jsonInnerData = pStart + CHAR_BYTES; + char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; + if (jsonInnerType == TSDB_DATA_TYPE_NULL) { + sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); + varDataSetLen(dst, strlen(varDataVal(dst))); + } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) { + int32_t length = + taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst)); + + if (length <= 0) { + tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, + varDataVal(jsonInnerData)); + length = 0; + } + varDataSetLen(dst, length); + } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" + *(char*)varDataVal(dst) = '\"'; + int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), + varDataVal(dst) + CHAR_BYTES); + if (length <= 0) { + tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, + varDataVal(jsonInnerData)); + length = 0; + } + varDataSetLen(dst, length + CHAR_BYTES * 2); + *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"'; + } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) { + double jsonVd = *(double*)(jsonInnerData); + sprintf(varDataVal(dst), "%.9lf", jsonVd); + varDataSetLen(dst, strlen(varDataVal(dst))); + } else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) { + int64_t jsonVd = *(int64_t*)(jsonInnerData); + sprintf(varDataVal(dst), "%" PRId64, jsonVd); + varDataSetLen(dst, strlen(varDataVal(dst))); + } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { + sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); + varDataSetLen(dst, strlen(varDataVal(dst))); + } else { + ASSERT(0); + } + + if (len + varDataTLen(dst) > colLength[i]) { + p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pResultInfo->convertBuf[i] = p; + } + p = pResultInfo->convertBuf[i] + len; + memcpy(p, dst, varDataTLen(dst)); + pCol->offset[j] = len; + len += varDataTLen(dst); + } + } + + pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; + pResultInfo->row[i] = pResultInfo->pCol[i].pData; + } } return TSDB_CODE_SUCCESS; @@ -815,3 +903,68 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, convertUcs4); } + +TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* details, int maxlen) { + TSDB_SERVER_STATUS code = TSDB_SRV_STATUS_UNAVAILABLE; + void* clientRpc = NULL; + SServerStatusRsp statusRsp = {0}; + SEpSet epSet = {.inUse = 0, .numOfEps = 1}; + SRpcMsg rpcMsg = {.ahandle = (void*)0x9526, .msgType = TDMT_DND_SERVER_STATUS}; + SRpcMsg rpcRsp = {0}; + SRpcInit rpcInit = {0}; + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + + taosEncryptPass_c((uint8_t*)("_pwd"), strlen("_pwd"), pass); + rpcInit.label = "CHK"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = NULL; + rpcInit.sessions = 16; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.user = "_dnd"; + rpcInit.ckey = "_key"; + rpcInit.spi = 1; + rpcInit.secret = pass; + + clientRpc = rpcOpen(&rpcInit); + if (clientRpc == NULL) { + tscError("failed to init server status client"); + goto _OVER; + } + + if (fqdn == NULL) { + fqdn = tsLocalFqdn; + } + + if (port == 0) { + port = tsServerPort; + } + + tstrncpy(epSet.eps[0].fqdn, fqdn, TSDB_FQDN_LEN); + epSet.eps[0].port = (uint16_t)port; + rpcSendRecv(clientRpc, &epSet, &rpcMsg, &rpcRsp); + + if (rpcRsp.code != 0 || rpcRsp.contLen <= 0 || rpcRsp.pCont == NULL) { + tscError("failed to send server status req since %s", terrstr()); + goto _OVER; + } + + if (tDeserializeSServerStatusRsp(rpcRsp.pCont, rpcRsp.contLen, &statusRsp) != 0) { + tscError("failed to parse server status rsp since %s", terrstr()); + goto _OVER; + } + + code = statusRsp.statusCode; + if (details != NULL && statusRsp.details != NULL) { + tstrncpy(details, statusRsp.details, maxlen); + } + +_OVER: + if (clientRpc != NULL) { + rpcClose(clientRpc); + } + if (rpcRsp.pCont != NULL) { + rpcFreeCont(rpcRsp.pCont); + } + return code; +} diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 903018d5c3b9f93ec9097c09443bb454b50b2b2c..27efcee76e5af71ed7f5c7b516b00eb262cdaa8d 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -14,12 +14,12 @@ */ #include "catalog.h" -#include "scheduler.h" #include "clientInt.h" -#include "clientStmt.h" #include "clientLog.h" +#include "clientStmt.h" #include "os.h" #include "query.h" +#include "scheduler.h" #include "tglobal.h" #include "tmsg.h" #include "tref.h" @@ -128,9 +128,23 @@ const char *taos_errstr(TAOS_RES *res) { } void taos_free_result(TAOS_RES *res) { + if (NULL == res) { + return; + } + if (TD_RES_QUERY(res)) { SRequestObj *pRequest = (SRequestObj *)res; destroyRequest(pRequest); + } else if (TD_RES_TMQ(res)) { + SMqRspObj *pRsp = (SMqRspObj *)res; + if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree); + if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen); + if (pRsp->rsp.blockSchema) taosArrayDestroy(pRsp->rsp.blockSchema); + if (pRsp->rsp.blockTbName) taosArrayDestroy(pRsp->rsp.blockTbName); + if (pRsp->rsp.blockTags) taosArrayDestroy(pRsp->rsp.blockTags); + if (pRsp->rsp.blockTagSchema) taosArrayDestroy(pRsp->rsp.blockTagSchema); + pRsp->resInfo.pRspMsg = NULL; + doFreeReqResultInfo(&pRsp->resInfo); } } @@ -177,25 +191,24 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { return doFetchRows(pRequest, true, true); } else if (TD_RES_TMQ(res)) { - SMqRspObj *msg = ((SMqRspObj *)res); - if (msg->resIter == -1) msg->resIter++; - SReqResultInfo *pResultInfo = taosArrayGet(msg->res, msg->resIter); + SMqRspObj *msg = ((SMqRspObj *)res); + SReqResultInfo *pResultInfo; + if (msg->resIter == -1) { + pResultInfo = tmqGetNextResInfo(res, true); + } else { + pResultInfo = tmqGetCurResInfo(res); + } if (pResultInfo->current < pResultInfo->numOfRows) { doSetOneRowPtr(pResultInfo); pResultInfo->current += 1; return pResultInfo->row; } else { - msg->resIter++; - if (msg->resIter < taosArrayGetSize(msg->res)) { - pResultInfo = taosArrayGet(msg->res, msg->resIter); - doSetOneRowPtr(pResultInfo); - pResultInfo->current += 1; - return pResultInfo->row; - } else { - return NULL; - } + pResultInfo = tmqGetNextResInfo(res, true); + if (pResultInfo == NULL) return NULL; + doSetOneRowPtr(pResultInfo); + pResultInfo->current += 1; + return pResultInfo->row; } - } else { // assert to avoid un-initialization error ASSERT(0); @@ -455,7 +468,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { (*numOfRows) = pResultInfo->numOfRows; return pRequest->code; } else if (TD_RES_TMQ(res)) { - SReqResultInfo *pResultInfo = tmqGetNextResInfo(res); + SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, true); if (pResultInfo == NULL) return -1; pResultInfo->current = pResultInfo->numOfRows; @@ -474,7 +487,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { } if (TD_RES_TMQ(res)) { - SReqResultInfo *pResultInfo = tmqGetNextResInfo(res); + SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, false); if (pResultInfo == NULL) { (*numOfRows) = 0; return 0; @@ -580,84 +593,115 @@ TAOS_STMT *taos_stmt_init(TAOS *taos) { return stmtInit(taos); } -int taos_stmt_close(TAOS_STMT *stmt) { - if (stmt == NULL) { +int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { + if (stmt == NULL || sql == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; } - return stmtClose(stmt); + return stmtPrepare(stmt, sql, length); } -int taos_stmt_execute(TAOS_STMT *stmt) { - if (stmt == NULL) { +int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags) { + if (stmt == NULL || name == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; } - return stmtExec(stmt); + int32_t code = stmtSetTbName(stmt, name); + if (code) { + return code; + } + + if (tags) { + return stmtSetTbTags(stmt, tags); + } + + return TSDB_CODE_SUCCESS; } -char *taos_stmt_errstr(TAOS_STMT *stmt) { - if (stmt == NULL) { +int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) { + if (stmt == NULL || name == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; - return NULL; + return terrno; } - return stmtErrstr(stmt); + return stmtSetTbName(stmt, name); } -int taos_stmt_affected_rows(TAOS_STMT *stmt) { - if (stmt == NULL) { +int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) { + return taos_stmt_set_tbname(stmt, name); +} + +int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { + if (stmt == NULL || bind == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; - return 0; + return terrno; } - return stmtAffectedRows(stmt); + if (bind->num > 1) { + tscError("invalid bind number %d for %s", bind->num, __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return stmtBindBatch(stmt, bind, -1); } -int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind) { +int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { if (stmt == NULL || bind == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; } - return stmtBind(stmt, bind); + if (bind->num <= 0 || bind->num > INT16_MAX) { + tscError("invalid bind num %d", bind->num); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return stmtBindBatch(stmt, bind, -1); } -int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { - if (stmt == NULL || sql == NULL) { +int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx) { + if (stmt == NULL || bind == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; } - return stmtPrepare(stmt, sql, length); + if (colIdx < 0) { + tscError("invalid bind column idx %d", colIdx); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return stmtBindBatch(stmt, bind, colIdx); } -int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags) { - if (stmt == NULL || name == NULL || tags == NULL) { +int taos_stmt_add_batch(TAOS_STMT *stmt) { + if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; } - return stmtSetTbNameTags(stmt, name, tags); + return stmtAddBatch(stmt); } -int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) { - if (stmt == NULL || name == NULL) { +int taos_stmt_execute(TAOS_STMT *stmt) { + if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; } - return stmtSetTbNameTags(stmt, name, NULL); + return stmtExec(stmt); } int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert) { @@ -680,40 +724,42 @@ int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) { return stmtGetParamNum(stmt, nums); } -int taos_stmt_add_batch(TAOS_STMT *stmt) { +TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) { if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; - return terrno; + return NULL; } - return stmtAddBatch(stmt); + return stmtUseResult(stmt); } -TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) { +char *taos_stmt_errstr(TAOS_STMT *stmt) { + return (char *)stmtErrstr(stmt); +} + +int taos_stmt_affected_rows(TAOS_STMT *stmt) { if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; - return NULL; + return 0; } - return stmtUseResult(stmt); + return stmtAffectedRows(stmt); } -int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { - if (stmt == NULL || bind == NULL) { +int taos_stmt_close(TAOS_STMT *stmt) { + if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; } - return stmtBindBatch(stmt, bind); + return stmtClose(stmt); } - TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { // TODO return NULL; } - diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 74347cabf7bd66170485fa370e313cb421e07024..06b4c7b8e0ee8a7329d470cfca511161d4a4d7d4 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -209,9 +209,9 @@ int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { } void initMsgHandleFp() { - handleRequestRspFp[TMSG_INDEX(TDMT_MND_CONNECT)] = processConnectRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = processCreateDbRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_MND_USE_DB)] = processUseDbRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_MND_CONNECT)] = processConnectRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = processCreateDbRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_MND_USE_DB)] = processUseDbRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = processCreateTableRsp; - handleRequestRspFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = processDropDbRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = processDropDbRsp; } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 8c4cff92514cc8c850c9a25458865c1741cf2045..0972ff347767824355766e72a109e27dd122cbfd 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -4,86 +4,540 @@ #include "clientStmt.h" #include "tdef.h" +int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { + switch (newStatus) { + case STMT_SETTBNAME: + break; + default: + break; + } + + //STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); + + pStmt->sql.status = newStatus; + + return TSDB_CODE_SUCCESS; +} + + +int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) { + STscStmt* pStmt = (STscStmt*)stmt; + + pStmt->sql.type = STMT_TYPE_MULTI_INSERT; + + if (NULL == pStmt->bInfo.tbName) { + tscError("no table name set"); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_TBNAME_ERROR); + } + + *tbName = pStmt->bInfo.tbName; + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) { + STscStmt* pStmt = (STscStmt*)stmt; + + pStmt->bInfo.tbUid = pTableMeta->uid; + pStmt->bInfo.tbSuid = pTableMeta->suid; + pStmt->bInfo.tbType = pTableMeta->tableType; + pStmt->bInfo.boundTags = tags; + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtSetExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash) { + STscStmt* pStmt = (STscStmt*)stmt; + + pStmt->exec.pVgHash = pVgHash; + pStmt->exec.pBlockHash = pBlockHash; + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHash) { + STscStmt* pStmt = (STscStmt*)stmt; + + *pVgHash = pStmt->exec.pVgHash; + *pBlockHash = pStmt->exec.pBlockHash; + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtCacheBlock(STscStmt *pStmt) { + if (pStmt->sql.type != STMT_TYPE_MULTI_INSERT) { + return TSDB_CODE_SUCCESS; + } + + uint64_t uid; + if (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) { + uid = pStmt->bInfo.tbSuid; + } else { + ASSERT(TSDB_NORMAL_TABLE == pStmt->bInfo.tbType); + uid = pStmt->bInfo.tbUid; + } + + if (taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid))) { + return TSDB_CODE_SUCCESS; + } + + STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, &uid, sizeof(uid)); + STableDataBlocks* pDst = NULL; + + STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc)); + + SStmtTableCache cache = { + .pDataBlock = pDst, + .boundTags = pStmt->bInfo.boundTags, + }; + + if (taosHashPut(pStmt->sql.pTableCache, &uid, sizeof(uid), &cache, sizeof(cache))) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pStmt->bInfo.boundTags = NULL; + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtParseSql(STscStmt* pStmt) { + SStmtCallback stmtCb = { + .pStmt = pStmt, + .getTbNameFn = stmtGetTbName, + .setBindInfoFn = stmtSetBindInfo, + .setExecInfoFn = stmtSetExecInfo, + .getExecInfoFn = stmtGetExecInfo, + }; + + if (NULL == pStmt->exec.pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); + } + + STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb)); + + pStmt->bInfo.needParse = false; + + switch (nodeType(pStmt->sql.pQuery->pRoot)) { + case QUERY_NODE_VNODE_MODIF_STMT: + if (0 == pStmt->sql.type) { + pStmt->sql.type = STMT_TYPE_INSERT; + } + break; + case QUERY_NODE_SELECT_STMT: + pStmt->sql.type = STMT_TYPE_QUERY; + break; + default: + tscError("not supported stmt type %d", nodeType(pStmt->sql.pQuery->pRoot)); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_CLAUSE_ERROR); + } + + STMT_ERR_RET(stmtCacheBlock(pStmt)); + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtCleanBindInfo(STscStmt* pStmt) { + pStmt->bInfo.tbUid = 0; + pStmt->bInfo.tbSuid = 0; + pStmt->bInfo.tbType = 0; + pStmt->bInfo.needParse = true; + + taosMemoryFreeClear(pStmt->bInfo.tbName); + destroyBoundColumnInfo(pStmt->bInfo.boundTags); + taosMemoryFreeClear(pStmt->bInfo.boundTags); + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) { + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; + + void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); + while (pIter) { + STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter; + uint64_t *key = taosHashGetKey(pIter, NULL); + + if (keepTable && (*key == pStmt->bInfo.tbUid)) { + STMT_ERR_RET(qResetStmtDataBlock(pBlocks, true)); + + pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); + continue; + } + + qFreeStmtDataBlock(pBlocks); + taosHashRemove(pStmt->exec.pBlockHash, key, sizeof(*key)); + + pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); + } + + if (keepTable) { + return TSDB_CODE_SUCCESS; + } + + taosHashCleanup(pStmt->exec.pBlockHash); + pStmt->exec.pBlockHash = NULL; + + STMT_ERR_RET(stmtCleanBindInfo(pStmt)); + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtCleanSQLInfo(STscStmt* pStmt) { + taosMemoryFree(pStmt->sql.sqlStr); + qDestroyQuery(pStmt->sql.pQuery); + + void *pIter = taosHashIterate(pStmt->sql.pTableCache, NULL); + while (pIter) { + SStmtTableCache* pCache = (SStmtTableCache*)pIter; + + qDestroyStmtDataBlock(pCache->pDataBlock); + destroyBoundColumnInfo(pCache->boundTags); + + pIter = taosHashIterate(pStmt->sql.pTableCache, pIter); + } + taosHashCleanup(pStmt->sql.pTableCache); + pStmt->sql.pTableCache = NULL; + + memset(&pStmt->sql, 0, sizeof(pStmt->sql)); + + STMT_ERR_RET(stmtCleanExecInfo(pStmt, false)); + STMT_ERR_RET(stmtCleanBindInfo(pStmt)); + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtGetFromCache(STscStmt* pStmt) { + pStmt->bInfo.needParse = true; + + if (NULL == pStmt->sql.pTableCache || taosHashGetSize(pStmt->sql.pTableCache) <= 0) { + return TSDB_CODE_SUCCESS; + } + + if (NULL == pStmt->pCatalog) { + STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &pStmt->pCatalog)); + } + + STableMeta *pTableMeta = NULL; + SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); + STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta)); + + if (pTableMeta->uid == pStmt->bInfo.tbUid) { + pStmt->bInfo.needParse = false; + + return TSDB_CODE_SUCCESS; + } + + if (taosHashGet(pStmt->exec.pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid))) { + SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid)); + if (NULL == pCache) { + tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", pTableMeta->uid); + STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + } + + pStmt->bInfo.needParse = false; + + pStmt->bInfo.tbUid = pTableMeta->uid; + pStmt->bInfo.tbSuid = pTableMeta->suid; + pStmt->bInfo.tbType = pTableMeta->tableType; + pStmt->bInfo.boundTags = pCache->boundTags; + + return TSDB_CODE_SUCCESS; + } + + SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid)); + if (pCache) { + pStmt->bInfo.needParse = false; + + pStmt->bInfo.tbUid = pTableMeta->uid; + pStmt->bInfo.tbSuid = pTableMeta->suid; + pStmt->bInfo.tbType = pTableMeta->tableType; + pStmt->bInfo.boundTags = pCache->boundTags; + + STableDataBlocks* pNewBlock = NULL; + STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock)); + + if (taosHashPut(pStmt->exec.pBlockHash, &pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid), &pNewBlock, POINTER_BYTES)) { + STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; + } + + STMT_ERR_RET(stmtCleanBindInfo(pStmt)); + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtResetStmt(STscStmt* pStmt) { + STMT_ERR_RET(stmtCleanSQLInfo(pStmt)); + + pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (NULL == pStmt->sql.pTableCache) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + STMT_ERR_RET(terrno); + } + + pStmt->sql.status = STMT_INIT; + + return TSDB_CODE_SUCCESS; +} + + TAOS_STMT *stmtInit(TAOS *taos) { STscObj* pObj = (STscObj*)taos; STscStmt* pStmt = NULL; -#if 0 pStmt = taosMemoryCalloc(1, sizeof(STscStmt)); - if (pStmt == NULL) { + if (NULL == pStmt) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("failed to allocate memory for statement"); return NULL; } - pStmt->taos = pObj; - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - free(pStmt); + pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (NULL == pStmt->sql.pTableCache) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("failed to allocate memory for statement"); + taosMemoryFree(pStmt); return NULL; } - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { - free(pSql); - free(pStmt); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("failed to malloc payload buffer"); - return NULL; + pStmt->taos = pObj; + pStmt->bInfo.needParse = true; + pStmt->sql.status = STMT_INIT; + + return pStmt; +} + +int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { + STscStmt* pStmt = (STscStmt*)stmt; + + if (pStmt->sql.status >= STMT_PREPARE) { + STMT_ERR_RET(stmtResetStmt(pStmt)); } - tsem_init(&pSql->rspSem, 0, 0); - pSql->signature = pSql; - pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_MAX_REPLICA; - pStmt->pSql = pSql; - pStmt->last = STMT_INIT; - pStmt->numOfRows = 0; - registerSqlObj(pSql); -#endif + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_PREPARE)); - return pStmt; -} + if (length <= 0) { + length = strlen(sql); + } + + pStmt->sql.sqlStr = strndup(sql, length); + pStmt->sql.sqlLen = length; -int stmtClose(TAOS_STMT *stmt) { return TSDB_CODE_SUCCESS; } -int stmtExec(TAOS_STMT *stmt) { + +int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME)); + + if (NULL == pStmt->exec.pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); + } + + STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + + STMT_ERR_RET(stmtGetFromCache(pStmt)); + + if (pStmt->bInfo.needParse) { + taosMemoryFree(pStmt->bInfo.tbName); + pStmt->bInfo.tbName = strdup(tbName); + } + return TSDB_CODE_SUCCESS; } -char *stmtErrstr(TAOS_STMT *stmt) { - return NULL; -} +int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); + + if (pStmt->bInfo.needParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); + } + + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + if (NULL == pDataBlock) { + tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); + STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, &pStmt->bInfo.sname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); -int stmtAffectedRows(TAOS_STMT *stmt) { return TSDB_CODE_SUCCESS; } -int stmtBind(TAOS_STMT *stmt, TAOS_BIND *bind) { + +int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_TAG_FIELDS)); + + if (pStmt->bInfo.needParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); + } + + if (STMT_TYPE_QUERY == pStmt->sql.type) { + tscError("invalid operation to get query tag fileds"); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); + } + + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + if (NULL == pDataBlock) { + tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); + STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + STMT_ERR_RET(qBuildStmtTagFields(*pDataBlock, pStmt->bInfo.boundTags, fieldNum, fields)); + return TSDB_CODE_SUCCESS; } -int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { - return TSDB_CODE_SUCCESS; +int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_COL_FIELDS)); + + if (pStmt->bInfo.needParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); + } + + if (STMT_TYPE_QUERY == pStmt->sql.type) { + tscError("invalid operation to get query column fileds"); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); + } + + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + if (NULL == pDataBlock) { + tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); + STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + STMT_ERR_RET(qBuildStmtColFields(*pDataBlock, fieldNum, fields)); + + return TSDB_CODE_SUCCESS; } -int stmtSetTbNameTags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags) { +int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); + + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + pStmt->bInfo.needParse = false; + } + + if (NULL == pStmt->exec.pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); + } + + if (pStmt->bInfo.needParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); + } + + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + if (NULL == pDataBlock) { + tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); + STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + if (colIdx < 0) { + qBindStmtColsValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); + } else { + if (colIdx != (pStmt->bInfo.sBindLastIdx + 1) && colIdx != 0) { + tscError("bind column index not in sequence"); + STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + pStmt->bInfo.sBindLastIdx = colIdx; + + if (0 == colIdx) { + pStmt->bInfo.sBindRowNum = bind->num; + } + + qBindStmtSingleColValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum); + } + return TSDB_CODE_SUCCESS; } -int stmtIsInsert(TAOS_STMT *stmt, int *insert) { + +int stmtAddBatch(TAOS_STMT *stmt) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_ADD_BATCH)); + + STMT_ERR_RET(stmtCacheBlock(pStmt)); + return TSDB_CODE_SUCCESS; } -int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { +int stmtExec(TAOS_STMT *stmt) { + STscStmt* pStmt = (STscStmt*)stmt; + int32_t code = 0; + + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); + + STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); + + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true); + + STMT_ERR_JRET(pStmt->exec.pRequest->code); + + pStmt->exec.affectedRows = taos_affected_rows(pStmt->exec.pRequest); + pStmt->affectedRows += pStmt->exec.affectedRows; + +_return: + + stmtCleanExecInfo(pStmt, (code ? false : true)); + + ++pStmt->sql.runTimes; + + STMT_RET(code); +} + + +int stmtClose(TAOS_STMT *stmt) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_RET(stmtCleanSQLInfo(pStmt)); +} + +const char *stmtErrstr(TAOS_STMT *stmt) { + STscStmt* pStmt = (STscStmt*)stmt; + + if (stmt == NULL) { + return (char*) tstrerror(terrno); + } + + if (pStmt->exec.pRequest) { + pStmt->exec.pRequest->code = terrno; + } + + return taos_errstr(pStmt->exec.pRequest); +} + +int stmtAffectedRows(TAOS_STMT *stmt) { + return ((STscStmt*)stmt)->affectedRows; +} + +int stmtIsInsert(TAOS_STMT *stmt, int *insert) { + STscStmt* pStmt = (STscStmt*)stmt; + + if (pStmt->sql.type) { + *insert = (STMT_TYPE_INSERT == pStmt->sql.type || STMT_TYPE_MULTI_INSERT == pStmt->sql.type); + } else { + *insert = isInsertSql(pStmt->sql.sqlStr, 0); + } + return TSDB_CODE_SUCCESS; } -int stmtAddBatch(TAOS_STMT *stmt) { +int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { + STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL)); + return TSDB_CODE_SUCCESS; } @@ -91,9 +545,5 @@ TAOS_RES *stmtUseResult(TAOS_STMT *stmt) { return NULL; } -int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { - return TSDB_CODE_SUCCESS; -} - diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 478e328a165e04903544590bdb48b4303b5cef74..3a1ba4cebc071bb52a023f67eeca26d6c8772b3a 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -24,14 +24,6 @@ #include "tqueue.h" #include "tref.h" -struct tmq_message_t { - SMqPollRsp msg; - char* topic; - SArray* res; // SArray - int32_t vgId; - int32_t resIter; -}; - typedef struct { int8_t tmqRspType; int32_t epoch; @@ -52,9 +44,7 @@ struct tmq_topic_vgroup_t { }; struct tmq_topic_vgroup_list_t { - int32_t cnt; - int32_t size; - tmq_topic_vgroup_t* elems; + SArray container; // SArray }; struct tmq_conf_t { @@ -63,6 +53,7 @@ struct tmq_conf_t { int8_t autoCommit; int8_t resetOffset; uint16_t port; + uint16_t autoCommitInterval; char* ip; char* user; char* pass; @@ -72,25 +63,25 @@ struct tmq_conf_t { struct tmq_t { // conf - char groupId[TSDB_CGROUP_LEN]; - char clientId[256]; - int8_t autoCommit; - int8_t inWaiting; + char groupId[TSDB_CGROUP_LEN]; + char clientId[256]; + int8_t autoCommit; + /*int8_t inWaiting;*/ int64_t consumerId; int32_t epoch; int32_t resetOffsetCfg; int64_t status; STscObj* pTscObj; tmq_commit_cb* commit_cb; - int32_t nextTopicIdx; - int8_t epStatus; - int32_t epSkipCnt; - int32_t waitingRequest; - int32_t readyRequest; - SArray* clientTopics; // SArray - STaosQueue* mqueue; // queue of tmq_message_t - STaosQall* qall; - tsem_t rspSem; + /*int32_t nextTopicIdx;*/ + int8_t epStatus; + int32_t epSkipCnt; + /*int32_t waitingRequest;*/ + /*int32_t readyRequest;*/ + SArray* clientTopics; // SArray + STaosQueue* mqueue; // queue of tmq_message_t + STaosQall* qall; + tsem_t rspSem; // stat int64_t pollCnt; }; @@ -134,7 +125,7 @@ typedef struct { int32_t epoch; SMqClientVg* vgHandle; SMqClientTopic* topicHandle; - SMqPollRspV2 msg; + SMqDataBlkRsp msg; } SMqPollRspWrapper; typedef struct { @@ -145,6 +136,7 @@ typedef struct { typedef struct { tmq_t* tmq; + int32_t code; int32_t sync; tsem_t rspSem; } SMqAskEpCbParam; @@ -201,6 +193,11 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } + if (strcmp(key, "auto.commit.interval.ms") == 0) { + conf->autoCommitInterval = atoi(value); + return TMQ_CONF_OK; + } + if (strcmp(key, "auto.offset.reset") == 0) { if (strcmp(value, "none") == 0) { conf->resetOffset = TMQ_CONF__RESET_OFFSET__NONE; @@ -255,7 +252,12 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { void tmq_list_destroy(tmq_list_t* list) { SArray* container = &list->container; /*taosArrayDestroy(container);*/ - taosArrayDestroyEx(container, (void (*)(void*))taosMemoryFree); + int32_t sz = taosArrayGetSize(container); + for (int32_t i = 0; i < sz; i++) { + char* str = taosArrayGetP(container, i); + taosMemoryFree(str); + } + taosArrayDestroy(container); } static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { @@ -294,7 +296,7 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; if (pParam->tmq->commit_cb) { - pParam->tmq->commit_cb(pParam->tmq, pParam->rspErr, NULL, NULL); + pParam->tmq->commit_cb(pParam->tmq, pParam->rspErr, NULL); } if (!pParam->async) tsem_post(&pParam->rspSem); return 0; @@ -316,18 +318,19 @@ tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq) { return tmq_subscribe(tmq, lst); } +#if 0 tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = taosMemoryCalloc(sizeof(tmq_t), 1); if (pTmq == NULL) { return NULL; } pTmq->pTscObj = (STscObj*)conn; - pTmq->inWaiting = 0; + /*pTmq->inWaiting = 0;*/ pTmq->status = 0; pTmq->pollCnt = 0; pTmq->epoch = 0; - pTmq->waitingRequest = 0; - pTmq->readyRequest = 0; + /*pTmq->waitingRequest = 0;*/ + /*pTmq->readyRequest = 0;*/ pTmq->epStatus = 0; pTmq->epSkipCnt = 0; // set conf @@ -351,8 +354,9 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs return pTmq; } +#endif -tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { +tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); if (pTmq == NULL) { return NULL; @@ -363,16 +367,17 @@ tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { ASSERT(user); ASSERT(pass); ASSERT(conf->db); + ASSERT(conf->groupId[0]); pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, conf->db, conf->port, CONN_TYPE__TMQ); if (pTmq->pTscObj == NULL) return NULL; - pTmq->inWaiting = 0; + /*pTmq->inWaiting = 0;*/ pTmq->status = 0; pTmq->pollCnt = 0; pTmq->epoch = 0; - pTmq->waitingRequest = 0; - pTmq->readyRequest = 0; + /*pTmq->waitingRequest = 0;*/ + /*pTmq->readyRequest = 0;*/ pTmq->epStatus = 0; pTmq->epSkipCnt = 0; // set conf @@ -423,8 +428,8 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in req.num = pArray->size; req.offsets = pArray->pData; } else { - req.num = offsets->cnt; - req.offsets = (SMqOffset*)offsets->elems; + req.num = taosArrayGetSize(&offsets->container); + req.offsets = (SMqOffset*)offsets->container.pData; } SCoder encoder; @@ -496,7 +501,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { SCMSubscribeReq req; req.topicNum = sz; req.consumerId = tmq->consumerId; - req.consumerGroup = strdup(tmq->groupId); + strcpy(req.cgroup, tmq->groupId); req.topicNames = taosArrayInit(sz, sizeof(void*)); for (int i = 0; i < sz; i++) { @@ -609,7 +614,7 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa int32_t code = 0; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); - CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode), _return); + CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode, NULL), _return); // todo check for invalid sql statement and return with error code @@ -755,109 +760,15 @@ _return: } #endif -static char* formatTimestamp(char* buf, int64_t val, int precision) { - time_t tt; - int32_t ms = 0; - if (precision == TSDB_TIME_PRECISION_NANO) { - tt = (time_t)(val / 1000000000); - ms = val % 1000000000; - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - tt = (time_t)(val / 1000000); - ms = val % 1000000; - } else { - tt = (time_t)(val / 1000); - ms = val % 1000; - } - - /* comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif - if (tt <= 0 && ms < 0) { - tt--; - if (precision == TSDB_TIME_PRECISION_NANO) { - ms += 1000000000; - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - ms += 1000000; - } else { - ms += 1000; - } - } - - struct tm* ptm = taosLocalTime(&tt, NULL); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); - - if (precision == TSDB_TIME_PRECISION_NANO) { - sprintf(buf + pos, ".%09d", ms); - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - sprintf(buf + pos, ".%06d", ms); - } else { - sprintf(buf + pos, ".%03d", ms); - } - - return buf; -} #if 0 int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { if (tmq_message == NULL) return 0; SMqPollRsp* pRsp = &tmq_message->msg; return pRsp->skipLogNum; } - -void tmqShowMsg(tmq_message_t* tmq_message) { - if (tmq_message == NULL) return; - - static bool noPrintSchema; - char pBuf[128]; - SMqPollRsp* pRsp = &tmq_message->msg; - int32_t colNum = 2; - if (!noPrintSchema) { - printf("|"); - for (int32_t i = 0; i < colNum; i++) { - if (i == 0) - printf(" %25s |", pRsp->schema->pSchema[i].name); - else - printf(" %15s |", pRsp->schema->pSchema[i].name); - } - printf("\n"); - printf("===============================================\n"); - noPrintSchema = true; - } - int32_t sz = taosArrayGetSize(pRsp->pBlockData); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pRsp->pBlockData, i); - int32_t rows = pDataBlock->info.rows; - for (int32_t j = 0; j < rows; j++) { - printf("|"); - for (int32_t k = 0; k < colNum; k++) { - SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); - printf(" %25s |", pBuf); - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - printf(" %15u |", *(uint32_t*)var); - break; - } - } - printf("\n"); - } - } -} #endif int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { - /*printf("recv poll\n");*/ SMqPollCbParam* pParam = (SMqPollCbParam*)param; SMqClientVg* pVg = pParam->pVg; SMqClientTopic* pTopic = pParam->pTopic; @@ -870,17 +781,15 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch; int32_t tmqEpoch = atomic_load_32(&tmq->epoch); if (msgEpoch < tmqEpoch) { - /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/ - /*tsem_post(&tmq->rspSem);*/ + // do not write into queue since updating epoch reset tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); + /*tsem_post(&tmq->rspSem);*/ return 0; } if (msgEpoch != tmqEpoch) { tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); - } else { - atomic_sub_fetch_32(&tmq->waitingRequest, 1); } #if 0 @@ -902,45 +811,33 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } #endif - /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/ - /*tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));*/ SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper)); if (pRspWrapper == NULL) { tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch); goto CREATE_MSG_FAIL; } + pRspWrapper->tmqRspType = TMQ_MSG_TYPE__POLL_RSP; pRspWrapper->vgHandle = pVg; pRspWrapper->topicHandle = pTopic; - /*memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));*/ + memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqPollRspV2(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg); - // TODO: alloc mem - /*pRsp->*/ - /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ -#if 0 - if (pRsp->msg.numOfTopics == 0) { - /*printf("no data\n");*/ - taosFreeQitem(pRsp); - goto CREATE_MSG_FAIL; - } -#endif + tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg); tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId, pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset); taosWriteQitem(tmq->mqueue, pRspWrapper); - atomic_add_fetch_32(&tmq->readyRequest, 1); /*tsem_post(&tmq->rspSem);*/ - return 0; + return 0; CREATE_MSG_FAIL: if (pParam->epoch == tmq->epoch) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } /*tsem_post(&tmq->rspSem);*/ - return code; + return -1; } bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { @@ -1023,6 +920,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = pParam->tmq; + pParam->code = code; if (code != 0) { tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->sync); goto END; @@ -1048,7 +946,6 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { } tDeleteSMqCMGetSubEpRsp(&rsp); } else { - /*SMqCMGetSubEpRsp* pRsp = taosAllocateQitem(sizeof(SMqCMGetSubEpRsp));*/ SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper)); if (pWrapper == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1062,6 +959,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { taosWriteQitem(tmq->mqueue, pWrapper); /*tsem_post(&tmq->rspSem);*/ + taosMemoryFree(pParam); } END: @@ -1073,7 +971,8 @@ END: } int32_t tmqAskEp(tmq_t* tmq, bool sync) { - int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1); + int32_t code = 0; + int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1); if (epStatus == 1) { int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1); tscTrace("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt); @@ -1130,8 +1029,12 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) { int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - if (sync) tsem_wait(&pParam->rspSem); - return 0; + if (sync) { + tsem_wait(&pParam->rspSem); + code = pParam->code; + taosMemoryFree(pParam); + } + return code; } tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { @@ -1157,7 +1060,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { return TMQ_RESP_ERR__FAIL; } -SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) { +SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) { int64_t reqOffset; if (pVg->currentOffset >= 0) { reqOffset = pVg->currentOffset; @@ -1169,13 +1072,18 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo reqOffset = tmq->resetOffsetCfg; } - SMqPollReq* pReq = taosMemoryMalloc(sizeof(SMqPollReq)); + SMqPollReqV2* pReq = taosMemoryMalloc(sizeof(SMqPollReqV2)); if (pReq == NULL) { return NULL; } - strcpy(pReq->topic, pTopic->topicName); - strcpy(pReq->cgroup, tmq->groupId); + /*strcpy(pReq->topic, pTopic->topicName);*/ + /*strcpy(pReq->cgroup, tmq->groupId);*/ + + int32_t tlen = strlen(tmq->groupId); + memcpy(pReq->subKey, tmq->groupId, tlen); + pReq->subKey[tlen] = TMQ_SEPARATOR; + strcpy(pReq->subKey + tlen + 1, pTopic->topicName); pReq->blockingTime = blockingTime; pReq->consumerId = tmq->consumerId; @@ -1184,101 +1092,25 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo pReq->reqId = generateRequestId(); pReq->head.vgId = htonl(pVg->vgId); - pReq->head.contLen = htonl(sizeof(SMqPollReq)); + pReq->head.contLen = htonl(sizeof(SMqPollReqV2)); return pReq; } SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); pRspObj->resType = RES_TYPE__TMQ; - pRspObj->topic = strdup(pWrapper->topicHandle->topicName); - pRspObj->resIter = -1; + strncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); pRspObj->vgId = pWrapper->vgHandle->vgId; - SMqPollRspV2* pRsp = &pWrapper->msg; - int32_t blockNum = taosArrayGetSize(pRsp->blockPos); - pRspObj->res = taosArrayInit(blockNum, sizeof(SReqResultInfo)); - for (int32_t i = 0; i < blockNum; i++) { - int32_t pos = *(int32_t*)taosArrayGet(pRsp->blockPos, i); - SRetrieveTableRsp* pRetrieve = POINTER_SHIFT(pRsp->blockData, pos); - SReqResultInfo resInfo = {0}; - resInfo.totalRows = 0; - resInfo.precision = TSDB_TIME_PRECISION_MILLI; - setResSchemaInfo(&resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); - setQueryResultFromRsp(&resInfo, pRetrieve, true); - taosArrayPush(pRspObj->res, &resInfo); - } - return pRspObj; -} - -#if 0 -tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { - tmq_message_t* msg = NULL; - for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); - /*if (vgStatus != TMQ_VG_STATUS__IDLE) {*/ - /*continue;*/ - /*}*/ - SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); - if (pReq == NULL) { - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - // TODO: out of mem - return NULL; - } - - SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); - if (pParam == NULL) { - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - // TODO: out of mem - return NULL; - } - pParam->tmq = tmq; - pParam->pVg = pVg; - pParam->epoch = tmq->epoch; - pParam->sync = 1; - pParam->msg = &msg; - tsem_init(&pParam->rspSem, 0, 0); - - SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); - if (sendInfo == NULL) { - return NULL; - } - - sendInfo->msgInfo = (SDataBuf){ - .pData = pReq, - .len = sizeof(SMqPollReq), - .handle = NULL, - }; - sendInfo->requestId = generateRequestId(); - sendInfo->requestObjRefId = 0; - sendInfo->param = pParam; - sendInfo->fp = tmqPollCb; - sendInfo->msgType = TDMT_VND_CONSUME; + pRspObj->resIter = -1; + memcpy(&pRspObj->rsp, &pWrapper->msg, sizeof(SMqDataBlkRsp)); - int64_t transporterId = 0; - /*printf("send poll\n");*/ - atomic_add_fetch_32(&tmq->waitingRequest, 1); - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - pVg->pollCnt++; - tmq->pollCnt++; + pRspObj->resInfo.totalRows = 0; + pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; + setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); - tsem_wait(&pParam->rspSem); - tmq_message_t* nmsg = NULL; - while (1) { - taosReadQitem(tmq->mqueue, (void**)&nmsg); - if (nmsg == NULL) continue; - while (nmsg->head.mqMsgType != TMQ_MSG_TYPE__POLL_RSP) { - taosReadQitem(tmq->mqueue, (void**)&nmsg); - } - return nmsg; - } - } - } - return NULL; + taosFreeQitem(pWrapper); + return pRspObj; } -#endif int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { /*printf("call poll\n");*/ @@ -1301,7 +1133,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { #endif } atomic_store_32(&pVg->vgSkipCnt, 0); - SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); + SMqPollReqV2* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); if (pReq == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); /*tsem_post(&tmq->rspSem);*/ @@ -1332,7 +1164,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { sendInfo->msgInfo = (SDataBuf){ .pData = pReq, - .len = sizeof(SMqPollReq), + .len = sizeof(SMqPollReqV2), .handle = NULL, }; sendInfo->requestId = pReq->reqId; @@ -1343,7 +1175,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int64_t transporterId = 0; /*printf("send poll\n");*/ - atomic_add_fetch_32(&tmq->waitingRequest, 1); + /*atomic_add_fetch_32(&tmq->waitingRequest, 1);*/ tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId); /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ @@ -1385,7 +1217,7 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; - atomic_sub_fetch_32(&tmq->readyRequest, 1); + /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/ /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/ if (pollRspWrapper->msg.head.epoch == atomic_load_32(&tmq->epoch)) { /*printf("epoch match\n");*/ @@ -1393,7 +1225,7 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ pVg->currentOffset = pollRspWrapper->msg.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - if (pollRspWrapper->msg.dataLen == 0) { + if (pollRspWrapper->msg.blockNum == 0) { taosFreeQitem(pollRspWrapper); rspWrapper = NULL; continue; @@ -1418,38 +1250,16 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { } } -#if 0 -tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) { - tmq_message_t* rspMsg = NULL; - int64_t startTime = taosGetTimestampMs(); - - int64_t status = atomic_load_64(&tmq->status); - tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT); - - while (1) { - rspMsg = tmqSyncPollImpl(tmq, blocking_time); - if (rspMsg && rspMsg->consumeRsp.numOfTopics) { - return rspMsg; - } - - if (blocking_time != 0) { - int64_t endTime = taosGetTimestampMs(); - if (endTime - startTime > blocking_time) { - return NULL; - } - } else - return NULL; - } -} -#endif - TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { SMqRspObj* rspObj; int64_t startTime = taosGetTimestampMs(); // TODO: put into another thread or delayed queue int64_t status = atomic_load_64(&tmq->status); - tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT); + while (0 != tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT)) { + tscDebug("not ready, retry\n"); + taosSsleep(1); + } rspObj = tmqHandleAllRsp(tmq, blocking_time, false); if (rspObj) { @@ -1477,147 +1287,10 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { } } -#if 0 - - if (blocking_time <= 0) blocking_time = 1; - if (blocking_time > 1000) blocking_time = 1000; - /*blocking_time = 1;*/ - - if (taosArrayGetSize(tmq->clientTopics) == 0) { - tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); - /*printf("over1\n");*/ - taosMsleep(blocking_time); - return NULL; - } - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); - if (taosArrayGetSize(pTopic->vgs) == 0) { - /*printf("over2\n");*/ - taosMsleep(blocking_time); - return NULL; - } - - tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); - int32_t beginVgIdx = pTopic->nextVgIdx; - while (1) { - pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); - /*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/ - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg); - if (pReq == NULL) { - ASSERT(false); - taosMsleep(blocking_time); - return NULL; - } - - SMqPollCbParam* param = taosMemoryMalloc(sizeof(SMqPollCbParam)); - if (param == NULL) { - ASSERT(false); - taosMsleep(blocking_time); - return NULL; - } - param->tmq = tmq; - param->retMsg = &tmq_message; - param->pVg = pVg; - tsem_init(¶m->rspSem, 0, 0); - - SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); - pRequest->body.requestMsg = (SDataBuf){ - .pData = pReq, - .len = sizeof(SMqConsumeReq), - .handle = NULL, - }; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->requestObjRefId = 0; - sendInfo->param = param; - sendInfo->fp = tmqPollCb; - - /*printf("req offset: %ld\n", pReq->offset);*/ - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - tmq->pollCnt++; - - tsem_wait(¶m->rspSem); - tsem_destroy(¶m->rspSem); - taosMemoryFree(param); - - if (tmq_message == NULL) { - if (beginVgIdx == pTopic->nextVgIdx) { - taosMsleep(blocking_time); - } else { - continue; - } - } - - return tmq_message; - } - - /*tsem_wait(&pRequest->body.rspSem);*/ - - /*if (body != NULL) {*/ - /*destroySendMsgInfo(body);*/ - /*}*/ - - /*if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {*/ - /*pRequest->code = terrno;*/ - /*}*/ - - /*return pRequest;*/ -} -#endif - -#if 0 -tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { - if (tmq_topic_vgroup_list != NULL) { - // TODO - } - - // TODO: change semaphore to gate - for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, pTopic, pVg); - - SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); - pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; - SMqCommitCbParam* pParam = taosMemoryMalloc(sizeof(SMqCommitCbParam)); - if (pParam == NULL) { - continue; - } - pParam->tmq = tmq; - pParam->pVg = pVg; - pParam->async = async; - if (!async) tsem_init(&pParam->rspSem, 0, 0); - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->requestObjRefId = 0; - sendInfo->param = pParam; - sendInfo->fp = tmqCommitCb; - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - - if (!async) tsem_wait(&pParam->rspSem); - } - } - - return 0; -} -#endif - -#if 0 -void tmq_message_destroy(tmq_message_t* tmq_message) { - if (tmq_message == NULL) return; - SMqPollRsp* pRsp = &tmq_message->msg; - tDeleteSMqConsumeRsp(pRsp); - /*taosMemoryFree(tmq_message);*/ - taosFreeQitem(tmq_message); +tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { + // TODO + return TMQ_RESP_ERR__SUCCESS; } -#endif - -tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { return TMQ_RESP_ERR__SUCCESS; } const char* tmq_err2str(tmq_resp_err_t err) { if (err == TMQ_RESP_ERR__SUCCESS) { @@ -1643,10 +1316,3 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { return -1; } } - -void tmq_message_destroy(TAOS_RES* res) { - if (res == NULL) return; - if (TD_RES_TMQ(res)) { - SMqRspObj* pRspObj = (SMqRspObj*)res; - } -} diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 53b0fec8ff7fdb7f8288ecdd47e0751c3d7cde70..8e8c62cd946562d26ef5d3062683fc2f9c5bdc10 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -662,7 +662,7 @@ TEST(testCase, agg_query_tables) { TAOS_RES* pRes = taos_query(pConn, "use abc1"); taos_free_result(pRes); - pRes = taos_query(pConn, "select count(*) from tu"); + pRes = taos_query(pConn, "select now() from m1"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); diff --git a/source/client/test/tmqTest.cpp b/source/client/test/tmqTest.cpp index d45050d7ffeaa0d4c47188441610a78b2d1738a8..00ed16944cc9b7ef55881c909dcc0b10e31e680b 100644 --- a/source/client/test/tmqTest.cpp +++ b/source/client/test/tmqTest.cpp @@ -108,7 +108,7 @@ TEST(testCase, tmq_subscribe_ctb_Test) { while (1) { tmq_message_t* msg = tmq_consumer_poll(tmq, 1000); - tmq_message_destroy(msg); + taos_free_result(msg); //printf("get msg\n"); //if (msg == NULL) break; } @@ -141,7 +141,7 @@ TEST(testCase, tmq_subscribe_stb_Test) { tmq_commit(tmq, NULL, 0); } //tmq_commit(tmq, NULL, 0); - tmq_message_destroy(msg); + taos_free_result(msg); //printf("get msg\n"); } } diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 63a04a88138184ee10ad0c78de9fcc1bcae5a0d1..e01b113a049466d1ccb1507a409f465c34f5e0e0 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -5,8 +5,8 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/include/common" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" IF(${TD_WINDOWS}) - PRIVATE "${TD_SOURCE_DIR}/contrib/pthread-win32" - PRIVATE "${TD_SOURCE_DIR}/contrib/gnuregex" + PRIVATE "${TD_SOURCE_DIR}/contrib/pthread" + PRIVATE "${TD_SOURCE_DIR}/contrib/msvcregex" ENDIF () ) target_link_libraries( diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 975d097205e8f09c531d706562eb396137637437..d796df16d367ff975f23c8b1c6c94c8f0f2c946d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -113,14 +113,27 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con int32_t type = pColumnInfoData->info.type; if (IS_VAR_DATA_TYPE(type)) { + int32_t dataLen = varDataTLen(pData); + if(type == TSDB_DATA_TYPE_JSON) { + if(*pData == TSDB_DATA_TYPE_NULL) { + dataLen = 0; + }else if(*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData+CHAR_BYTES); + }else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + dataLen = LONG_BYTES; + }else if(*pData == TSDB_DATA_TYPE_BOOL) { + dataLen = CHAR_BYTES; + } + dataLen += CHAR_BYTES; + } SVarColAttr* pAttr = &pColumnInfoData->varmeta; - if (pAttr->allocLen < pAttr->length + varDataTLen(pData)) { + if (pAttr->allocLen < pAttr->length + dataLen) { uint32_t newSize = pAttr->allocLen; if (newSize == 0) { newSize = 8; } - while (newSize < pAttr->length + varDataTLen(pData)) { + while (newSize < pAttr->length + dataLen) { newSize = newSize * 1.5; } @@ -136,8 +149,8 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con uint32_t len = pColumnInfoData->varmeta.length; pColumnInfoData->varmeta.offset[currentRow] = len; - memcpy(pColumnInfoData->pData + len, pData, varDataTLen(pData)); - pColumnInfoData->varmeta.length += varDataTLen(pData); + memcpy(pColumnInfoData->pData + len, pData, dataLen); + pColumnInfoData->varmeta.length += dataLen; } else { memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes); } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f2a5a98d96c0fb524e510c3a49dd9c82f1a1dfd3..2ed08ac81fda23e1ef12a95d840e1e4eb7043c3e 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -110,7 +110,7 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { for (int i = 0; i < numOfCols; i++) { col_type_t type = 0; - int8_t sma = TSDB_BSMA_TYPE_NONE; + int8_t sma = 0; col_id_t colId = 0; col_bytes_t bytes = 0; buf = taosDecodeFixedI8(buf, &type); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 49f4afc12b88a30cb1083c8fa76094ff47564005..73031c410d40433e77d84e80b7d1eb6b86ff5fbb 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -30,7 +30,6 @@ char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port uint16_t tsServerPort = 6030; int32_t tsVersion = 30000000; int32_t tsStatusInterval = 1; // second -bool tsEnableTelemetryReporting = false; // common int32_t tsRpcTimer = 300; @@ -75,6 +74,12 @@ uint16_t tsMonitorPort = 6043; int32_t tsMonitorMaxLogs = 100; bool tsMonitorComp = false; +// telem +bool tsEnableTelem = false; +int32_t tsTelemInterval = 86400; +char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com"; +uint16_t tsTelemPort = 80; + /* * denote if the server needs to compress response message at the application layer to client, including query rsp, * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server. @@ -284,6 +289,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "fnDebugFlag", fnDebugFlag, 0, 255, 0) != 0) return -1; return 0; } @@ -354,7 +360,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddDir(pCfg, "dataDir", tsDataDir, 0) != 0) return -1; if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; - if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelemetryReporting, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxConnections", tsMaxConnections, 1, 100000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0) != 0) return -1; @@ -430,12 +435,17 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeUniqueThreads, 1, 1024, 0) != 0) return -1; if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 360000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1; if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, 0) != 0) return -1; if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, 0) != 0) return -1; + if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, 0) != 0) return -1; + if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, 0) != 0) return -1; + return 0; } @@ -464,6 +474,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) { tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; + fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32; } static int32_t taosSetClientCfg(SConfig *pCfg) { @@ -528,7 +539,6 @@ static void taosSetSystemCfg(SConfig *pCfg) { static int32_t taosSetServerCfg(SConfig *pCfg) { tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; - tsEnableTelemetryReporting = cfgGetItem(pCfg, "telemetryReporting")->bval; tsMaxConnections = cfgGetItem(pCfg, "maxConnections")->i32; tsMaxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32; @@ -572,6 +582,11 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32; tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval; + tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; + tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; + tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); + tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; + if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index f4ef55f86520a081dd096b487b631329af6b3f06..0ea196f1a1bae68f937061abce6f63c42ffb2a5f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -125,14 +125,14 @@ int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) { return tlen; } -void *taosDecodeSEpSet(void *buf, SEpSet *pEp) { +void *taosDecodeSEpSet(const void *buf, SEpSet *pEp) { buf = taosDecodeFixedI8(buf, &pEp->inUse); buf = taosDecodeFixedI8(buf, &pEp->numOfEps); for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) { buf = taosDecodeFixedU16(buf, &pEp->eps[i].port); buf = taosDecodeStringTo(buf, pEp->eps[i].fqdn); } - return buf; + return (void *)buf; } static int32_t tSerializeSClientHbReq(SCoder *pEncoder, const SClientHbReq *pReq) { @@ -434,6 +434,15 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { for (int8_t i = 0; i < param->nFuncIds; ++i) { tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]); } + tlen += taosEncodeFixedI32(buf, param->qmsg1Len); + if (param->qmsg1Len > 0) { + tlen += taosEncodeString(buf, param->qmsg1); + } + + tlen += taosEncodeFixedI32(buf, param->qmsg2Len); + if (param->qmsg2Len > 0) { + tlen += taosEncodeString(buf, param->qmsg2); + } } break; case TD_CHILD_TABLE: @@ -496,18 +505,25 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); } if (pReq->rollup) { - pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam)); + pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam)); SRSmaParam *param = pReq->stbCfg.pRSmaParam; buf = taosDecodeBinaryTo(buf, (void *)¶m->xFilesFactor, sizeof(param->xFilesFactor)); buf = taosDecodeFixedI32(buf, ¶m->delay); buf = taosDecodeFixedI8(buf, ¶m->nFuncIds); if (param->nFuncIds > 0) { - param->pFuncIds = (func_id_t *)taosMemoryMalloc(param->nFuncIds * sizeof(func_id_t)); + param->pFuncIds = (func_id_t *)taosMemoryCalloc(param->nFuncIds, sizeof(func_id_t)); for (int8_t i = 0; i < param->nFuncIds; ++i) { buf = taosDecodeFixedI32(buf, param->pFuncIds + i); } - } else { - param->pFuncIds = NULL; + } + buf = taosDecodeFixedI32(buf, ¶m->qmsg1Len); + if (param->qmsg1Len > 0) { + buf = taosDecodeString(buf, ¶m->qmsg1); + } + + buf = taosDecodeFixedI32(buf, ¶m->qmsg2Len); + if (param->qmsg2Len > 0) { + buf = taosDecodeString(buf, ¶m->qmsg2); } } else { pReq->stbCfg.pRSmaParam = NULL; @@ -611,6 +627,8 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfSmas) < 0) return -1; if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1; + if (tEncodeI32(&encoder, pReq->ast1Len) < 0) return -1; + if (tEncodeI32(&encoder, pReq->ast2Len) < 0) return -1; for (int32_t i = 0; i < pReq->numOfColumns; ++i) { SField *pField = taosArrayGet(pReq->pColumns, i); @@ -636,6 +654,12 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (pReq->commentLen > 0) { if (tEncodeBinary(&encoder, pReq->comment, pReq->commentLen) < 0) return -1; } + if (pReq->ast1Len > 0) { + if (tEncodeBinary(&encoder, pReq->pAst1, pReq->ast1Len) < 0) return -1; + } + if (pReq->ast2Len > 0) { + if (tEncodeBinary(&encoder, pReq->pAst2, pReq->ast2Len) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -658,6 +682,8 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfSmas) < 0) return -1; if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->ast1Len) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->ast2Len) < 0) return -1; pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField)); pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField)); @@ -706,6 +732,18 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1; } + if (pReq->ast1Len > 0) { + pReq->pAst1 = taosMemoryMalloc(pReq->ast1Len); + if (pReq->pAst1 == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->pAst1) < 0) return -1; + } + + if (pReq->ast2Len > 0) { + pReq->pAst2 = taosMemoryMalloc(pReq->ast2Len); + if (pReq->pAst2 == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->pAst2) < 0) return -1; + } + tEndDecode(&decoder); tCoderClear(&decoder); @@ -717,6 +755,8 @@ void tFreeSMCreateStbReq(SMCreateStbReq *pReq) { taosArrayDestroy(pReq->pTags); taosArrayDestroy(pReq->pSmas); taosMemoryFreeClear(pReq->comment); + taosMemoryFreeClear(pReq->pAst1); + taosMemoryFreeClear(pReq->pAst2); pReq->pColumns = NULL; pReq->pTags = NULL; pReq->pSmas = NULL; @@ -804,6 +844,27 @@ void tFreeSMAltertbReq(SMAltertbReq *pReq) { taosArrayDestroy(pReq->pFields); pReq->pFields = NULL; } +int32_t tSerializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeSEpSet(&encoder, &pReq->epSet) < 0) return -1; + + tEndEncode(&encoder); + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} +int32_t tDeserializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeSEpSet(&decoder, &pReq->epSet) < 0) return -1; + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) { SCoder encoder = {0}; @@ -1003,7 +1064,7 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { for (int32_t i = 0; i < vlen; ++i) { SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i); if (tEncodeI32(&encoder, pload->vgId) < 0) return -1; - if (tEncodeI8(&encoder, pload->role) < 0) return -1; + if (tEncodeI32(&encoder, pload->syncState) < 0) return -1; if (tEncodeI64(&encoder, pload->numOfTables) < 0) return -1; if (tEncodeI64(&encoder, pload->numOfTimeSeries) < 0) return -1; if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1; @@ -1054,7 +1115,7 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { for (int32_t i = 0; i < vlen; ++i) { SVnodeLoad vload = {0}; if (tDecodeI32(&decoder, &vload.vgId) < 0) return -1; - if (tDecodeI8(&decoder, &vload.role) < 0) return -1; + if (tDecodeI32(&decoder, &vload.syncState) < 0) return -1; if (tDecodeI64(&decoder, &vload.numOfTables) < 0) return -1; if (tDecodeI64(&decoder, &vload.numOfTimeSeries) < 0) return -1; if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1; @@ -1491,6 +1552,7 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq if (tEncodeI8(&encoder, pReq->outputType) < 0) return -1; if (tEncodeI32(&encoder, pReq->outputLen) < 0) return -1; if (tEncodeI32(&encoder, pReq->bufSize) < 0) return -1; + if (tEncodeI32(&encoder, pReq->codeLen) < 0) return -1; if (tEncodeI64(&encoder, pReq->signature) < 0) return -1; int32_t codeSize = 0; @@ -1530,6 +1592,7 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR if (tDecodeI8(&decoder, &pReq->outputType) < 0) return -1; if (tDecodeI32(&decoder, &pReq->outputLen) < 0) return -1; if (tDecodeI32(&decoder, &pReq->bufSize) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->codeLen) < 0) return -1; if (tDecodeI64(&decoder, &pReq->signature) < 0) return -1; int32_t codeSize = 0; @@ -1598,6 +1661,7 @@ int32_t tSerializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq * if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfFuncs) < 0) return -1; + if (tEncodeI8(&encoder, pReq->ignoreCodeComment) < 0) return -1; if (pReq->numOfFuncs != (int32_t)taosArrayGetSize(pReq->pFuncNames)) return -1; for (int32_t i = 0; i < pReq->numOfFuncs; ++i) { @@ -1618,6 +1682,7 @@ int32_t tDeserializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfFuncs) < 0) return -1; + if (tDecodeI8(&decoder, (int8_t *)&pReq->ignoreCodeComment) < 0) return -1; pReq->pFuncNames = taosArrayInit(pReq->numOfFuncs, TSDB_FUNC_NAME_LEN); if (pReq->pFuncNames == NULL) return -1; @@ -1655,8 +1720,12 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp * if (tEncodeI64(&encoder, pInfo->signature) < 0) return -1; if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1; if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1; - if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1; - if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1; + if (pInfo->codeSize) { + if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1; + } + if (pInfo->commentSize) { + if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1; + } } tEndEncode(&encoder); @@ -1687,15 +1756,23 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp if (tDecodeI64(&decoder, &fInfo.signature) < 0) return -1; if (tDecodeI32(&decoder, &fInfo.codeSize) < 0) return -1; if (tDecodeI32(&decoder, &fInfo.commentSize) < 0) return -1; - fInfo.pCode = taosMemoryCalloc(1, fInfo.codeSize); - fInfo.pComment = taosMemoryCalloc(1, fInfo.commentSize); - if (fInfo.pCode == NULL || fInfo.pComment == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + if (fInfo.codeSize) { + fInfo.pCode = taosMemoryCalloc(1, fInfo.codeSize); + if (fInfo.pCode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + if (tDecodeCStrTo(&decoder, fInfo.pCode) < 0) return -1; + } + if (fInfo.commentSize) { + fInfo.pComment = taosMemoryCalloc(1, fInfo.commentSize); + if (fInfo.pComment == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + if (tDecodeCStrTo(&decoder, fInfo.pComment) < 0) return -1; } - if (tDecodeCStrTo(&decoder, fInfo.pCode) < 0) return -1; - if (tDecodeCStrTo(&decoder, fInfo.pComment) < 0) return -1; taosArrayPush(pRsp->pFuncInfos, &fInfo); } tEndDecode(&decoder); @@ -1704,12 +1781,20 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp return 0; } +void tFreeSFuncInfo(SFuncInfo *pInfo) { + if (NULL == pInfo) { + return; + } + + taosMemoryFree(pInfo->pCode); + taosMemoryFree(pInfo->pComment); +} + void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp *pRsp) { int32_t size = taosArrayGetSize(pRsp->pFuncInfos); for (int32_t i = 0; i < size; ++i) { SFuncInfo *pInfo = taosArrayGet(pRsp->pFuncInfos, i); - taosMemoryFree(pInfo->pCode); - taosMemoryFree(pInfo->pComment); + tFreeSFuncInfo(pInfo); } taosArrayDestroy(pRsp->pFuncInfos); } @@ -1736,7 +1821,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { if (tEncodeI8(&encoder, pReq->precision) < 0) return -1; if (tEncodeI8(&encoder, pReq->compression) < 0) return -1; if (tEncodeI8(&encoder, pReq->replications) < 0) return -1; - if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1; + if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; if (tEncodeI8(&encoder, pReq->update) < 0) return -1; if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1; @@ -1779,7 +1864,7 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1; if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1; if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; if (tDecodeI8(&decoder, &pReq->update) < 0) return -1; if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1; @@ -1827,7 +1912,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; - if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1; + if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; if (tEncodeI8(&encoder, pReq->replications) < 0) return -1; tEndEncode(&encoder); @@ -1849,7 +1934,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1; tEndDecode(&decoder); @@ -2006,7 +2091,7 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->epSetList); } -int32_t tSerializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) { +int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -2019,7 +2104,7 @@ int32_t tSerializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) { return tlen; } -int32_t tDeserializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) { +int32_t tDeserializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) { SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -2207,10 +2292,18 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) { if (tEncodeI8(&encoder, pRsp->precision) < 0) return -1; if (tEncodeI8(&encoder, pRsp->compression) < 0) return -1; if (tEncodeI8(&encoder, pRsp->replications) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->quorum) < 0) return -1; + if (tEncodeI8(&encoder, pRsp->strict) < 0) return -1; if (tEncodeI8(&encoder, pRsp->update) < 0) return -1; if (tEncodeI8(&encoder, pRsp->cacheLastRow) < 0) return -1; if (tEncodeI8(&encoder, pRsp->streamMode) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->numOfRetensions) < 0) return -1; + for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { + SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i); + if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1; + if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1; + if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; + if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2238,11 +2331,28 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { if (tDecodeI8(&decoder, &pRsp->precision) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->compression) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->replications) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->quorum) < 0) return -1; + if (tDecodeI8(&decoder, &pRsp->strict) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->update) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->cacheLastRow) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->streamMode) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->numOfRetensions) < 0) return -1; + pRsp->pRetensions = taosArrayInit(pRsp->numOfRetensions, sizeof(SRetention)); + if (pRsp->pRetensions == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { + SRetention rentension = {0}; + if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1; + if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1; + if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; + if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; + if (taosArrayPush(pRsp->pRetensions, &rentension) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } tEndDecode(&decoder); tCoderClear(&decoder); @@ -2352,7 +2462,6 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI64(&encoder, pReq->showId) < 0) return -1; - if (tEncodeI32(&encoder, pReq->type) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1; tEndEncode(&encoder); @@ -2368,7 +2477,6 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->type) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1; tEndDecode(&decoder); @@ -2623,6 +2731,10 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; + if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1; + if (tEncodeI8(&encoder, pReq->withSchema) < 0) return -1; + if (tEncodeI8(&encoder, pReq->withTag) < 0) return -1; + if (tEncodeI8(&encoder, pReq->withTagSchema) < 0) return -1; if (tEncodeI32(&encoder, sqlLen) < 0) return -1; if (tEncodeI32(&encoder, astLen) < 0) return -1; if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; @@ -2645,6 +2757,10 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->withSchema) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->withTag) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->withTagSchema) < 0) return -1; if (tDecodeI32(&decoder, &sqlLen) < 0) return -1; if (tDecodeI32(&decoder, &astLen) < 0) return -1; @@ -2705,6 +2821,8 @@ int32_t tSerializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) { if (tEncodeI32(&encoder, pReq->pid) < 0) return -1; if (tEncodeCStr(&encoder, pReq->app) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->passwd) < 0) return -1; if (tEncodeI64(&encoder, pReq->startTime) < 0) return -1; tEndEncode(&encoder); @@ -2722,6 +2840,8 @@ int32_t tDeserializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) { if (tDecodeI32(&decoder, &pReq->pid) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->app) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->passwd) < 0) return -1; if (tDecodeI64(&decoder, &pReq->startTime) < 0) return -1; tEndDecode(&decoder); @@ -2831,7 +2951,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pReq->precision) < 0) return -1; if (tEncodeI8(&encoder, pReq->compression) < 0) return -1; - if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1; + if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; if (tEncodeI8(&encoder, pReq->update) < 0) return -1; if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; if (tEncodeI8(&encoder, pReq->replica) < 0) return -1; @@ -2882,7 +3002,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1; if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; if (tDecodeI8(&decoder, &pReq->update) < 0) return -1; if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1; @@ -2954,6 +3074,85 @@ int32_t tDeserializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq return 0; } +int32_t tSerializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; + if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; + if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; + if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; + if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; + if (tEncodeI8(&encoder, pReq->replica) < 0) return -1; + if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + SReplica *pReplica = &pReq->replicas[i]; + if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; + } + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + SReplica *pReplica = &pReq->replicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} + int32_t tSerializeSKillQueryReq(void *buf, int32_t bufLen, SKillQueryReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -3099,6 +3298,34 @@ int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { return 0; } +int32_t tSerializeSServerStatusRsp(void *buf, int32_t bufLen, SServerStatusRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->statusCode) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->details) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSServerStatusRsp(void *buf, int32_t bufLen, SServerStatusRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->statusCode) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->details) < 0) return -1; + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} + int32_t tEncodeSMqOffset(SCoder *encoder, const SMqOffset *pOffset) { if (tEncodeI32(encoder, pOffset->vgId) < 0) return -1; if (tEncodeI64(encoder, pOffset->offset) < 0) return -1; @@ -3439,8 +3666,6 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; if (tEncodeI32(&encoder, sqlLen) < 0) return -1; if (tEncodeI32(&encoder, astLen) < 0) return -1; - if (tEncodeI8(&encoder, pReq->triggerType) < 0) return -1; - if (tEncodeI64(&encoder, pReq->watermark) < 0) return -1; if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1; @@ -3488,3 +3713,27 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { taosMemoryFreeClear(pReq->sql); taosMemoryFreeClear(pReq->ast); } + +STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) { + STSchemaBuilder schemaBuilder = {0}; + if (tdInitTSchemaBuilder(&schemaBuilder, 0) < 0) { + return NULL; + } + + for (int i = 0; i < nCols; i++) { + SSchema *schema = *pSchema + i; + if (tdAddColToSchema(&schemaBuilder, schema->type, schema->flags, schema->colId, schema->bytes) < 0) { + tdDestroyTSchemaBuilder(&schemaBuilder); + return NULL; + } + } + + STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder); + if (pNSchema == NULL) { + tdDestroyTSchemaBuilder(&schemaBuilder); + return NULL; + } + + tdDestroyTSchemaBuilder(&schemaBuilder); + return pNSchema; +} diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c index cb5e2b07c1b7dd8864bd505bb35527868254a6f3..5b4bb539e3ba0790d42a23a7018833981e270712 100644 --- a/source/common/src/tmsgcb.c +++ b/source/common/src/tmsgcb.c @@ -40,4 +40,8 @@ void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) { void tmsgReleaseHandle(void* handle, int8_t type) { (*tsDefaultMsgCb.releaseHandleFp)(tsDefaultMsgCb.pWrapper, handle, type); +} + +void tmsgReportStartup(const char* name, const char* desc) { + (*tsDefaultMsgCb.reportStartupFp)(tsDefaultMsgCb.pWrapper, name, desc); } \ No newline at end of file diff --git a/source/common/src/trow.c b/source/common/src/trow.c index a1a2d236f90b58c9b1bb0122af22e21ab98d35a3..c73f26e6dab78828965788e78a21ece9231d9aff 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -24,9 +24,8 @@ const uint8_t tdVTypeByte[2][3] = {{ }, { // 1 bit - TD_VTYPE_NORM_BYTE_I, - TD_VTYPE_NULL_BYTE_I, - TD_VTYPE_NULL_BYTE_I, // padding + TD_VTYPE_NORM_BYTE_I, TD_VTYPE_NULL_BYTE_I, + TD_VTYPE_NULL_BYTE_I, // padding } }; @@ -224,17 +223,40 @@ static uint8_t tdGetMergedBitmapByte(uint8_t byte) { * @brief Merge bitmap from 2 bits to 1 bits, and the memory buffer should be guaranteed by the invoker. * * @param srcBitmap - * @param srcLen + * @param nBits * @param dstBitmap */ -void tdMergeBitmap(uint8_t *srcBitmap, int32_t srcLen, uint8_t *dstBitmap) { +void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap) { int32_t i = 0, j = 0; + int32_t nBytes = TD_BITMAP_BYTES(nBits); + int32_t nStrictBytes = nBits / 4; + int32_t nPartialBits = nBits - nStrictBytes * 4; - if (srcLen > 0) { + switch (nPartialBits) { + case 0: + // NOTHING TODO + break; + case 1: { + void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + *(uint8_t *)lastByte &= 0xC0; + } break; + case 2: { + void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + *(uint8_t *)lastByte &= 0xF0; + } break; + case 3: { + void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + *(uint8_t *)lastByte &= 0xFC; + } break; + default: + ASSERT(0); + } + + if (nBytes > 0) { dstBitmap[j] = (tdGetMergedBitmapByte(srcBitmap[i]) << 4); } - while ((++i) < srcLen) { + while ((++i) < nBytes) { if ((i & 1) == 0) { dstBitmap[j] = (tdGetMergedBitmapByte(srcBitmap[i]) << 4); } else { @@ -381,17 +403,18 @@ STSRow *tdRowDup(STSRow *row) { } /** - * @brief - * - * @param pCol - * @param valType - * @param val - * @param numOfRows - * @param maxPoints + * @brief + * + * @param pCol + * @param valType + * @param val + * @param numOfRows + * @param maxPoints * @param bitmapMode default is 0(2 bits), otherwise 1(1 bit) - * @return int + * @return int */ -int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints, int8_t bitmapMode) { +int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints, + int8_t bitmapMode) { TASSERT(pCol != NULL); // Assume that the columns not specified during insert/upsert mean None. @@ -426,7 +449,7 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int pCol->len += pCol->bytes; } #ifdef TD_SUPPORT_BITMAP - tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode); + tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode); #endif return 0; } @@ -496,8 +519,9 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SKvRowIdx *pIdx = tdKvRowColIdxAt(pRow, rcol); int16_t colIdx = -1; if (pIdx) { - colIdx = POINTER_DISTANCE(pRow->data, pIdx) / sizeof(SKvRowIdx); + colIdx = POINTER_DISTANCE(pIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx); } + TASSERT(colIdx >= 0); SCellVal sVal = {0}; if (pIdx->colId == pDataCol->colId) { if (tdGetKvRowValOfCol(&sVal, pRow, pBitmap, pIdx->offset, colIdx) < 0) { @@ -537,7 +561,8 @@ int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCol return TSDB_CODE_SUCCESS; } -int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull, TDRowVerT maxVer) { +int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull, + TDRowVerT maxVer) { ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); ASSERT(target->numOfCols == source->numOfCols); int offset = 0; @@ -558,7 +583,8 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * if (tdGetColDataOfRow(&sVal, source->cols + j, i + (*pOffset), source->bitmapMode) < 0) { TASSERT(0); } - tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode); + tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints, + target->bitmapMode); } } ++target->numOfRows; @@ -605,7 +631,8 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) { TASSERT(0); } - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode); + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, + target->bitmapMode); } } @@ -621,12 +648,14 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i TASSERT(0); } if (src2->cols[i].len > 0 && !tdValTypeIsNull(sVal.valType)) { - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode); + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, + target->bitmapMode); } else if (!forceSetNull && key1 == key2 && src1->cols[i].len > 0) { if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) { TASSERT(0); } - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode); + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, + target->bitmapMode); } else if (target->cols[i].len > 0) { dataColSetNullAt(&target->cols[i], target->numOfRows, true, target->bitmapMode); } diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 2f3677330bafaaa44bf25e538a2ce4018fd962d6..43554aed8b231d79a7f98a9c0cc464681471eccc 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -415,7 +415,7 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec if (type == TSDB_DATA_TYPE_BINARY) { newColData = taosMemoryCalloc(1, charLen + 1); memcpy(newColData, varDataVal(inputData), charLen); - bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, 0); + bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight); if (ret != TSDB_CODE_SUCCESS) { taosMemoryFree(newColData); return ret; @@ -429,7 +429,7 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec return TSDB_CODE_FAILED; } newColData[len] = 0; - bool ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, 0); + bool ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, tsDaylight); if (ret != TSDB_CODE_SUCCESS) { taosMemoryFree(newColData); return ret; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index cb01a1727315b3026ba778d8c4b26a39f9f1e153..1c7a7986e9a83ffdad57a771e5295079ba3a90da 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -379,7 +379,7 @@ static void getStatics_nchr(int8_t bitmapMode, const void *pBitmap, const void * *maxIndex = 0; } -tDataTypeDescriptor tDataTypes[15] = { +tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL}, {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool}, {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, @@ -402,6 +402,7 @@ tDataTypeDescriptor tDataTypes[15] = { {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64}, + {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, }; char tTokenTypeSwitcher[13] = { diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index a0503e43a187d8b136b7e8ab7673be6a15ebe230..a546d9bb6b120712cc24ac660db1fa1d1b12fc15 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -118,7 +118,7 @@ void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type) { } case TSDB_DATA_TYPE_BINARY: { pVar->pz = strndup(z, n); - pVar->nLen = strRmquote(pVar->pz, n); + //pVar->nLen = strRmquote(pVar->pz, n); break; } case TSDB_DATA_TYPE_TIMESTAMP: { diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index d60261dfca74c3d7e658678718e8c606568e6815..fea6c4c891ed64aac4d981a5dfa69de3ffe5356a 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -167,7 +167,7 @@ TEST(testCase, Datablock_test) { printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData)); SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo)); - SBlockOrderInfo order = {.nullFirst = true, .order = TSDB_ORDER_ASC, .slotId = 0}; + SBlockOrderInfo order = { true, TSDB_ORDER_ASC, 0, NULL }; taosArrayPush(pOrderInfo, &order); blockDataSort(b, pOrderInfo); diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 3f5c22dd8488870cca64b1c831d1a4d3a7cf6b1a..7e4505ec95d3154200c97dc09e1d276b9e795f63 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -40,10 +40,12 @@ static void dmSetSignalHandle() { taosSetSignal(SIGTERM, dmStopDnode); taosSetSignal(SIGHUP, dmStopDnode); taosSetSignal(SIGINT, dmStopDnode); - taosSetSignal(SIGTSTP, dmStopDnode); taosSetSignal(SIGABRT, dmStopDnode); taosSetSignal(SIGBREAK, dmStopDnode); +#ifndef WINDOWS + taosSetSignal(SIGTSTP, dmStopDnode); taosSetSignal(SIGQUIT, dmStopDnode); +#endif if (!tsMultiProcess) { } else if (global.ntype == DNODE || global.ntype == NODE_END) { diff --git a/source/dnode/mgmt/implement/inc/dmImp.h b/source/dnode/mgmt/implement/inc/dmImp.h index 52a56305fdaf97fb17c2aa616308bc1f2831b672..32869aee9e196bdf765f763926113eec018f70ff 100644 --- a/source/dnode/mgmt/implement/inc/dmImp.h +++ b/source/dnode/mgmt/implement/inc/dmImp.h @@ -49,6 +49,7 @@ int32_t dmProcessDropNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg) // dmMonitor.c void dmGetVnodeLoads(SDnode *pDnode, SMonVloadInfo *pInfo); +void dmGetMnodeLoads(SDnode *pDnode, SMonMloadInfo *pInfo); void dmSendMonitorReport(SDnode *pDnode); // dmWorker.c @@ -70,6 +71,7 @@ void vmSetMgmtFp(SMgmtWrapper *pWrapper); void mmSetMgmtFp(SMgmtWrapper *pWrapper); void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo); +void mmGetMnodeLoads(SMgmtWrapper *pWrapper, SMonMloadInfo *pInfo); void mmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonMmInfo *mmInfo); void vmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonVmInfo *vmInfo); void qmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonQmInfo *qmInfo); diff --git a/source/dnode/mgmt/implement/src/dmExec.c b/source/dnode/mgmt/implement/src/dmExec.c index f66e7cd9d329e8fbb7590e0bab7d17e0a54ad444..c0b85ed70517c199d5d6c68176e2ab525312fc6e 100644 --- a/source/dnode/mgmt/implement/src/dmExec.c +++ b/source/dnode/mgmt/implement/src/dmExec.c @@ -140,6 +140,7 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) { if (dmRunParentProc(pWrapper) != 0) return -1; } + dmReportStartup(pWrapper->pDnode, pWrapper->name, "openned"); return 0; } @@ -161,6 +162,7 @@ int32_t dmStartNode(SMgmtWrapper *pWrapper) { } } + dmReportStartup(pWrapper->pDnode, pWrapper->name, "started"); return 0; } @@ -172,6 +174,11 @@ void dmStopNode(SMgmtWrapper *pWrapper) { void dmCloseNode(SMgmtWrapper *pWrapper) { dInfo("node:%s, start to close", pWrapper->name); + + while (pWrapper->refCount > 0) { + taosMsleep(10); + } + if (pWrapper->procType == DND_PROC_PARENT) { if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) { dInfo("node:%s, send kill signal to the child process:%d", pWrapper->name, pWrapper->procId); @@ -184,18 +191,10 @@ void dmCloseNode(SMgmtWrapper *pWrapper) { dmStopNode(pWrapper); - pWrapper->required = false; taosWLockLatch(&pWrapper->latch); - if (pWrapper->deployed) { - (*pWrapper->fp.closeFp)(pWrapper); - pWrapper->deployed = false; - } + (*pWrapper->fp.closeFp)(pWrapper); taosWUnLockLatch(&pWrapper->latch); - while (pWrapper->refCount > 0) { - taosMsleep(10); - } - if (pWrapper->procObj) { taosProcCleanup(pWrapper->procObj); pWrapper->procObj = NULL; diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index 1179db8ae22432705084430501ee6483fe32a46b..122c121ed8a073d4a51b060dc8abc0b115c9664c 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -36,7 +36,8 @@ static int32_t dmProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) { } } else { SStatusRsp statusRsp = {0}; - if (pRsp->pCont != NULL && pRsp->contLen > 0 && tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { + if (pRsp->pCont != NULL && pRsp->contLen > 0 && + tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { pDnode->data.dnodeVer = statusRsp.dnodeVer; dmUpdateDnodeCfg(pDnode, &statusRsp.dnodeCfg); dmUpdateEps(pDnode, statusRsp.pDnodeEps); @@ -71,12 +72,26 @@ void dmSendStatusReq(SDnode *pDnode) { memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); taosRUnLockLatch(&pDnode->data.latch); - SMonVloadInfo info = {0}; - dmGetVnodeLoads(pDnode, &info); - req.pVloads = info.pVloads; + SMonVloadInfo vinfo = {0}; + dmGetVnodeLoads(pDnode, &vinfo); + req.pVloads = vinfo.pVloads; + pDnode->data.unsyncedVgId = 0; + pDnode->data.vndState = TAOS_SYNC_STATE_LEADER; + for (int32_t i = 0; i < taosArrayGetSize(req.pVloads); ++i) { + SVnodeLoad *pLoad = taosArrayGet(req.pVloads, i); + if (pLoad->syncState != TAOS_SYNC_STATE_LEADER && pLoad->syncState != TAOS_SYNC_STATE_FOLLOWER) { + pDnode->data.unsyncedVgId = pLoad->vgId; + pDnode->data.vndState = pLoad->syncState; + } + } + + SMonMloadInfo minfo = {0}; + dmGetMnodeLoads(pDnode, &minfo); + pDnode->data.isMnode = minfo.isMnode; + pDnode->data.mndState = minfo.load.syncState; int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); - void * pHead = rpcMallocCont(contLen); + void *pHead = rpcMallocCont(contLen); tSerializeSStatusReq(pHead, contLen, &req); tFreeSStatusReq(&req); @@ -101,7 +116,7 @@ int32_t dmProcessGrantRsp(SDnode *pDnode, SNodeMsg *pMsg) { } int32_t dmProcessConfigReq(SDnode *pDnode, SNodeMsg *pMsg) { - SRpcMsg * pReq = &pMsg->rpcMsg; + SRpcMsg *pReq = &pMsg->rpcMsg; SDCfgDnodeReq *pCfg = pReq->pCont; dError("config req is received, but not supported yet"); return TSDB_CODE_OPS_NOT_SUPPORT; @@ -160,9 +175,9 @@ int32_t dmProcessDropNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg) dmReleaseWrapper(pWrapper); if (code == 0) { - dmCloseNode(pWrapper); pWrapper->required = false; pWrapper->deployed = false; + dmCloseNode(pWrapper); taosRemoveDir(pWrapper->path); } taosThreadMutexUnlock(&pDnode->mutex); @@ -201,6 +216,126 @@ static void dmStopMgmt(SMgmtWrapper *pWrapper) { dmStopStatusThread(pWrapper->pDnode); } +static int32_t dmSpawnUdfd(SDnode *pDnode); + +void dmUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) { + dInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal); + uv_close((uv_handle_t*)process, NULL); + SDnode *pDnode = process->data; + SUdfdData *pData = &pDnode->udfdData; + if (atomic_load_8(&pData->stopping) != 0) { + dDebug("udfd process exit due to stopping"); + } else { + uv_close((uv_handle_t*)&pData->ctrlPipe, NULL); + dmSpawnUdfd(pDnode); + } +} + +static int32_t dmSpawnUdfd(SDnode *pDnode) { + dInfo("dnode start spawning udfd"); + uv_process_options_t options = {0}; + + char path[PATH_MAX] = {0}; + if (tsProcPath == NULL) { + path[0] = '.'; + } else { + strncpy(path, tsProcPath, strlen(tsProcPath)); + taosDirName(path); + } + strcat(path, "/udfd"); + char* argsUdfd[] = {path, "-c", configDir, NULL}; + options.args = argsUdfd; + options.file = path; + + options.exit_cb = dmUdfdExit; + SUdfdData *pData = &pDnode->udfdData; + uv_pipe_init(&pData->loop, &pData->ctrlPipe, 1); + + uv_stdio_container_t child_stdio[3]; + child_stdio[0].flags = UV_CREATE_PIPE | UV_READABLE_PIPE; + child_stdio[0].data.stream = (uv_stream_t*) &pData->ctrlPipe; + child_stdio[1].flags = UV_IGNORE; + child_stdio[2].flags = UV_INHERIT_FD; + child_stdio[2].data.fd = 2; + options.stdio_count = 3; + options.stdio = child_stdio; + + char dnodeIdEnvItem[32] = {0}; + char thrdPoolSizeEnvItem[32] = {0}; + snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pDnode->data.dnodeId); + float numCpuCores = 4; + taosGetCpuCores(&numCpuCores); + snprintf(thrdPoolSizeEnvItem,32, "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores*2); + char* envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, NULL}; + options.env = envUdfd; + + int err = uv_spawn(&pData->loop, &pData->process, &options); + pData->process.data = (void*)pDnode; + + if (err != 0) { + dError("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err)); + } + return err; +} + +static void dmUdfdCloseWalkCb(uv_handle_t* handle, void* arg) { + if (!uv_is_closing(handle)) { + uv_close(handle, NULL); + } +} + +void dmWatchUdfd(void *args) { + SDnode *pDnode = args; + SUdfdData *pData = &pDnode->udfdData; + uv_loop_init(&pData->loop); + int32_t err = dmSpawnUdfd(pDnode); + atomic_store_32(&pData->spawnErr, err); + uv_barrier_wait(&pData->barrier); + uv_run(&pData->loop, UV_RUN_DEFAULT); + err = uv_loop_close(&pData->loop); + while (err == UV_EBUSY) { + uv_walk(&pData->loop, dmUdfdCloseWalkCb, NULL); + uv_run(&pData->loop, UV_RUN_DEFAULT); + err = uv_loop_close(&pData->loop); + } + return; +} + +int32_t dmStartUdfd(SDnode *pDnode) { + SUdfdData *pData = &pDnode->udfdData; + if (pData->startCalled) { + dInfo("dnode-mgmt start udfd already called"); + return 0; + } + pData->startCalled = true; + uv_barrier_init(&pData->barrier, 2); + pData->stopping = 0; + uv_thread_create(&pData->thread, dmWatchUdfd, pDnode); + uv_barrier_wait(&pData->barrier); + pData->needCleanUp = true; + return pData->spawnErr; +} + +int32_t dmStopUdfd(SDnode *pDnode) { + dInfo("dnode-mgmt to stop udfd. need cleanup: %d, spawn err: %d", + pDnode->udfdData.needCleanUp, pDnode->udfdData.spawnErr); + SUdfdData *pData = &pDnode->udfdData; + if (!pData->needCleanUp) { + return 0; + } + atomic_store_8(&pData->stopping, 1); + + uv_barrier_destroy(&pData->barrier); + if (pData->spawnErr == 0) { + uv_process_kill(&pData->process, SIGINT); + } + uv_stop(&pData->loop); + uv_thread_join(&pData->thread); + + atomic_store_8(&pData->stopping, 0); + return 0; +} + static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { dInfo("dnode-mgmt start to init"); SDnode *pDnode = pWrapper->pDnode; @@ -230,6 +365,11 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { dError("failed to init transport since %s", terrstr()); return -1; } + dmReportStartup(pDnode, "dnode-transport", "initialized"); + + if (dmStartUdfd(pDnode) != 0) { + dError("failed to start udfd"); + } dInfo("dnode-mgmt is initialized"); return 0; @@ -238,6 +378,7 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { static void dmCleanupMgmt(SMgmtWrapper *pWrapper) { dInfo("dnode-mgmt start to clean up"); SDnode *pDnode = pWrapper->pDnode; + dmStopUdfd(pDnode); dmStopWorker(pDnode); taosWLockLatch(&pDnode->data.latch); diff --git a/source/dnode/mgmt/implement/src/dmMonitor.c b/source/dnode/mgmt/implement/src/dmMonitor.c index b0774cd66f1620dedfaabe38c1a08a66808ca857..8543310eb54479fe4edd331b8d38b840892cac20 100644 --- a/source/dnode/mgmt/implement/src/dmMonitor.c +++ b/source/dnode/mgmt/implement/src/dmMonitor.c @@ -183,3 +183,29 @@ void dmGetVnodeLoads(SDnode *pDnode, SMonVloadInfo *pInfo) { } dmReleaseWrapper(pWrapper); } + +void dmGetMnodeLoads(SDnode *pDnode, SMonMloadInfo *pInfo) { + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, MNODE); + if (pWrapper == NULL) { + pInfo->isMnode = 0; + return; + } + + bool getFromAPI = !tsMultiProcess; + if (getFromAPI) { + mmGetMnodeLoads(pWrapper, pInfo); + } else { + SRpcMsg req = {.msgType = TDMT_MON_MM_LOAD}; + SRpcMsg rsp = {0}; + SEpSet epset = {.inUse = 0, .numOfEps = 1}; + tstrncpy(epset.eps[0].fqdn, pDnode->data.localFqdn, TSDB_FQDN_LEN); + epset.eps[0].port = tsServerPort; + + dmSendRecv(pDnode, &epset, &req, &rsp); + if (rsp.code == 0 && rsp.contLen > 0) { + tDeserializeSMonMloadInfo(rsp.pCont, rsp.contLen, pInfo); + } + rpcFreeCont(rsp.pCont); + } + dmReleaseWrapper(pWrapper); +} diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index aa38c2492d0d9d62e317476179c2a19eab99e3c8..a574d802f9838b2219add4807d503db523eb982a 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -119,15 +119,15 @@ _OVER: } static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SDnodeTrans *pTrans = &pDnode->trans; + SDnodeTrans * pTrans = &pDnode->trans; tmsg_t msgType = pMsg->msgType; bool isReq = msgType & 1u; - SMsgHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; + SMsgHandle * pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; SMgmtWrapper *pWrapper = pHandle->pNdWrapper; - if (msgType == TDMT_DND_NETWORK_TEST) { - dTrace("network test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); - dmProcessStartupReq(pDnode, pMsg); + if (msgType == TDMT_DND_SERVER_STATUS) { + dTrace("server status req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); + dmProcessServerStatusReq(pDnode, pMsg); return; } @@ -443,7 +443,7 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void *pReq = rpcMallocCont(contLen); + void * pReq = rpcMallocCont(contLen); tSerializeSAuthReq(pReq, contLen, &authReq); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; @@ -519,7 +519,8 @@ SMsgCb dmGetMsgcb(SMgmtWrapper *pWrapper) { .sendRspFp = dmSendRsp, .registerBrokenLinkArgFp = dmRegisterBrokenLinkArg, .releaseHandleFp = dmReleaseHandle, + .reportStartupFp = dmReportStartupByWrapper, .pWrapper = pWrapper, }; return msgCb; -} \ No newline at end of file +} diff --git a/source/dnode/mgmt/implement/src/dmWorker.c b/source/dnode/mgmt/implement/src/dmWorker.c index 6ffb8e1a23f531b0f1a3f523450600cafc14ccac..505efeb8c6ef1cb4a0205367cf783c79dc81d59b 100644 --- a/source/dnode/mgmt/implement/src/dmWorker.c +++ b/source/dnode/mgmt/implement/src/dmWorker.c @@ -74,6 +74,7 @@ int32_t dmStartStatusThread(SDnode *pDnode) { return -1; } + dmReportStartup(pDnode, "dnode-status", "initialized"); return 0; } @@ -92,6 +93,7 @@ int32_t dmStartMonitorThread(SDnode *pDnode) { return -1; } + dmReportStartup(pDnode, "dnode-monitor", "initialized"); return 0; } diff --git a/source/dnode/mgmt/interface/inc/dmDef.h b/source/dnode/mgmt/interface/inc/dmDef.h index a38349f852cc5df3485581c1534295c29dedfec7..4f4a2ed3499f1c135626a702370bdb9536a2f9d8 100644 --- a/source/dnode/mgmt/interface/inc/dmDef.h +++ b/source/dnode/mgmt/interface/inc/dmDef.h @@ -16,6 +16,7 @@ #ifndef _TD_DM_DEF_H_ #define _TD_DM_DEF_H_ +#include "uv.h" #include "dmLog.h" #include "cJSON.h" @@ -38,6 +39,7 @@ #include "dnode.h" #include "mnode.h" #include "monitor.h" +#include "sync.h" #ifdef __cplusplus extern "C" { @@ -110,6 +112,10 @@ typedef struct { int64_t dnodeVer; int64_t updateTime; int64_t rebootTime; + int32_t unsyncedVgId; + ESyncState vndState; + ESyncState mndState; + bool isMnode; bool dropped; SEpSet mnodeEps; SArray *dnodeEps; @@ -132,14 +138,32 @@ typedef struct { uint16_t serverPort; } SDnodeData; +typedef struct { + char name[TSDB_STEP_NAME_LEN]; + char desc[TSDB_STEP_DESC_LEN]; +} SStartupInfo; + +typedef struct SUdfdData { + bool startCalled; + bool needCleanUp; + uv_loop_t loop; + uv_thread_t thread; + uv_barrier_t barrier; + uv_process_t process; + int spawnErr; + int8_t stopping; + uv_pipe_t ctrlPipe; +} SUdfdData; + typedef struct SDnode { EDndProcType ptype; EDndNodeType ntype; EDndRunStatus status; EDndEvent event; - SStartupReq startup; + SStartupInfo startup; SDnodeTrans trans; SDnodeData data; + SUdfdData udfdData; TdThreadMutex mutex; SMgmtWrapper wrappers[NODE_END]; } SDnode; diff --git a/source/dnode/mgmt/interface/inc/dmInt.h b/source/dnode/mgmt/interface/inc/dmInt.h index 3c8ecdc71bf052e76a29943087dbfd678496f4cc..a2368f31735be850a03dcb98bc14613cc7f23d27 100644 --- a/source/dnode/mgmt/interface/inc/dmInt.h +++ b/source/dnode/mgmt/interface/inc/dmInt.h @@ -35,7 +35,8 @@ void dmSetStatus(SDnode *pDnode, EDndRunStatus stat); void dmSetEvent(SDnode *pDnode, EDndEvent event); void dmSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId); void dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); -void dmProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); +void dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc); +void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pMsg); void dmGetMonitorSysInfo(SMonSysInfo *pInfo); // dmFile.c diff --git a/source/dnode/mgmt/interface/src/dmInt.c b/source/dnode/mgmt/interface/src/dmInt.c index 10599c004385a8faa3a1e592fbdaa4e6ea4778b3..00abbd01993764afc541483fd42152287d2053ab 100644 --- a/source/dnode/mgmt/interface/src/dmInt.c +++ b/source/dnode/mgmt/interface/src/dmInt.c @@ -137,26 +137,65 @@ void dmReleaseWrapper(SMgmtWrapper *pWrapper) { } void dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { - SStartupReq *pStartup = &pDnode->startup; + SStartupInfo *pStartup = &pDnode->startup; tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); - pStartup->finished = 0; + dInfo("step:%s, %s", pStartup->name, pStartup->desc); } -static void dmGetStartup(SDnode *pDnode, SStartupReq *pStartup) { - memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); - pStartup->finished = (pDnode->status == DND_STAT_RUNNING); +void dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc) { + dmReportStartup(pWrapper->pDnode, pName, pDesc); } -void dmProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("startup req is received"); - SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); - dmGetStartup(pDnode, pStartup); +static void dmGetServerStatus(SDnode *pDnode, SServerStatusRsp *pStatus) { + pStatus->details[0] = 0; - dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - SRpcMsg rpcRsp = { - .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; - rpcSendResponse(&rpcRsp); + if (pDnode->status == DND_STAT_INIT) { + pStatus->statusCode = TSDB_SRV_STATUS_NETWORK_OK; + snprintf(pStatus->details, sizeof(pStatus->details), "%s: %s", pDnode->startup.name, pDnode->startup.desc); + } else if (pDnode->status == DND_STAT_STOPPED) { + pStatus->statusCode = TSDB_SRV_STATUS_EXTING; + } else { + SDnodeData *pData = &pDnode->data; + if (pData->isMnode && pData->mndState != TAOS_SYNC_STATE_LEADER && pData->mndState == TAOS_SYNC_STATE_FOLLOWER) { + pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; + snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(pData->mndState)); + } else if (pData->unsyncedVgId != 0 && pData->vndState != TAOS_SYNC_STATE_LEADER && + pData->vndState != TAOS_SYNC_STATE_FOLLOWER) { + pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; + snprintf(pStatus->details, sizeof(pStatus->details), "vnode:%d sync state is %s", pData->unsyncedVgId, + syncStr(pData->vndState)); + } else { + pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_OK; + } + } +} + +void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pReq) { + dDebug("server status req is received"); + + SServerStatusRsp statusRsp = {0}; + dmGetServerStatus(pDnode, &statusRsp); + + SRpcMsg rspMsg = {.handle = pReq->handle, .ahandle = pReq->ahandle}; + int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp); + if (rspLen < 0) { + rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + tSerializeSServerStatusRsp(pRsp, rspLen, &statusRsp); + rspMsg.pCont = pRsp; + rspMsg.contLen = rspLen; + +_OVER: + rpcSendResponse(&rspMsg); } void dmGetMonitorSysInfo(SMonSysInfo *pInfo) { diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmInt.c b/source/dnode/mgmt/mgmt_bnode/src/bmInt.c index f635df8ec78382ed2b34d8b45f2f3c614fa7b4cb..2920d12eb4a4cf4c680f49f96eecdfe0a1f23302 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmInt.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmInt.c @@ -61,12 +61,14 @@ int32_t bmOpen(SMgmtWrapper *pWrapper) { bmClose(pWrapper); return -1; } + dmReportStartup(pWrapper->pDnode, "bnode-impl", "initialized"); if (bmStartWorker(pMgmt) != 0) { dError("failed to start bnode worker since %s", terrstr()); bmClose(pWrapper); return -1; } + dmReportStartup(pWrapper->pDnode, "bnode-worker", "initialized"); return 0; } diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index 5f66ae230a5e50e36d52f861652dc334077386d4..4d40d1fa287417aff15600ae7dc56618c44a3cb0 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -52,6 +52,8 @@ int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t mmProcessGetMonMmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); +int32_t mmProcessGetMnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); +void mmGetMnodeLoads(SMgmtWrapper *pWrapper, SMonMloadInfo *pInfo); // mmWorker.c int32_t mmStartWorker(SMnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 38885333c9253badd1d9af26cb6ee74510269c80..411eff53743619cf5f58c5b58c1852d51dd04694 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -46,6 +46,34 @@ int32_t mmProcessGetMonMmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { return 0; } +void mmGetMnodeLoads(SMgmtWrapper *pWrapper, SMonMloadInfo *pInfo) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + pInfo->isMnode = 1; + mndGetLoad(pMgmt->pMnode, &pInfo->load); +} + +int32_t mmProcessGetMnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { + SMonMloadInfo mloads = {0}; + mmGetMnodeLoads(pWrapper, &mloads); + + int32_t rspLen = tSerializeSMonMloadInfo(NULL, 0, &mloads); + if (rspLen < 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + tSerializeSMonMloadInfo(pRsp, rspLen, &mloads); + pReq->pRsp = pRsp; + pReq->rspLen = rspLen; + return 0; +} + int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { SDnode *pDnode = pWrapper->pDnode; SRpcMsg *pReq = &pMsg->rpcMsg; @@ -117,6 +145,7 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { void mmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_MON_MM_INFO, mmProcessMonitorMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_MON_MM_LOAD, mmProcessMonitorMsg, DEFAULT_HANDLE); // Requests handled by DNODE dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); @@ -129,10 +158,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); // Requests handled by MNODE @@ -159,7 +185,6 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_USE_DB, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, mmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); @@ -186,6 +211,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_GET_DB_CFG, mmProcessReadMsg, DEFAULT_HANDLE); @@ -206,4 +232,8 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MNODE_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MNODE_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MNODE_HANDLE); + + dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_COMPACT_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 69b4d509399cbf0a6e8cd831b4217913a883d530..f707426ab4a52c7f2dc9c6f3f04d337ce4621e75 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -176,12 +176,14 @@ static int32_t mmOpen(SMgmtWrapper *pWrapper) { mmClose(pWrapper); return -1; } + dmReportStartup(pWrapper->pDnode, "mnode-impl", "initialized"); if (mmStartWorker(pMgmt) != 0) { dError("failed to start mnode worker since %s", terrstr()); mmClose(pWrapper); return -1; } + dmReportStartup(pWrapper->pDnode, "mnode-worker", "initialized"); if (!deployed) { deployed = true; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index c575266b44a5849163fa5c92c421581d1b917ce2..e9c40fdd0fb4ddf8478d6a4a93eb1fb064066b68 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -36,6 +36,8 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { code = mmProcessAlterReq(pMgmt, pMsg); } else if (pMsg->rpcMsg.msgType == TDMT_MON_MM_INFO) { code = mmProcessGetMonMmInfoReq(pMgmt->pWrapper, pMsg); + } else if (pMsg->rpcMsg.msgType == TDMT_MON_MM_LOAD) { + code = mmProcessGetMnodeLoadsReq(pMgmt->pWrapper, pMsg); } else { pMsg->pNode = pMgmt->pMnode; code = mndProcessMsg(pMsg); diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmInt.c b/source/dnode/mgmt/mgmt_qnode/src/qmInt.c index d8d02b2619995396888a04aae7966d52d0a18d02..d03f001a8d71c7a6fcfa329c6d667db7157ea461 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmInt.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmInt.c @@ -64,12 +64,14 @@ static int32_t qmOpen(SMgmtWrapper *pWrapper) { qmClose(pWrapper); return -1; } + dmReportStartup(pWrapper->pDnode, "qnode-impl", "initialized"); if (qmStartWorker(pMgmt) != 0) { dError("failed to start qnode worker since %s", terrstr()); qmClose(pWrapper); return -1; } + dmReportStartup(pWrapper->pDnode, "qnode-worker", "initialized"); dInfo("qnode-mgmt is initialized"); return 0; diff --git a/source/dnode/mgmt/mgmt_snode/src/smInt.c b/source/dnode/mgmt/mgmt_snode/src/smInt.c index 8adf643a2b4038f3199953f5ed5b6a92b4e5e2ea..3757dcd72a24d30cadb413a428738c8fed1a45c1 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smInt.c +++ b/source/dnode/mgmt/mgmt_snode/src/smInt.c @@ -60,11 +60,13 @@ int32_t smOpen(SMgmtWrapper *pWrapper) { dError("failed to open snode since %s", terrstr()); return -1; } + dmReportStartup(pWrapper->pDnode, "snode-impl", "initialized"); if (smStartWorker(pMgmt) != 0) { dError("failed to start snode worker since %s", terrstr()); return -1; } + dmReportStartup(pWrapper->pDnode, "snode-worker", "initialized"); return 0; } diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index 73cf07690d4acda496657a70121ac2a70d9fe013..4b42f97f53af81e4faab94df877b06cf04b5a346 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -29,7 +29,6 @@ typedef struct SVnodesMgmt { SHashObj *hash; SRWLatch latch; SVnodesStat state; - SVnodesStat lastState; STfs *pTfs; SQWorkerPool queryPool; SQWorkerPool fetchPool; @@ -90,10 +89,7 @@ void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); // vmHandle.c void vmInitMsgHandle(SMgmtWrapper *pWrapper); int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); -int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); -int32_t vmProcessSyncVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); -int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); int32_t vmProcessGetMonVmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); int32_t vmProcessGetVnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 59089d0cdb06d2c7a71e64946abbba0907f1609e..65180f84dd7b3c17b1fd1c1a50f729da061f2576 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -16,21 +16,42 @@ #define _DEFAULT_SOURCE #include "vmInt.h" -void vmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonVmInfo *vmInfo) { +void vmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonVmInfo *pInfo) { SVnodesMgmt *pMgmt = pWrapper->pMgmt; - tfsGetMonitorInfo(pMgmt->pTfs, &vmInfo->tfs); - - taosWLockLatch(&pMgmt->latch); - vmInfo->vstat.totalVnodes = pMgmt->state.totalVnodes; - vmInfo->vstat.masterNum = pMgmt->state.masterNum; - vmInfo->vstat.numOfSelectReqs = pMgmt->state.numOfSelectReqs - pMgmt->lastState.numOfSelectReqs; - vmInfo->vstat.numOfInsertReqs = pMgmt->state.numOfInsertReqs - pMgmt->lastState.numOfInsertReqs; - vmInfo->vstat.numOfInsertSuccessReqs = pMgmt->state.numOfInsertSuccessReqs - pMgmt->lastState.numOfInsertSuccessReqs; - vmInfo->vstat.numOfBatchInsertReqs = pMgmt->state.numOfBatchInsertReqs - pMgmt->lastState.numOfBatchInsertReqs; - vmInfo->vstat.numOfBatchInsertSuccessReqs = - pMgmt->state.numOfBatchInsertSuccessReqs - pMgmt->lastState.numOfBatchInsertSuccessReqs; - pMgmt->lastState = pMgmt->state; - taosWUnLockLatch(&pMgmt->latch); + + SMonVloadInfo vloads = {0}; + vmGetVnodeLoads(pWrapper, &vloads); + if (vloads.pVloads == NULL) return; + + int32_t totalVnodes = 0; + int32_t masterNum = 0; + int64_t numOfSelectReqs = 0; + int64_t numOfInsertReqs = 0; + int64_t numOfInsertSuccessReqs = 0; + int64_t numOfBatchInsertReqs = 0; + int64_t numOfBatchInsertSuccessReqs = 0; + + for (int32_t i = 0; i < taosArrayGetSize(vloads.pVloads); ++i) { + SVnodeLoad *pLoad = taosArrayGet(vloads.pVloads, i); + numOfSelectReqs += pLoad->numOfSelectReqs; + numOfInsertReqs += pLoad->numOfInsertReqs; + numOfInsertSuccessReqs += pLoad->numOfInsertSuccessReqs; + numOfBatchInsertReqs += pLoad->numOfBatchInsertReqs; + numOfBatchInsertSuccessReqs += pLoad->numOfBatchInsertSuccessReqs; + if (pLoad->syncState == TAOS_SYNC_STATE_LEADER) masterNum++; + totalVnodes++; + } + + pInfo->vstat.totalVnodes = totalVnodes; + pInfo->vstat.masterNum = masterNum; + pInfo->vstat.numOfSelectReqs = numOfSelectReqs - pMgmt->state.numOfSelectReqs; + pInfo->vstat.numOfInsertReqs = numOfInsertReqs - pMgmt->state.numOfInsertReqs; + pInfo->vstat.numOfInsertSuccessReqs = numOfInsertSuccessReqs - pMgmt->state.numOfInsertSuccessReqs; + pInfo->vstat.numOfBatchInsertReqs = numOfBatchInsertReqs - pMgmt->state.numOfBatchInsertReqs; + pInfo->vstat.numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs - pMgmt->state.numOfBatchInsertSuccessReqs; + pMgmt->state = pInfo->vstat; + + taosArrayDestroy(vloads.pVloads); } int32_t vmProcessGetMonVmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { @@ -193,48 +214,6 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return 0; } -int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { - SRpcMsg *pReq = &pMsg->rpcMsg; - SAlterVnodeReq alterReq = {0}; - if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - dDebug("vgId:%d, alter vnode req is received", alterReq.vgId); - - SVnodeCfg vnodeCfg = {0}; - vmGenerateVnodeCfg(&alterReq, &vnodeCfg); - - SVnodeObj *pVnode = vmAcquireVnode(pMgmt, alterReq.vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); - return -1; - } - - if (alterReq.vgVersion == pVnode->vgVersion) { - vmReleaseVnode(pMgmt, pVnode); - dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", alterReq.vgId); - return 0; - } - - if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { - dError("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); - vmReleaseVnode(pMgmt, pVnode); - return -1; - } - - int32_t oldVersion = pVnode->vgVersion; - pVnode->vgVersion = alterReq.vgVersion; - int32_t code = vmWriteVnodesToFile(pMgmt); - if (code != 0) { - pVnode->vgVersion = oldVersion; - } - - vmReleaseVnode(pMgmt, pVnode); - return code; -} - int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { SRpcMsg *pReq = &pMsg->rpcMsg; SDropVnodeReq dropReq = {0}; @@ -266,102 +245,57 @@ int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return 0; } -int32_t vmProcessSyncVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { - SRpcMsg *pReq = &pMsg->rpcMsg; - SSyncVnodeReq syncReq = {0}; - tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &syncReq); - - int32_t vgId = syncReq.vgId; - dDebug("vgId:%d, sync vnode req is received", vgId); - - SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to sync since %s", vgId, terrstr()); - return -1; - } - - if (vnodeSync(pVnode->pImpl) != 0) { - dError("vgId:%d, failed to sync vnode since %s", vgId, terrstr()); - vmReleaseVnode(pMgmt, pVnode); - return -1; - } - - vmReleaseVnode(pMgmt, pVnode); - return 0; -} - -int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { - SRpcMsg *pReq = &pMsg->rpcMsg; - SCompactVnodeReq compatcReq = {0}; - tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &compatcReq); - - int32_t vgId = compatcReq.vgId; - dDebug("vgId:%d, compact vnode req is received", vgId); - - SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to compact since %s", vgId, terrstr()); - return -1; - } - - if (vnodeCompact(pVnode->pImpl) != 0) { - dError("vgId:%d, failed to compact vnode since %s", vgId, terrstr()); - vmReleaseVnode(pMgmt, pVnode); - return -1; - } - - vmReleaseVnode(pMgmt, pVnode); - return 0; -} - void vmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_MON_VM_INFO, vmProcessMonitorMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MON_VM_LOAD, vmProcessMonitorMsg, DEFAULT_HANDLE); // Requests handled by VNODE - dmSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_QUERY, vmProcessQueryMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, vmProcessQueryMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_FETCH, vmProcessFetchMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, vmProcessFetchMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, vmProcessFetchMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, vmProcessFetchMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, vmProcessQueryMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, vmProcessQueryMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, vmProcessWriteMsg, DEFAULT_HANDLE); + // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_RES_READY, vmProcessFetchMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, vmProcessFetchMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, vmProcessFetchMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, vmProcessFetchMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); + // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, vmProcessWriteMsg, DEFAULT_HANDLE); + // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, vmProcessWriteMsg, DEFAULT_HANDLE); + // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, vmProcessWriteMsg, DEFAULT_HANDLE); + // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessFetchMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, vmProcessFetchMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, vmProcessFetchMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, vmProcessMergeMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, vmProcessFetchMsg, DEFAULT_HANDLE); + + dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_VNODE, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_COMPACT_VNODE, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + // dmSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + // dmSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); // sync integration dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_TIMEOUT, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 48bae163d9b35046688a2c180e7aa4c1a71bd99f..77171e8b4ba15062440ac062108e5105c6f38ccb 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -136,7 +136,7 @@ static void *vmOpenVnodeFunc(void *param) { char stepDesc[TSDB_STEP_DESC_LEN] = {0}; snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId, pMgmt->state.openVnodes, pMgmt->state.totalVnodes); - dmReportStartup(pDnode, "open-vnodes", stepDesc); + dmReportStartup(pDnode, "vnode-open", stepDesc); SMsgCb msgCb = pMgmt->pDnode->data.msgCb; msgCb.pWrapper = pMgmt->pWrapper; @@ -305,11 +305,13 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) { dError("failed to init tfs since %s", terrstr()); goto _OVER; } + dmReportStartup(pDnode, "vnode-tfs", "initialized"); if (walInit() != 0) { dError("failed to init wal since %s", terrstr()); goto _OVER; } + dmReportStartup(pDnode, "vnode-wal", "initialized"); // sync integration if (syncInit() != 0) { @@ -321,15 +323,18 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) { dError("failed to init vnode since %s", terrstr()); goto _OVER; } + dmReportStartup(pDnode, "vnode-commit", "initialized"); if (vmStartWorker(pMgmt) != 0) { dError("failed to init workers since %s", terrstr()) goto _OVER; } + dmReportStartup(pDnode, "vnode-worker", "initialized"); if (vmOpenVnodes(pMgmt) != 0) { dError("failed to open vnode since %s", terrstr()); return -1; } + dmReportStartup(pDnode, "vnode-vnodes", "initialized"); code = 0; @@ -364,19 +369,9 @@ void vmSetMgmtFp(SMgmtWrapper *pWrapper) { void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo) { SVnodesMgmt *pMgmt = pWrapper->pMgmt; - SVnodesStat *pStat = &pMgmt->state; - SArray *pLoads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoad)); - int32_t totalVnodes = 0; - int32_t masterNum = 0; - int64_t numOfSelectReqs = 0; - int64_t numOfInsertReqs = 0; - int64_t numOfInsertSuccessReqs = 0; - int64_t numOfBatchInsertReqs = 0; - int64_t numOfBatchInsertSuccessReqs = 0; - - pInfo->pVloads = pLoads; - if (pLoads == NULL) return; + pInfo->pVloads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoad)); + if (pInfo->pVloads == NULL) return; taosRLockLatch(&pMgmt->latch); @@ -388,28 +383,9 @@ void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo) { SVnodeObj *pVnode = *ppVnode; SVnodeLoad vload = {0}; vnodeGetLoad(pVnode->pImpl, &vload); - taosArrayPush(pLoads, &vload); - - numOfSelectReqs += vload.numOfSelectReqs; - numOfInsertReqs += vload.numOfInsertReqs; - numOfInsertSuccessReqs += vload.numOfInsertSuccessReqs; - numOfBatchInsertReqs += vload.numOfBatchInsertReqs; - numOfBatchInsertSuccessReqs += vload.numOfBatchInsertSuccessReqs; - totalVnodes++; - if (vload.role == TAOS_SYNC_STATE_LEADER) masterNum++; - + taosArrayPush(pInfo->pVloads, &vload); pIter = taosHashIterate(pMgmt->hash, pIter); } taosRUnLockLatch(&pMgmt->latch); - - taosWLockLatch(&pMgmt->latch); - pStat->totalVnodes = totalVnodes; - pStat->masterNum = masterNum; - pStat->numOfSelectReqs = numOfSelectReqs; - pStat->numOfInsertReqs = numOfInsertReqs; - pStat->numOfInsertSuccessReqs = numOfInsertSuccessReqs; - pStat->numOfBatchInsertReqs = numOfBatchInsertReqs; - pStat->numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs; - taosWUnLockLatch(&pMgmt->latch); } \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 206f53d36b9cd95c8afa9c51897db6369064c685..41f0c276922c63e4da6128c3d09d09cf725e0c19 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -14,9 +14,11 @@ */ #define _DEFAULT_SOURCE + +#include "vmInt.h" + #include "sync.h" #include "syncTools.h" -#include "vmInt.h" static inline void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, @@ -44,18 +46,9 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { case TDMT_DND_CREATE_VNODE: code = vmProcessCreateVnodeReq(pMgmt, pMsg); break; - case TDMT_DND_ALTER_VNODE: - code = vmProcessAlterVnodeReq(pMgmt, pMsg); - break; case TDMT_DND_DROP_VNODE: code = vmProcessDropVnodeReq(pMgmt, pMsg); break; - case TDMT_DND_SYNC_VNODE: - code = vmProcessSyncVnodeReq(pMgmt, pMsg); - break; - case TDMT_DND_COMPACT_VNODE: - code = vmProcessCompactVnodeReq(pMgmt, pMsg); - break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; dError("msg:%p, not processed in vnode-mgmt queue", pMsg); diff --git a/source/dnode/mgmt/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp index 9b4c4798653858ad22beb7ed637da3b6352674f4..cc32f047afcd1baf9e3d0857fe68248187a29e0e 100644 --- a/source/dnode/mgmt/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -89,7 +89,6 @@ int32_t Testbase::SendShowReq(int8_t showType, const char *tb, const char* db) { } SRetrieveTableReq retrieveReq = {0}; - retrieveReq.type = showType; strcpy(retrieveReq.db, db); strcpy(retrieveReq.tb, tb); diff --git a/source/dnode/mgmt/test/vnode/vnode.cpp b/source/dnode/mgmt/test/vnode/vnode.cpp index 0daea3f66629e01a81780acca0af89749b216b9f..51b7d6818fdce72103f830eb4a9ba57c8b335197 100644 --- a/source/dnode/mgmt/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/test/vnode/vnode.cpp @@ -47,7 +47,7 @@ TEST_F(DndTestVnode, 01_Create_Vnode) { createReq.precision = 0; createReq.compression = 2; createReq.replica = 1; - createReq.quorum = 1; + createReq.strict = 1; createReq.update = 0; createReq.cacheLastRow = 0; createReq.selfIndex = 0; @@ -94,7 +94,7 @@ TEST_F(DndTestVnode, 02_Alter_Vnode) { alterReq.precision = 0; alterReq.compression = 2; alterReq.replica = 1; - alterReq.quorum = 1; + alterReq.strict = 1; alterReq.update = 0; alterReq.cacheLastRow = 0; alterReq.selfIndex = 0; @@ -108,7 +108,7 @@ TEST_F(DndTestVnode, 02_Alter_Vnode) { void* pReq = rpcMallocCont(contLen); tSerializeSCreateVnodeReq(pReq, contLen, &alterReq); - SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_VNODE, pReq, contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_VND_ALTER_VNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); } diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndAuth.h index 7dcb518f9b0f9dcf84c27c6ca6639a7dba151d90..ae7663a775b61ece77565f30dd75dfa215a8a866 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndAuth.h @@ -33,7 +33,7 @@ int32_t mndCheckNodeAuth(SUserObj *pOperUser); int32_t mndCheckFuncAuth(SUserObj *pOperUser); int32_t mndCheckCreateDbAuth(SUserObj *pOperUser); -int32_t mndCheckAlterDropCompactSyncDbAuth(SUserObj *pOperUser, SDbObj *pDb); +int32_t mndCheckAlterDropCompactDbAuth(SUserObj *pOperUser, SDbObj *pDb); int32_t mndCheckUseDbAuth(SUserObj *pOperUser, SDbObj *pDb); int32_t mndCheckWriteAuth(SUserObj *pOperUser, SDbObj *pDb); diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index cc345379cfc1e59959a02585c892528b52c3a9da..a818f5d7c3ed4b29c41bcfe5fcfbc79f7126d032 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -22,27 +22,30 @@ extern "C" { #endif - enum { - MQ_CONSUMER_STATUS__INIT = 1, - MQ_CONSUMER_STATUS__IDLE, - MQ_CONSUMER_STATUS__ACTIVE, + // MQ_CONSUMER_STATUS__INIT = 1, + MQ_CONSUMER_STATUS__MODIFY = 1, + MQ_CONSUMER_STATUS__MODIFY_IN_REB, + // MQ_CONSUMER_STATUS__IDLE, + MQ_CONSUMER_STATUS__READY, MQ_CONSUMER_STATUS__LOST, - MQ_CONSUMER_STATUS__MODIFY + MQ_CONSUMER_STATUS__LOST_IN_REB, + MQ_CONSUMER_STATUS__LOST_REBD, }; - int32_t mndInitConsumer(SMnode *pMnode); void mndCleanupConsumer(SMnode *pMnode); SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId); void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer); -SMqConsumerObj* mndCreateConsumer(int64_t consumerId, const char* cgroup); +SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup); SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer); SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw); +int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndDb.h b/source/dnode/mnode/impl/inc/mndDb.h index 146c6e2523c441ed94444f22e6f6e94d6eff5171..38a5ecd273c588ca842477b4024e743c1a7cc802 100644 --- a/source/dnode/mnode/impl/inc/mndDb.h +++ b/source/dnode/mnode/impl/inc/mndDb.h @@ -27,9 +27,10 @@ void mndCleanupDb(SMnode *pMnode); SDbObj *mndAcquireDb(SMnode *pMnode, const char *db); void mndReleaseDb(SMnode *pMnode, SDbObj *pDb); int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen); -char *mndGetDbStr(char *src); int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUseDbReq *pReq); +const char *mndGetDbStr(const char *src); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 4a6455c0eddfee061735fd15e36376c831cacd7f..59a00f0eaf3c012ea62ce1259ab916ddd5929351 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -88,6 +88,7 @@ typedef enum { TRN_TYPE_CREATE_STREAM = 1019, TRN_TYPE_DROP_STREAM = 1020, TRN_TYPE_ALTER_STREAM = 1021, + TRN_TYPE_CONSUMER_LOST = 1022, TRN_TYPE_BASIC_SCOPE_END, TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_CREATE_DNODE = 2001, @@ -265,11 +266,12 @@ typedef struct { int8_t precision; int8_t compression; int8_t replications; - int8_t quorum; + int8_t strict; int8_t update; int8_t cacheLastRow; int8_t streamMode; int8_t singleSTable; + int8_t hashMethod; // default is 1 int32_t numOfRetensions; SArray* pRetensions; } SDbCfg; @@ -283,7 +285,6 @@ typedef struct { int64_t uid; int32_t cfgVersion; int32_t vgVersion; - int8_t hashMethod; // default is 1 SDbCfg cfg; SRWLatch lock; } SDbObj; @@ -355,10 +356,14 @@ typedef struct { int32_t numOfTags; int32_t numOfSmas; int32_t commentLen; + int32_t ast1Len; + int32_t ast2Len; SSchema* pColumns; SSchema* pTags; SSchema* pSmas; char* comment; + char* pAst1; + char* pAst2; SRWLatch lock; } SStbObj; @@ -386,7 +391,6 @@ typedef struct { int16_t numOfColumns; int32_t rowSize; int32_t numOfRows; - int32_t payloadLen; void* pIter; SMnode* pMnode; STableMetaRsp* pMeta; @@ -413,82 +417,6 @@ typedef struct { char payload[]; } SSysTableRetrieveObj; -typedef struct { - int32_t vgId; // -1 for unassigned - int32_t status; - int32_t epoch; - SEpSet epSet; - int64_t oldConsumerId; - int64_t consumerId; // -1 for unassigned - char* qmsg; -} SMqConsumerEp; - -static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp* pConsumerEp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId); - tlen += taosEncodeFixedI32(buf, pConsumerEp->status); - tlen += taosEncodeFixedI32(buf, pConsumerEp->epoch); - tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet); - tlen += taosEncodeFixedI64(buf, pConsumerEp->oldConsumerId); - tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); - tlen += taosEncodeString(buf, pConsumerEp->qmsg); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { - buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId); - buf = taosDecodeFixedI32(buf, &pConsumerEp->status); - buf = taosDecodeFixedI32(buf, &pConsumerEp->epoch); - buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet); - buf = taosDecodeFixedI64(buf, &pConsumerEp->oldConsumerId); - buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); - buf = taosDecodeString(buf, &pConsumerEp->qmsg); - return buf; -} - -static FORCE_INLINE void tDeleteSMqConsumerEp(SMqConsumerEp* pConsumerEp) { - if (pConsumerEp) { - taosMemoryFreeClear(pConsumerEp->qmsg); - } -} - -typedef struct { - int64_t consumerId; - SArray* vgInfo; // SArray -} SMqSubConsumer; - -static FORCE_INLINE int32_t tEncodeSMqSubConsumer(void** buf, const SMqSubConsumer* pConsumer) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); - int32_t sz = taosArrayGetSize(pConsumer->vgInfo); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp* pCEp = taosArrayGet(pConsumer->vgInfo, i); - tlen += tEncodeSMqConsumerEp(buf, pCEp); - } - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqSubConsumer(void** buf, SMqSubConsumer* pConsumer) { - int32_t sz; - buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); - buf = taosDecodeFixedI32(buf, &sz); - pConsumer->vgInfo = taosArrayInit(sz, sizeof(SMqConsumerEp)); - for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp consumerEp; - buf = tDecodeSMqConsumerEp(buf, &consumerEp); - taosArrayPush(pConsumer->vgInfo, &consumerEp); - } - return buf; -} - -static FORCE_INLINE void tDeleteSMqSubConsumer(SMqSubConsumer* pSubConsumer) { - if (pSubConsumer->vgInfo) { - taosArrayDestroyEx(pSubConsumer->vgInfo, (void (*)(void*))tDeleteSMqConsumerEp); - pSubConsumer->vgInfo = NULL; - } -} - typedef struct { char key[TSDB_PARTITION_KEY_LEN]; int64_t offset; @@ -508,144 +436,20 @@ static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset) } typedef struct { - char key[TSDB_SUBSCRIBE_KEY_LEN]; - int32_t status; - int32_t vgNum; - SArray* consumers; // SArray - SArray* lostConsumers; // SArray - SArray* unassignedVg; // SArray -} SMqSubscribeObj; - -static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { - SMqSubscribeObj* pSub = taosMemoryCalloc(1, sizeof(SMqSubscribeObj)); - if (pSub == NULL) { - return NULL; - } - - pSub->consumers = taosArrayInit(0, sizeof(SMqSubConsumer)); - if (pSub->consumers == NULL) { - goto _err; - } - - pSub->lostConsumers = taosArrayInit(0, sizeof(SMqSubConsumer)); - if (pSub->lostConsumers == NULL) { - goto _err; - } - - pSub->unassignedVg = taosArrayInit(0, sizeof(SMqConsumerEp)); - if (pSub->unassignedVg == NULL) { - goto _err; - } - - pSub->key[0] = 0; - pSub->vgNum = 0; - pSub->status = 0; - - return pSub; - -_err: - taosMemoryFreeClear(pSub->consumers); - taosMemoryFreeClear(pSub->lostConsumers); - taosMemoryFreeClear(pSub->unassignedVg); - taosMemoryFreeClear(pSub); - return NULL; -} - -static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pSub->key); - tlen += taosEncodeFixedI32(buf, pSub->vgNum); - tlen += taosEncodeFixedI32(buf, pSub->status); - int32_t sz; - - sz = taosArrayGetSize(pSub->consumers); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqSubConsumer* pSubConsumer = taosArrayGet(pSub->consumers, i); - tlen += tEncodeSMqSubConsumer(buf, pSubConsumer); - } - - sz = taosArrayGetSize(pSub->lostConsumers); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqSubConsumer* pSubConsumer = taosArrayGet(pSub->lostConsumers, i); - tlen += tEncodeSMqSubConsumer(buf, pSubConsumer); - } - - sz = taosArrayGetSize(pSub->unassignedVg); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp* pCEp = taosArrayGet(pSub->unassignedVg, i); - tlen += tEncodeSMqConsumerEp(buf, pCEp); - } - - return tlen; -} - -static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) { - buf = taosDecodeStringTo(buf, pSub->key); - buf = taosDecodeFixedI32(buf, &pSub->vgNum); - buf = taosDecodeFixedI32(buf, &pSub->status); - - int32_t sz; - - buf = taosDecodeFixedI32(buf, &sz); - pSub->consumers = taosArrayInit(sz, sizeof(SMqSubConsumer)); - if (pSub->consumers == NULL) { - return NULL; - } - for (int32_t i = 0; i < sz; i++) { - SMqSubConsumer subConsumer = {0}; - buf = tDecodeSMqSubConsumer(buf, &subConsumer); - taosArrayPush(pSub->consumers, &subConsumer); - } - - buf = taosDecodeFixedI32(buf, &sz); - pSub->lostConsumers = taosArrayInit(sz, sizeof(SMqSubConsumer)); - if (pSub->lostConsumers == NULL) { - return NULL; - } - for (int32_t i = 0; i < sz; i++) { - SMqSubConsumer subConsumer = {0}; - buf = tDecodeSMqSubConsumer(buf, &subConsumer); - taosArrayPush(pSub->lostConsumers, &subConsumer); - } - - buf = taosDecodeFixedI32(buf, &sz); - pSub->unassignedVg = taosArrayInit(sz, sizeof(SMqConsumerEp)); - if (pSub->unassignedVg == NULL) { - return NULL; - } - for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp consumerEp = {0}; - buf = tDecodeSMqConsumerEp(buf, &consumerEp); - taosArrayPush(pSub->unassignedVg, &consumerEp); - } - return buf; -} - -static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) { - if (pSub->consumers) { - // taosArrayDestroyEx(pSub->consumers, (void (*)(void*))tDeleteSMqSubConsumer); - // taosArrayDestroy(pSub->consumers); - pSub->consumers = NULL; - } - - if (pSub->unassignedVg) { - // taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp); - // taosArrayDestroy(pSub->unassignedVg); - pSub->unassignedVg = NULL; - } -} - -typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createTime; - int64_t updateTime; - int64_t uid; + char name[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + int64_t uid; + // TODO: use subDbUid int64_t dbUid; + int64_t subDbUid; int32_t version; + int8_t subType; // db or table + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + int8_t withTagSchema; SRWLatch lock; int32_t sqlLen; int32_t astLen; @@ -655,76 +459,115 @@ typedef struct { SSchemaWrapper schema; } SMqTopicObj; +enum { + CONSUMER_UPDATE__TOUCH = 1, + CONSUMER_UPDATE__ADD, + CONSUMER_UPDATE__REMOVE, + CONSUMER_UPDATE__LOST, + CONSUMER_UPDATE__MODIFY, +}; + typedef struct { - int64_t consumerId; - int64_t connId; - SRWLatch lock; - char cgroup[TSDB_CGROUP_LEN]; - SArray* currentTopics; // SArray - SArray* recentRemovedTopics; // SArray - int32_t epoch; - // stat - int64_t pollCnt; - // status + int64_t consumerId; + char cgroup[TSDB_CGROUP_LEN]; + int8_t updateType; // used only for update + int32_t epoch; int32_t status; - // heartbeat from the consumer reset hbStatus to 0 - // each checkConsumerAlive msg add hbStatus by 1 - // if checkConsumerAlive > CONSUMER_REBALANCE_CNT, mask to lost + // hbStatus is not applicable to serialization int32_t hbStatus; + // lock is used for topics update + SRWLatch lock; + SArray* currentTopics; // SArray + SArray* rebNewTopics; // SArray + SArray* rebRemovedTopics; // SArray } SMqConsumerObj; -static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer) { - int32_t sz; - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); - tlen += taosEncodeFixedI64(buf, pConsumer->connId); - tlen += taosEncodeFixedI32(buf, pConsumer->epoch); - tlen += taosEncodeFixedI64(buf, pConsumer->pollCnt); - tlen += taosEncodeFixedI32(buf, pConsumer->status); - tlen += taosEncodeString(buf, pConsumer->cgroup); - - sz = taosArrayGetSize(pConsumer->currentTopics); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - char* topic = taosArrayGetP(pConsumer->currentTopics, i); - tlen += taosEncodeString(buf, topic); - } - - sz = taosArrayGetSize(pConsumer->recentRemovedTopics); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - char* topic = taosArrayGetP(pConsumer->recentRemovedTopics, i); - tlen += taosEncodeString(buf, topic); - } - return tlen; -} +SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]); +void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer); +int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer); +void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer); -static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pConsumer) { - int32_t sz; - buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); - buf = taosDecodeFixedI64(buf, &pConsumer->connId); - buf = taosDecodeFixedI32(buf, &pConsumer->epoch); - buf = taosDecodeFixedI64(buf, &pConsumer->pollCnt); - buf = taosDecodeFixedI32(buf, &pConsumer->status); - buf = taosDecodeStringTo(buf, pConsumer->cgroup); - - buf = taosDecodeFixedI32(buf, &sz); - pConsumer->currentTopics = taosArrayInit(sz, sizeof(SMqConsumerObj)); - for (int32_t i = 0; i < sz; i++) { - char* topic; - buf = taosDecodeString(buf, &topic); - taosArrayPush(pConsumer->currentTopics, &topic); - } - - buf = taosDecodeFixedI32(buf, &sz); - pConsumer->recentRemovedTopics = taosArrayInit(sz, sizeof(SMqConsumerObj)); - for (int32_t i = 0; i < sz; i++) { - char* topic; - buf = taosDecodeString(buf, &topic); - taosArrayPush(pConsumer->recentRemovedTopics, &topic); - } - return buf; -} +typedef struct { + int32_t vgId; + char* qmsg; + SEpSet epSet; +} SMqVgEp; + +SMqVgEp* tCloneSMqVgEp(const SMqVgEp* pVgEp); +void tDeleteSMqVgEp(SMqVgEp* pVgEp); +int32_t tEncodeSMqVgEp(void** buf, const SMqVgEp* pVgEp); +void* tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp); + +typedef struct { + int64_t consumerId; // -1 for unassigned + SArray* vgs; // SArray +} SMqConsumerEpInSub; + +SMqConsumerEpInSub* tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub* pEpInSub); +void tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub* pEpInSub); +int32_t tEncodeSMqConsumerEpInSub(void** buf, const SMqConsumerEpInSub* pEpInSub); +void* tDecodeSMqConsumerEpInSub(const void* buf, SMqConsumerEpInSub* pEpInSub); + +typedef struct { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + SRWLatch lock; + int32_t vgNum; + int8_t subType; + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + int8_t withTagSchema; + SHashObj* consumerHash; // consumerId -> SMqConsumerEpInSub + // TODO put -1 into unassignVgs + // SArray* unassignedVgs; +} SMqSubscribeObj; + +SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]); +SMqSubscribeObj* tCloneSubscribeObj(const SMqSubscribeObj* pSub); +void tDeleteSubscribeObj(SMqSubscribeObj* pSub); +int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub); +void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub); + +typedef struct { + int32_t epoch; + SArray* consumers; // SArray +} SMqSubActionLogEntry; + +SMqSubActionLogEntry* tCloneSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry); +void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry); +int32_t tEncodeSMqSubActionLogEntry(void** buf, const SMqSubActionLogEntry* pEntry); +void* tDecodeSMqSubActionLogEntry(const void* buf, SMqSubActionLogEntry* pEntry); + +typedef struct { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + SArray* logs; // SArray +} SMqSubActionLogObj; + +SMqSubActionLogObj* tCloneSMqSubActionLogObj(SMqSubActionLogObj* pLog); +void tDeleteSMqSubActionLogObj(SMqSubActionLogObj* pLog); +int32_t tEncodeSMqSubActionLogObj(void** buf, const SMqSubActionLogObj* pLog); +void* tDecodeSMqSubActionLogObj(const void* buf, SMqSubActionLogObj* pLog); + +typedef struct { + const SMqSubscribeObj* pOldSub; + const SMqTopicObj* pTopic; + const SMqRebSubscribe* pRebInfo; +} SMqRebInputObj; + +typedef struct { + int64_t oldConsumerId; + int64_t newConsumerId; + SMqVgEp* pVgEp; +} SMqRebOutputVg; + +typedef struct { + SArray* rebVgs; // SArray + SArray* newConsumers; // SArray + SArray* removedConsumers; // SArray + SArray* touchedConsumers; // SArray + SMqSubscribeObj* pSub; + SMqSubActionLogEntry* pLogEntry; +} SMqRebOutputObj; typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 7b67308876a661252c2c3c0884307cfb6e8fba33..b9ec1905e6875baaf379c83903795af777e0dcf6 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -47,21 +47,6 @@ typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); - -typedef struct SMnodeLoad { - int64_t numOfDnode; - int64_t numOfMnode; - int64_t numOfVgroup; - int64_t numOfDatabase; - int64_t numOfSuperTable; - int64_t numOfChildTable; - int64_t numOfNormalTable; - int64_t numOfColumn; - int64_t totalPoints; - int64_t totalStorage; - int64_t compStorage; -} SMnodeLoad; - typedef struct SQWorkerMgmt SQHandle; typedef struct { @@ -82,7 +67,6 @@ typedef struct { } SProfileMgmt; typedef struct { - bool enable; SRWLatch lock; char email[TSDB_FQDN_LEN]; } STelemMgmt; @@ -129,7 +113,6 @@ struct SMnode { void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); int64_t mndGenerateUid(char *name, int32_t len); -void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndMnode.h b/source/dnode/mnode/impl/inc/mndMnode.h index 8041cc8fefe44d4c696d4ab33d82de57c0164f0f..a5cdfa1061034c25f2162ffe1812ea3ee235bf36 100644 --- a/source/dnode/mnode/impl/inc/mndMnode.h +++ b/source/dnode/mnode/impl/inc/mndMnode.h @@ -22,14 +22,13 @@ extern "C" { #endif -int32_t mndInitMnode(SMnode *pMnode); -void mndCleanupMnode(SMnode *pMnode); -SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId); -void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj); -bool mndIsMnode(SMnode *pMnode, int32_t dnodeId); -void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet); -void mndUpdateMnodeRole(SMnode *pMnode); -const char *mndGetRoleStr(int32_t role); +int32_t mndInitMnode(SMnode *pMnode); +void mndCleanupMnode(SMnode *pMnode); +SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId); +void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj); +bool mndIsMnode(SMnode *pMnode, int32_t dnodeId); +void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet); +void mndUpdateMnodeRole(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndOffset.h b/source/dnode/mnode/impl/inc/mndOffset.h index e18eec973f87129570983ad4724df115c6e18276..556bfe6a53f3fc9da48d0c471c35215868a468e2 100644 --- a/source/dnode/mnode/impl/inc/mndOffset.h +++ b/source/dnode/mnode/impl/inc/mndOffset.h @@ -31,7 +31,7 @@ void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset); SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset); SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw); -int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs); +int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs); static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) { return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName); diff --git a/source/dnode/mnode/impl/inc/mndScheduler.h b/source/dnode/mnode/impl/inc/mndScheduler.h index 42951beca2e414611543d92e08b86d19f6247636..33af040915688fd83c4a82af3c89047be5d20dae 100644 --- a/source/dnode/mnode/impl/inc/mndScheduler.h +++ b/source/dnode/mnode/impl/inc/mndScheduler.h @@ -29,6 +29,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream); +int32_t mndConvertRSmaTask(const char* ast, int8_t triggerType, int64_t watermark, char** pStr, int32_t* pLen); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndSubscribe.h b/source/dnode/mnode/impl/inc/mndSubscribe.h index 3f897067a20b31bfae52f812271854baa0b811cd..7915006f274ce8db1a0e0ae4607da4deb0effe99 100644 --- a/source/dnode/mnode/impl/inc/mndSubscribe.h +++ b/source/dnode/mnode/impl/inc/mndSubscribe.h @@ -26,9 +26,11 @@ int32_t mndInitSubscribe(SMnode *pMnode); void mndCleanupSubscribe(SMnode *pMnode); SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *CGroup, const char *topicName); -SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char* key); +SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key); void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub); +int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index fc94ec364590872d87577211e8e8f8780673f643..696d8502669184ace0d00c2ac86956a53a8f2966 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -146,7 +146,7 @@ int32_t mndCheckFuncAuth(SUserObj *pOperUser) { int32_t mndCheckCreateDbAuth(SUserObj *pOperUser) { return 0; } -int32_t mndCheckAlterDropCompactSyncDbAuth(SUserObj *pOperUser, SDbObj *pDb) { +int32_t mndCheckAlterDropCompactDbAuth(SUserObj *pOperUser, SDbObj *pDb) { if (pOperUser->superUser || strcmp(pOperUser->user, pDb->createUser) == 0) { return 0; } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index d2a3c381351f7e2718d2c83466240fb0e4301fbe..7f5df5d356914eeedb3f1b4654d26faade782cc3 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -19,8 +19,10 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndOffset.h" #include "mndShow.h" #include "mndStb.h" +#include "mndSubscribe.h" #include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" @@ -28,9 +30,13 @@ #include "tcompare.h" #include "tname.h" -#define MND_CONSUMER_VER_NUMBER 1 +#define MND_CONSUMER_VER_NUMBER 1 #define MND_CONSUMER_RESERVE_SIZE 64 +#define MND_CONSUMER_LOST_HB_CNT 3 + +static int8_t mqInRebFlag = 0; + static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer); @@ -38,6 +44,11 @@ static int32_t mndProcessConsumerMetaMsg(SNodeMsg *pMsg); static int32_t mndRetrieveConsumer(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); +static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg); +static int32_t mndProcessAskEpReq(SNodeMsg *pMsg); +static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg); +static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg); + int32_t mndInitConsumer(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_CONSUMER, .keyType = SDB_KEY_INT64, @@ -47,25 +58,397 @@ int32_t mndInitConsumer(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndConsumerActionUpdate, .deleteFp = (SdbDeleteFp)mndConsumerActionDelete}; + mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); + mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessAskEpReq); + mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); + mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg); return sdbSetTable(pMnode->pSdb, table); } void mndCleanupConsumer(SMnode *pMnode) {} -SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup) { - SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); +static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; + SMqConsumerLostMsg *pLostMsg = pMsg->rpcMsg.pCont; + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId); + ASSERT(pConsumer); + + SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup); + pConsumerNew->updateType = CONSUMER_UPDATE__LOST; + + mndReleaseConsumer(pMnode, pConsumer); + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CONSUMER_LOST, &pMsg->rpcMsg); + if (pTrans == NULL) goto FAIL; + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL; + if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL; + + mndTransDrop(pTrans); + return 0; +FAIL: + // TODO delete consumer + mndTransDrop(pTrans); + return -1; +} + +static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { + SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); + if (pRebSub == NULL) { + pRebSub = tNewSMqRebSubscribe(key); + if (pRebSub == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebSubscribe)); + } + return pRebSub; +} + +static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; + SSdb *pSdb = pMnode->pSdb; + SMqConsumerObj *pConsumer; + void *pIter = NULL; + + // rebalance cannot be parallel + int8_t old = atomic_val_compare_exchange_8(&mqInRebFlag, 0, 1); + if (old != 0) { + mInfo("mq rebalance already in progress, do nothing"); + return 0; + } + + SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg)); + pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); + // TODO set cleanfp + pRebMsg->mqInReb = &mqInRebFlag; + + // iterate all consumers, find all modification + while (1) { + pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); + if (pIter == NULL) break; + + int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); + int32_t status = atomic_load_32(&pConsumer->status); + if (status == MQ_CONSUMER_STATUS__READY && hbStatus > MND_CONSUMER_LOST_HB_CNT) { + SMqConsumerLostMsg *pLostMsg = rpcMallocCont(sizeof(SMqConsumerLostMsg)); + + pLostMsg->consumerId = pConsumer->consumerId; + SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg)); + pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_LOST; + pRpcMsg->pCont = pLostMsg; + pRpcMsg->contLen = sizeof(SMqConsumerLostMsg); + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg); + } + if (status == MQ_CONSUMER_STATUS__LOST_REBD || status == MQ_CONSUMER_STATUS__READY) { + // do nothing + } else if (status == MQ_CONSUMER_STATUS__LOST) { + taosRLockLatch(&pConsumer->lock); + int32_t topicNum = taosArrayGetSize(pConsumer->currentTopics); + for (int32_t i = 0; i < topicNum; i++) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + char *removedTopic = taosArrayGetP(pConsumer->currentTopics, i); + mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); + SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); + taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); + } + taosRUnLockLatch(&pConsumer->lock); + } else if (status == MQ_CONSUMER_STATUS__MODIFY) { + taosRLockLatch(&pConsumer->lock); + int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics); + for (int32_t i = 0; i < newTopicNum; i++) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + char *newTopic = taosArrayGetP(pConsumer->rebNewTopics, i); + mndMakeSubscribeKey(key, pConsumer->cgroup, newTopic); + SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); + taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId); + } + + int32_t removedTopicNum = taosArrayGetSize(pConsumer->rebRemovedTopics); + for (int32_t i = 0; i < removedTopicNum; i++) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + char *removedTopic = taosArrayGetP(pConsumer->rebRemovedTopics, i); + mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); + SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); + taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); + } + taosRUnLockLatch(&pConsumer->lock); + } else { + // do nothing + } + + mndReleaseConsumer(pMnode, pConsumer); + } + + if (taosHashGetSize(pRebMsg->rebSubHash) != 0) { + mInfo("mq rebalance will be triggered"); + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_MQ_DO_REBALANCE, + .pCont = pRebMsg, + .contLen = sizeof(SMqDoRebalanceMsg), + }; + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + } else { + taosHashCleanup(pRebMsg->rebSubHash); + rpcFreeCont(pRebMsg); + mTrace("mq rebalance finished, no modification"); + atomic_store_8(&mqInRebFlag, 0); + } + return 0; +} + +static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; + SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; + SMqCMGetSubEpRsp rsp = {0}; + int64_t consumerId = be64toh(pReq->consumerId); + int32_t epoch = ntohl(pReq->epoch); + + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pNode, consumerId); if (pConsumer == NULL) { + terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; + return -1; + } + + ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); + /*int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus);*/ + atomic_store_32(&pConsumer->hbStatus, 0); + + // 1. check consumer status + int32_t status = atomic_load_32(&pConsumer->status); + + if (status == MQ_CONSUMER_STATUS__LOST) { + // TODO: recover consumer + } + + if (status != MQ_CONSUMER_STATUS__READY) { + terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; + return -1; + } + + int32_t serverEpoch = atomic_load_32(&pConsumer->epoch); + + // 2. check epoch, only send ep info when epoches do not match + if (epoch != serverEpoch) { + taosRLockLatch(&pConsumer->lock); + mInfo("process ask ep, consumer %ld(epoch %d), server epoch %d", consumerId, epoch, serverEpoch); + int32_t numOfTopics = taosArrayGetSize(pConsumer->currentTopics); + + rsp.topics = taosArrayInit(numOfTopics, sizeof(SMqSubTopicEp)); + if (rsp.topics == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosRUnLockLatch(&pConsumer->lock); + goto FAIL; + } + + // handle all topic subscribed by the consumer + for (int32_t i = 0; i < numOfTopics; i++) { + char *topic = taosArrayGetP(pConsumer->currentTopics, i); + SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); + + // txn guarantees pSub is created + ASSERT(pSub); + taosRLockLatch(&pSub->lock); + + SMqSubTopicEp topicEp = {0}; + strcpy(topicEp.topic, topic); + + // 2.1 fetch topic schema + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); + ASSERT(pTopic); + taosRLockLatch(&pTopic->lock); + topicEp.schema.nCols = pTopic->schema.nCols; + topicEp.schema.pSchema = taosMemoryCalloc(topicEp.schema.nCols, sizeof(SSchema)); + memcpy(topicEp.schema.pSchema, pTopic->schema.pSchema, topicEp.schema.nCols * sizeof(SSchema)); + taosRUnLockLatch(&pTopic->lock); + mndReleaseTopic(pMnode, pTopic); + + // 2.2 iterate all vg assigned to the consumer of that topic + SMqConsumerEpInSub *pEpInSub = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t)); + int32_t vgNum = taosArrayGetSize(pEpInSub->vgs); + + topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp)); + if (topicEp.vgs == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosRUnLockLatch(&pConsumer->lock); + goto FAIL; + } + + for (int32_t j = 0; j < vgNum; j++) { + SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, j); + char offsetKey[TSDB_PARTITION_KEY_LEN]; + mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId); + // 2.2.1 build vg ep + SMqSubVgEp vgEp = { + .epSet = pVgEp->epSet, + .vgId = pVgEp->vgId, + .offset = -1, + }; + + // 2.2.2 fetch vg offset + SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey); + if (pOffsetObj != NULL) { + vgEp.offset = atomic_load_64(&pOffsetObj->offset); + mndReleaseOffset(pMnode, pOffsetObj); + } + taosArrayPush(topicEp.vgs, &vgEp); + } + taosArrayPush(rsp.topics, &topicEp); + + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); + } + taosRUnLockLatch(&pConsumer->lock); + } + // encode rsp + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp); + void *buf = rpcMallocCont(tlen); + if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return -1; } + ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP; + ((SMqRspHead *)buf)->epoch = serverEpoch; + ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId; - pConsumer->recentRemovedTopics = taosArrayInit(1, sizeof(char *)); - pConsumer->epoch = 1; - pConsumer->consumerId = consumerId; - atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__INIT); - strcpy(pConsumer->cgroup, cgroup); - taosInitRWLatch(&pConsumer->lock); - return pConsumer; + void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); + + // release consumer and free memory + tDeleteSMqCMGetSubEpRsp(&rsp); + mndReleaseConsumer(pMnode, pConsumer); + + // send rsp + pMsg->pRsp = buf; + pMsg->rspLen = tlen; + return 0; +FAIL: + tDeleteSMqCMGetSubEpRsp(&rsp); + mndReleaseConsumer(pMnode, pConsumer); + return -1; +} + +int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer) { + SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; + char *msgStr = pMsg->rpcMsg.pCont; + SCMSubscribeReq subscribe = {0}; + tDeserializeSCMSubscribeReq(msgStr, &subscribe); + int64_t consumerId = subscribe.consumerId; + char *cgroup = subscribe.cgroup; + SMqConsumerObj *pConsumerOld = NULL; + SMqConsumerObj *pConsumerNew = NULL; + + int32_t code = -1; + SArray *newSub = subscribe.topicNames; + taosArraySortString(newSub, taosArrayCompareString); + + int32_t newTopicNum = taosArrayGetSize(newSub); + // check topic existance + for (int32_t i = 0; i < newTopicNum; i++) { + char *topic = taosArrayGetP(newSub, i); + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); + if (pTopic == NULL) { + terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; + goto SUBSCRIBE_OVER; + } + // TODO lock topic to prevent drop + mndReleaseTopic(pMnode, pTopic); + } + + pConsumerOld = mndAcquireConsumer(pMnode, consumerId); + if (pConsumerOld == NULL) { + pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); + pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; + /*pConsumerNew->waitingRebTopics = newSub;*/ + pConsumerNew->rebNewTopics = newSub; + subscribe.topicNames = NULL; + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg); + if (pTrans == NULL) goto SUBSCRIBE_OVER; + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER; + + } else { + /*taosRLockLatch(&pConsumerOld->lock);*/ + int32_t status = atomic_load_32(&pConsumerOld->status); + if (status != MQ_CONSUMER_STATUS__READY) { + terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; + goto SUBSCRIBE_OVER; + } + + pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); + if (pConsumerNew == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto SUBSCRIBE_OVER; + } + pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; + /*pConsumerOld->waitingRebTopics = newSub;*/ + + int32_t oldTopicNum = 0; + if (pConsumerOld->currentTopics) { + oldTopicNum = taosArrayGetSize(pConsumerOld->currentTopics); + } + + int32_t i = 0, j = 0; + while (i < oldTopicNum || j < newTopicNum) { + if (i >= oldTopicNum) { + char *newTopicCopy = strdup(taosArrayGetP(newSub, j)); + taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy); + j++; + continue; + } else if (j >= newTopicNum) { + char *oldTopicCopy = strdup(taosArrayGetP(pConsumerOld->currentTopics, i)); + taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy); + i++; + continue; + } else { + char *oldTopic = taosArrayGetP(pConsumerOld->currentTopics, i); + char *newTopic = taosArrayGetP(newSub, j); + int comp = compareLenPrefixedStr(oldTopic, newTopic); + if (comp == 0) { + i++; + j++; + continue; + } else if (comp < 0) { + char *oldTopicCopy = strdup(oldTopic); + taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy); + i++; + continue; + } else { + char *newTopicCopy = strdup(newTopic); + taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy); + j++; + continue; + } + } + } + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg); + if (pTrans == NULL) goto SUBSCRIBE_OVER; + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER; + } + + code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +SUBSCRIBE_OVER: + if (pConsumerOld) { + /*taosRUnLockLatch(&pConsumerOld->lock);*/ + mndReleaseConsumer(pMnode, pConsumerOld); + } + if (pConsumerNew) { + tDeleteSMqConsumerObj(pConsumerNew); + } + // TODO: replace with destroy subscribe msg + if (subscribe.topicNames) taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree); + return code; } SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { @@ -154,15 +537,141 @@ static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) { static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { mTrace("consumer:%" PRId64 ", perform delete action", pConsumer->consumerId); + tDeleteSMqConsumerObj(pConsumer); return 0; } static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId); - /*taosWLockLatch(&pOldConsumer->lock);*/ - atomic_add_fetch_32(&pOldConsumer->epoch, 1); - /*taosWUnLockLatch(&pOldConsumer->lock);*/ + taosWLockLatch(&pOldConsumer->lock); + + if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) { + ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); + ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); + SArray *tmp = pOldConsumer->rebNewTopics; + pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics; + pNewConsumer->rebNewTopics = tmp; + + tmp = pOldConsumer->rebRemovedTopics; + pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics; + pNewConsumer->rebRemovedTopics = tmp; + + pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; + } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { + int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); + pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *)); + for (int32_t i = 0; i < sz; i++) { + char *topic = strdup(taosArrayGetP(pOldConsumer->currentTopics, i)); + taosArrayPush(pNewConsumer->rebRemovedTopics, &topic); + } + pOldConsumer->status = MQ_CONSUMER_STATUS__LOST; + } else if (pNewConsumer->updateType == CONSUMER_UPDATE__TOUCH) { + atomic_add_fetch_32(&pOldConsumer->epoch, 1); + } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) { + ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1); + ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0); + + char *addedTopic = strdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); + // not exist in current topic +#if 1 + for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->currentTopics); i++) { + char *topic = taosArrayGetP(pOldConsumer->currentTopics, i); + ASSERT(strcmp(topic, addedTopic) != 0); + } +#endif + + // remove from new topic + for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) { + char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i); + if (strcmp(addedTopic, topic) == 0) { + taosArrayRemove(pOldConsumer->rebNewTopics, i); + taosMemoryFree(topic); + break; + } + } + + // add to current topic + taosArrayPush(pOldConsumer->currentTopics, &addedTopic); + taosArraySortString(pOldConsumer->currentTopics, taosArrayCompareString); + // set status + if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) { + if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY || + pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + pOldConsumer->status = MQ_CONSUMER_STATUS__READY; + } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB || + pOldConsumer->status == MQ_CONSUMER_STATUS__LOST) { + pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; + } + } else { + if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY || + pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB; + } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST || + pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB) { + pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB; + } + } + atomic_add_fetch_32(&pOldConsumer->epoch, 1); + } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) { + ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0); + ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1); + char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0); + + // not exist in new topic +#if 1 + for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) { + char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i); + ASSERT(strcmp(topic, removedTopic) != 0); + } +#endif + + // remove from removed topic + for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebRemovedTopics); i++) { + char *topic = taosArrayGetP(pOldConsumer->rebRemovedTopics, i); + if (strcmp(removedTopic, topic) == 0) { + taosArrayRemove(pOldConsumer->rebRemovedTopics, i); + taosMemoryFree(topic); + break; + } + } + + // remove from current topic + int32_t i = 0; + int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); + for (i = 0; i < sz; i++) { + char *topic = taosArrayGetP(pOldConsumer->currentTopics, i); + if (strcmp(removedTopic, topic) == 0) { + taosArrayRemove(pOldConsumer->currentTopics, i); + taosMemoryFree(topic); + break; + } + } + // must find the topic + ASSERT(i < sz); + + // set status + if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) { + if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY || + pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + pOldConsumer->status = MQ_CONSUMER_STATUS__READY; + } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB || + pOldConsumer->status == MQ_CONSUMER_STATUS__LOST) { + pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; + } + } else { + if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY || + pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB; + } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST || + pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB) { + pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB; + } + } + atomic_add_fetch_32(&pOldConsumer->epoch, 1); + } + + taosWUnLockLatch(&pOldConsumer->lock); return 0; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 755e2828defb88e41d2c94e652b0e88aeb4085b7..dab587432e2d2188920e664a4d8b66efaa66f54e 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -18,14 +18,14 @@ #include "mndAuth.h" #include "mndDnode.h" #include "mndShow.h" +#include "mndSma.h" #include "mndStb.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" -#include "mndSma.h" -#define TSDB_DB_VER_NUMBER 1 -#define TSDB_DB_RESERVE_SIZE 64 +#define DB_VER_NUMBER 1 +#define DB_RESERVE_SIZE 64 static SSdbRaw *mndDbActionEncode(SDbObj *pDb); static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); @@ -36,9 +36,8 @@ static int32_t mndProcessCreateDbReq(SNodeMsg *pReq); static int32_t mndProcessAlterDbReq(SNodeMsg *pReq); static int32_t mndProcessDropDbReq(SNodeMsg *pReq); static int32_t mndProcessUseDbReq(SNodeMsg *pReq); -static int32_t mndProcessSyncDbReq(SNodeMsg *pReq); static int32_t mndProcessCompactDbReq(SNodeMsg *pReq); -static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rowsCapacity); +static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq); static int32_t mndProcessGetIndexReq(SNodeMsg *pReq); @@ -56,7 +55,6 @@ int32_t mndInitDb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_ALTER_DB, mndProcessAlterDbReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_DB, mndProcessDropDbReq); mndSetMsgHandle(pMnode, TDMT_MND_USE_DB, mndProcessUseDbReq); - mndSetMsgHandle(pMnode, TDMT_MND_SYNC_DB, mndProcessSyncDbReq); mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetIndexReq); @@ -72,54 +70,57 @@ void mndCleanupDb(SMnode *pMnode) {} static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, - sizeof(SDbObj) + pDb->cfg.numOfRetensions * sizeof(SRetention) + TSDB_DB_RESERVE_SIZE); - if (pRaw == NULL) goto DB_ENCODE_OVER; + int32_t size = sizeof(SDbObj) + pDb->cfg.numOfRetensions * sizeof(SRetention) + DB_RESERVE_SIZE; + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, DB_VER_NUMBER, size); + if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pDb->name, TSDB_DB_FNAME_LEN, DB_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pDb->acct, TSDB_USER_LEN, DB_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pDb->createUser, TSDB_USER_LEN, DB_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pDb->createdTime, DB_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pDb->updateTime, DB_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pDb->uid, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfgVersion, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->vgVersion, DB_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pDb->hashMethod, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfVgroups, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep0, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep1, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep2, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.minRows, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRows, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.commitTime, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.fsyncPeriod, DB_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pDb->cfg.walLevel, DB_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pDb->cfg.precision, DB_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compression, DB_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pDb->cfg.replications, DB_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pDb->cfg.quorum, DB_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pDb->cfg.update, DB_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLastRow, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, DB_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, pDb->name, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pDb->acct, TSDB_USER_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pDb->createUser, TSDB_USER_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pDb->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pDb->updateTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pDb->uid, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfgVersion, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->vgVersion, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfVgroups, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep0, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep1, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep2, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.minRows, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRows, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.commitTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.fsyncPeriod, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.ttl, _OVER) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.walLevel, _OVER) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.precision, _OVER) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compression, _OVER) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.replications, _OVER) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.strict, _OVER) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.update, _OVER) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLastRow, _OVER) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.streamMode, _OVER) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.singleSTable, _OVER) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.hashMethod, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, _OVER) for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { TASSERT(taosArrayGetSize(pDb->cfg.pRetensions) == pDb->cfg.numOfRetensions); SRetention *pRetension = taosArrayGet(pDb->cfg.pRetensions, i); - SDB_SET_INT32(pRaw, dataPos, pRetension->freq, DB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pRetension->keep, DB_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pRetension->freqUnit, DB_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pRetension->keepUnit, DB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pRetension->freq, _OVER) + SDB_SET_INT32(pRaw, dataPos, pRetension->keep, _OVER) + SDB_SET_INT8(pRaw, dataPos, pRetension->freqUnit, _OVER) + SDB_SET_INT8(pRaw, dataPos, pRetension->keepUnit, _OVER) } - SDB_SET_RESERVE(pRaw, dataPos, TSDB_DB_RESERVE_SIZE, DB_ENCODE_OVER) - SDB_SET_DATALEN(pRaw, dataPos, DB_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; -DB_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("db:%s, failed to encode to raw:%p since %s", pDb->name, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -134,68 +135,71 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto DB_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != TSDB_DB_VER_NUMBER) { + if (sver != DB_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto DB_DECODE_OVER; + goto _OVER; } SSdbRow *pRow = sdbAllocRow(sizeof(SDbObj)); - if (pRow == NULL) goto DB_DECODE_OVER; + if (pRow == NULL) goto _OVER; SDbObj *pDb = sdbGetRowObj(pRow); - if (pDb == NULL) goto DB_DECODE_OVER; + if (pDb == NULL) goto _OVER; int32_t dataPos = 0; - SDB_GET_BINARY(pRaw, dataPos, pDb->name, TSDB_DB_FNAME_LEN, DB_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pDb->acct, TSDB_USER_LEN, DB_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pDb->createUser, TSDB_USER_LEN, DB_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pDb->createdTime, DB_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pDb->updateTime, DB_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pDb->uid, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfgVersion, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->vgVersion, DB_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pDb->hashMethod, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfVgroups, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.cacheBlockSize, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.totalBlocks, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysPerFile, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep0, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep1, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep2, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.minRows, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.maxRows, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.commitTime, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.fsyncPeriod, DB_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.walLevel, DB_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.precision, DB_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.compression, DB_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.replications, DB_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.quorum, DB_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.update, DB_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.cacheLastRow, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfRetensions, DB_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, pDb->name, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pDb->acct, TSDB_USER_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pDb->createUser, TSDB_USER_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pDb->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pDb->updateTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pDb->uid, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfgVersion, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->vgVersion, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfVgroups, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.cacheBlockSize, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.totalBlocks, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysPerFile, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep0, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep1, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep2, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.minRows, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.maxRows, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.commitTime, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.fsyncPeriod, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.ttl, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.walLevel, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.precision, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.compression, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.replications, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.strict, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.update, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.cacheLastRow, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.streamMode, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.singleSTable, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.hashMethod, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfRetensions, _OVER) if (pDb->cfg.numOfRetensions > 0) { pDb->cfg.pRetensions = taosArrayInit(pDb->cfg.numOfRetensions, sizeof(SRetention)); - if (pDb->cfg.pRetensions == NULL) goto DB_DECODE_OVER; + if (pDb->cfg.pRetensions == NULL) goto _OVER; for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { SRetention retension = {0}; - SDB_GET_INT32(pRaw, dataPos, &retension.freq, DB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &retension.keep, DB_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &retension.freqUnit, DB_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &retension.keepUnit, DB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &retension.freq, _OVER) + SDB_GET_INT32(pRaw, dataPos, &retension.keep, _OVER) + SDB_GET_INT8(pRaw, dataPos, &retension.freqUnit, _OVER) + SDB_GET_INT8(pRaw, dataPos, &retension.keepUnit, _OVER) if (taosArrayPush(pDb->cfg.pRetensions, &retension) == NULL) { - goto DB_DECODE_OVER; + goto _OVER; } } } - SDB_GET_RESERVE(pRaw, dataPos, TSDB_DB_RESERVE_SIZE, DB_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) terrno = 0; -DB_DECODE_OVER: +_OVER: if (terrno != 0) { mError("db:%s, failed to decode from raw:%p since %s", pDb->name, pRaw, terrstr()); taosMemoryFreeClear(pRow); @@ -244,7 +248,7 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mndCheckDbName(char *dbName, SUserObj *pUser) { +static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { char *pos = strstr(dbName, TS_PATH_DELIMITER); if (pos == NULL) { terrno = TSDB_CODE_MND_INVALID_DB; @@ -271,23 +275,24 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1; if (pCfg->daysToKeep0 > pCfg->daysToKeep1) return -1; if (pCfg->daysToKeep1 > pCfg->daysToKeep2) return -1; - if (pCfg->minRows < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRows > TSDB_MAX_MIN_ROW_FBLOCK) return -1; - if (pCfg->maxRows < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRows > TSDB_MAX_MAX_ROW_FBLOCK) return -1; + if (pCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1; + if (pCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1; if (pCfg->minRows > pCfg->maxRows) return -1; if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) return -1; if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1; - if (pCfg->ttl < TSDB_MIN_DB_TTL_OPTION && pCfg->ttl != TSDB_DEFAULT_DB_TTL_OPTION) return -1; + if (pCfg->ttl < TSDB_MIN_DB_TTL) return -1; if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1; if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1; if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1; - if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) return -1; + if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1; if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1; - if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) return -1; - if (pCfg->quorum > pCfg->replications) return -1; + if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; + if (pCfg->strict > pCfg->replications) return -1; if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) return -1; if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; - if (pCfg->streamMode < TSDB_MIN_DB_STREAM_MODE || pCfg->streamMode > TSDB_MAX_DB_STREAM_MODE) return -1; - if (pCfg->singleSTable < TSDB_MIN_DB_SINGLE_STABLE_OPTION || pCfg->streamMode > TSDB_MAX_DB_SINGLE_STABLE_OPTION) return -1; + if (pCfg->streamMode < TSDB_DB_STREAM_MODE_OFF || pCfg->streamMode > TSDB_DB_STREAM_MODE_ON) return -1; + if (pCfg->singleSTable < TSDB_DB_SINGLE_STABLE_ON || pCfg->streamMode > TSDB_DB_SINGLE_STABLE_OFF) return -1; + if (pCfg->hashMethod != 1) return -1; return TSDB_CODE_SUCCESS; } @@ -299,20 +304,20 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->daysToKeep0 < 0) pCfg->daysToKeep0 = TSDB_DEFAULT_KEEP; if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep0; if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep1; - if (pCfg->minRows < 0) pCfg->minRows = TSDB_DEFAULT_MIN_ROW_FBLOCK; - if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAX_ROW_FBLOCK; + if (pCfg->minRows < 0) pCfg->minRows = TSDB_DEFAULT_MINROWS_FBLOCK; + if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAXROWS_FBLOCK; if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME; if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; - if (pCfg->ttl < 0) pCfg->ttl = TSDB_DEFAULT_DB_TTL_OPTION; + if (pCfg->ttl < 0) pCfg->ttl = TSDB_DEFAULT_DB_TTL; if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL; if (pCfg->precision < 0) pCfg->precision = TSDB_DEFAULT_PRECISION; if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL; - if (pCfg->replications < 0) pCfg->replications = TSDB_DEFAULT_DB_REPLICA_OPTION; - if (pCfg->quorum < 0) pCfg->quorum = TSDB_DEFAULT_DB_QUORUM_OPTION; - if (pCfg->update < 0) pCfg->update = TSDB_DEFAULT_DB_UPDATE_OPTION; + if (pCfg->replications < 0) pCfg->replications = TSDB_DEFAULT_DB_REPLICA; + if (pCfg->strict < 0) pCfg->strict = TSDB_DEFAULT_DB_STRICT; + if (pCfg->update < 0) pCfg->update = TSDB_DEFAULT_DB_UPDATE; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; if (pCfg->streamMode < 0) pCfg->streamMode = TSDB_DEFAULT_DB_STREAM_MODE; - if (pCfg->singleSTable < 0) pCfg->singleSTable = TSDB_DEFAULT_DB_SINGLE_STABLE_OPTION; + if (pCfg->singleSTable < 0) pCfg->singleSTable = TSDB_DEFAULT_DB_SINGLE_STABLE; if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0; } @@ -435,7 +440,6 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate dbObj.uid = mndGenerateUid(dbObj.name, TSDB_DB_FNAME_LEN); dbObj.cfgVersion = 1; dbObj.vgVersion = 1; - dbObj.hashMethod = 1; memcpy(dbObj.createUser, pUser->user, TSDB_USER_LEN); dbObj.cfg = (SDbCfg){ .numOfVgroups = pCreate->numOfVgroups, @@ -454,11 +458,12 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate .precision = pCreate->precision, .compression = pCreate->compression, .replications = pCreate->replications, - .quorum = pCreate->quorum, + .strict = pCreate->strict, .update = pCreate->update, .cacheLastRow = pCreate->cacheLastRow, .streamMode = pCreate->streamMode, .singleSTable = pCreate->singleSTable, + .hashMethod = 1, }; dbObj.cfg.numOfRetensions = pCreate->numOfRetensions; @@ -486,21 +491,21 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_DB, &pReq->rpcMsg); - if (pTrans == NULL) goto CREATE_DB_OVER; + if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); mndTransSetDbInfo(pTrans, &dbObj); - if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto CREATE_DB_OVER; - if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto CREATE_DB_OVER; - if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto CREATE_DB_OVER; - if (mndSetCreateDbRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) goto CREATE_DB_OVER; - if (mndSetCreateDbUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) goto CREATE_DB_OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto CREATE_DB_OVER; + if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; + if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; + if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; + if (mndSetCreateDbRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; + if (mndSetCreateDbUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; -CREATE_DB_OVER: +_OVER: taosMemoryFree(pVgroups); mndTransDrop(pTrans); return code; @@ -515,7 +520,7 @@ static int32_t mndProcessCreateDbReq(SNodeMsg *pReq) { if (tDeserializeSCreateDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto CREATE_DB_OVER; + goto _OVER; } mDebug("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups); @@ -525,28 +530,28 @@ static int32_t mndProcessCreateDbReq(SNodeMsg *pReq) { if (createReq.ignoreExist) { mDebug("db:%s, already exist, ignore exist is set", createReq.db); code = 0; - goto CREATE_DB_OVER; + goto _OVER; } else { terrno = TSDB_CODE_MND_DB_ALREADY_EXIST; - goto CREATE_DB_OVER; + goto _OVER; } } else if (terrno != TSDB_CODE_MND_DB_NOT_EXIST) { - goto CREATE_DB_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto CREATE_DB_OVER; + goto _OVER; } if (mndCheckCreateDbAuth(pUser) != 0) { - goto CREATE_DB_OVER; + goto _OVER; } code = mndCreateDb(pMnode, pReq, &createReq, pUser); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -CREATE_DB_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to create since %s", createReq.db, terrstr()); } @@ -591,8 +596,8 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = 0; } - if (pAlter->quorum >= 0 && pAlter->quorum != pDb->cfg.quorum) { - pDb->cfg.quorum = pAlter->quorum; + if (pAlter->strict >= 0 && pAlter->strict != pDb->cfg.strict) { + pDb->cfg.strict = pAlter->strict; terrno = 0; } @@ -601,10 +606,15 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = 0; } + if (pAlter->replications >= 0 && pAlter->replications != pDb->cfg.replications) { + pDb->cfg.replications = pAlter->replications; + terrno = 0; + } + return terrno; } -static int32_t mndSetUpdateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { +static int32_t mndSetAlterDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { SSdbRaw *pRedoRaw = mndDbActionEncode(pOld); if (pRedoRaw == NULL) return -1; if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; @@ -613,7 +623,7 @@ static int32_t mndSetUpdateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pO return 0; } -static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { +static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { SSdbRaw *pCommitRaw = mndDbActionEncode(pNew); if (pCommitRaw == NULL) return -1; if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; @@ -622,7 +632,65 @@ static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj * return 0; } -static int32_t mndBuildUpdateVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { + SAlterVnodeReq alterReq = {0}; + alterReq.vgVersion = pVgroup->version; + alterReq.totalBlocks = pDb->cfg.totalBlocks; + alterReq.daysToKeep0 = pDb->cfg.daysToKeep0; + alterReq.daysToKeep1 = pDb->cfg.daysToKeep1; + alterReq.daysToKeep2 = pDb->cfg.daysToKeep2; + alterReq.walLevel = pDb->cfg.walLevel; + alterReq.strict = pDb->cfg.strict; + alterReq.cacheLastRow = pDb->cfg.cacheLastRow; + alterReq.replica = pVgroup->replica; + alterReq.selfIndex = -1; + + for (int32_t v = 0; v < pVgroup->replica; ++v) { + SReplica *pReplica = &alterReq.replicas[v]; + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; + SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pVgidDnode == NULL) { + return NULL; + } + + pReplica->id = pVgidDnode->id; + pReplica->port = pVgidDnode->port; + memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); + mndReleaseDnode(pMnode, pVgidDnode); + + if (pDnode->id == pVgid->dnodeId) { + alterReq.selfIndex = v; + } + } + + if (alterReq.selfIndex == -1) { + terrno = TSDB_CODE_MND_APP_ERROR; + return NULL; + } + + int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); + if (contLen < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + contLen += +sizeof(SMsgHead); + + void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + SMsgHead *pHead = pReq; + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + tSerializeSAlterVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &alterReq); + *pContLen = contLen; + return pReq; +} + +static int32_t mndBuilAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { STransAction action = {0}; SVnodeGid *pVgid = pVgroup->vnodeGid + vn; @@ -633,12 +701,12 @@ static int32_t mndBuildUpdateVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj mndReleaseDnode(pMnode, pDnode); int32_t contLen = 0; - void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); if (pReq == NULL) return -1; action.pCont = pReq; action.contLen = contLen; - action.msgType = TDMT_DND_ALTER_VNODE; + action.msgType = TDMT_VND_ALTER_VNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); return -1; @@ -648,7 +716,7 @@ static int32_t mndBuildUpdateVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { +static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; @@ -658,7 +726,7 @@ static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj if (pIter == NULL) break; if (pVgroup->dbUid == pNew->uid) { - if (mndBuildUpdateVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { + if (mndBuilAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -671,17 +739,17 @@ static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndUpdateDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) { +static int32_t mndAlterDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_ALTER_DB, &pReq->rpcMsg); if (pTrans == NULL) goto UPDATE_DB_OVER; - mDebug("trans:%d, used to update db:%s", pTrans->id, pOld->name); + mDebug("trans:%d, used to alter db:%s", pTrans->id, pOld->name); mndTransSetDbInfo(pTrans, pOld); - if (mndSetUpdateDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndSetUpdateDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndSetUpdateDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; + if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; + if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; + if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto UPDATE_DB_OVER; code = 0; @@ -716,7 +784,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { goto ALTER_DB_OVER; } - if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { + if (mndCheckAlterDropCompactDbAuth(pUser, pDb) != 0) { goto ALTER_DB_OVER; } @@ -730,7 +798,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); - code = mndUpdateDb(pMnode, pReq, pDb, &dbObj); + code = mndAlterDb(pMnode, pReq, pDb, &dbObj); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; ALTER_DB_OVER: @@ -745,11 +813,11 @@ ALTER_DB_OVER: } static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { - SMnode *pMnode = pReq->pNode; - int32_t code = -1; - SDbObj *pDb = NULL; - SDbCfgReq cfgReq = {0}; - SDbCfgRsp cfgRsp = {0}; + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SDbObj *pDb = NULL; + SDbCfgReq cfgReq = {0}; + SDbCfgRsp cfgRsp = {0}; if (tDeserializeSDbCfgReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &cfgReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -762,27 +830,29 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { goto GET_DB_CFG_OVER; } - cfgRsp.numOfVgroups = pDb->cfg.numOfVgroups; + cfgRsp.numOfVgroups = pDb->cfg.numOfVgroups; cfgRsp.cacheBlockSize = pDb->cfg.cacheBlockSize; - cfgRsp.totalBlocks = pDb->cfg.totalBlocks; - cfgRsp.daysPerFile = pDb->cfg.daysPerFile; - cfgRsp.daysToKeep0 = pDb->cfg.daysToKeep0; - cfgRsp.daysToKeep1 = pDb->cfg.daysToKeep1; - cfgRsp.daysToKeep2 = pDb->cfg.daysToKeep2; - cfgRsp.minRows = pDb->cfg.minRows; - cfgRsp.maxRows = pDb->cfg.maxRows; - cfgRsp.commitTime = pDb->cfg.commitTime; - cfgRsp.fsyncPeriod = pDb->cfg.fsyncPeriod; - cfgRsp.ttl = pDb->cfg.ttl; - cfgRsp.walLevel = pDb->cfg.walLevel; - cfgRsp.precision = pDb->cfg.precision; - cfgRsp.compression = pDb->cfg.compression; - cfgRsp.replications = pDb->cfg.replications; - cfgRsp.quorum = pDb->cfg.quorum; - cfgRsp.update = pDb->cfg.update; - cfgRsp.cacheLastRow = pDb->cfg.cacheLastRow; - cfgRsp.streamMode = pDb->cfg.streamMode; - cfgRsp.singleSTable = pDb->cfg.singleSTable; + cfgRsp.totalBlocks = pDb->cfg.totalBlocks; + cfgRsp.daysPerFile = pDb->cfg.daysPerFile; + cfgRsp.daysToKeep0 = pDb->cfg.daysToKeep0; + cfgRsp.daysToKeep1 = pDb->cfg.daysToKeep1; + cfgRsp.daysToKeep2 = pDb->cfg.daysToKeep2; + cfgRsp.minRows = pDb->cfg.minRows; + cfgRsp.maxRows = pDb->cfg.maxRows; + cfgRsp.commitTime = pDb->cfg.commitTime; + cfgRsp.fsyncPeriod = pDb->cfg.fsyncPeriod; + cfgRsp.ttl = pDb->cfg.ttl; + cfgRsp.walLevel = pDb->cfg.walLevel; + cfgRsp.precision = pDb->cfg.precision; + cfgRsp.compression = pDb->cfg.compression; + cfgRsp.replications = pDb->cfg.replications; + cfgRsp.strict = pDb->cfg.strict; + cfgRsp.update = pDb->cfg.update; + cfgRsp.cacheLastRow = pDb->cfg.cacheLastRow; + cfgRsp.streamMode = pDb->cfg.streamMode; + cfgRsp.singleSTable = pDb->cfg.singleSTable; + cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions; + cfgRsp.pRetensions = pDb->cfg.pRetensions; int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &cfgRsp); void *pRsp = rpcMallocCont(contLen); @@ -797,6 +867,8 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { pReq->pRsp = pRsp; pReq->rspLen = contLen; + code = 0; + GET_DB_CFG_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { @@ -808,7 +880,6 @@ GET_DB_CFG_OVER: return code; } - static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { SSdbRaw *pRedoRaw = mndDbActionEncode(pDb); if (pRedoRaw == NULL) return -1; @@ -945,25 +1016,25 @@ static int32_t mndBuildDropDbRsp(SDbObj *pDb, int32_t *pRspLen, void **ppRsp, bo static int32_t mndDropDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_DB, &pReq->rpcMsg); - if (pTrans == NULL) goto DROP_DB_OVER; + if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); mndTransSetDbInfo(pTrans, pDb); - if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; - if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; - if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; + if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER; + if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER; + if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; int32_t rspLen = 0; void *pRsp = NULL; - if (mndBuildDropDbRsp(pDb, &rspLen, &pRsp, false) < 0) goto DROP_DB_OVER; + if (mndBuildDropDbRsp(pDb, &rspLen, &pRsp, false) < 0) goto _OVER; mndTransSetRpcRsp(pTrans, pRsp, rspLen); - if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_DB_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; -DROP_DB_OVER: +_OVER: mndTransDrop(pTrans); return code; } @@ -977,7 +1048,7 @@ static int32_t mndProcessDropDbReq(SNodeMsg *pReq) { if (tDeserializeSDropDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto DROP_DB_OVER; + goto _OVER; } mDebug("db:%s, start to drop", dropReq.db); @@ -986,26 +1057,26 @@ static int32_t mndProcessDropDbReq(SNodeMsg *pReq) { if (pDb == NULL) { if (dropReq.ignoreNotExists) { code = mndBuildDropDbRsp(pDb, &pReq->rspLen, &pReq->pRsp, true); - goto DROP_DB_OVER; + goto _OVER; } else { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto DROP_DB_OVER; + goto _OVER; } } pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto DROP_DB_OVER; + goto _OVER; } - if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { - goto DROP_DB_OVER; + if (mndCheckAlterDropCompactDbAuth(pUser, pDb) != 0) { + goto _OVER; } code = mndDropDb(pMnode, pReq, pDb); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -DROP_DB_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to drop since %s", dropReq.db, terrstr()); } @@ -1100,7 +1171,7 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs pRsp->uid = pDb->uid; pRsp->vgVersion = pDb->vgVersion; pRsp->vgNum = taosArrayGetSize(pRsp->pVgroupInfos); - pRsp->hashMethod = pDb->hashMethod; + pRsp->hashMethod = pDb->cfg.hashMethod; return 0; } @@ -1243,7 +1314,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, usedbRsp.uid = pDb->uid; usedbRsp.vgVersion = pDb->vgVersion; usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos); - usedbRsp.hashMethod = pDb->hashMethod; + usedbRsp.hashMethod = pDb->cfg.hashMethod; taosArrayPush(batchUseRsp.pArray, &usedbRsp); mndReleaseDb(pMnode, pDb); @@ -1265,47 +1336,6 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, return 0; } -static int32_t mndProcessSyncDbReq(SNodeMsg *pReq) { - SMnode *pMnode = pReq->pNode; - int32_t code = -1; - SDbObj *pDb = NULL; - SUserObj *pUser = NULL; - SSyncDbReq syncReq = {0}; - - if (tDeserializeSSyncDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &syncReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto SYNC_DB_OVER; - } - - mDebug("db:%s, start to sync", syncReq.db); - - pDb = mndAcquireDb(pMnode, syncReq.db); - if (pDb == NULL) { - goto SYNC_DB_OVER; - } - - pUser = mndAcquireUser(pMnode, pReq->user); - if (pUser == NULL) { - goto SYNC_DB_OVER; - } - - if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { - goto SYNC_DB_OVER; - } - - // code = mndSyncDb(); - -SYNC_DB_OVER: - if (code != 0) { - mError("db:%s, failed to process sync db req since %s", syncReq.db, terrstr()); - } - - mndReleaseDb(pMnode, pDb); - mndReleaseUser(pMnode, pUser); - - return code; -} - static int32_t mndProcessCompactDbReq(SNodeMsg *pReq) { SMnode *pMnode = pReq->pNode; int32_t code = -1; @@ -1313,30 +1343,30 @@ static int32_t mndProcessCompactDbReq(SNodeMsg *pReq) { SUserObj *pUser = NULL; SCompactDbReq compactReq = {0}; - if (tDeserializeSSyncDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &compactReq) != 0) { + if (tDeserializeSCompactDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &compactReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto SYNC_DB_OVER; + goto _OVER; } mDebug("db:%s, start to sync", compactReq.db); pDb = mndAcquireDb(pMnode, compactReq.db); if (pDb == NULL) { - goto SYNC_DB_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto SYNC_DB_OVER; + goto _OVER; } - if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { - goto SYNC_DB_OVER; + if (mndCheckAlterDropCompactDbAuth(pUser, pDb) != 0) { + goto _OVER; } - // code = mndSyncDb(); + // code = mndCompactDb(); -SYNC_DB_OVER: +_OVER: if (code != 0) { mError("db:%s, failed to process compact db req since %s", compactReq.db, terrstr()); } @@ -1347,27 +1377,23 @@ SYNC_DB_OVER: return code; } -char *mndGetDbStr(char *src) { +const char *mndGetDbStr(const char *src) { char *pos = strstr(src, TS_PATH_DELIMITER); if (pos != NULL) ++pos; - - if (pos == NULL) { - return src; - } - + if (pos == NULL) return src; return pos; } -static void dumpDbInfoData(SSDataBlock* pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables, bool sysDb) { +static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables, + bool sysDb) { int32_t cols = 0; - char* buf = taosMemoryMalloc(pShow->bytes[cols]); - char *name = mndGetDbStr(pDb->name); + char *buf = taosMemoryMalloc(pShow->bytes[cols]); + const char *name = mndGetDbStr(pDb->name); if (name != NULL) { STR_WITH_MAXSIZE_TO_VARSTR(buf, name, pShow->bytes[cols]); } else { -// STR_TO_VARSTR(pWrite, "NULL"); - ASSERT(0); + STR_WITH_MAXSIZE_TO_VARSTR(buf, "NULL", pShow->bytes[cols]); } char *status = "ready"; @@ -1375,8 +1401,8 @@ static void dumpDbInfoData(SSDataBlock* pBlock, SDbObj *pDb, SShowObj *pShow, in STR_WITH_SIZE_TO_VARSTR(b, status, strlen(status)); if (sysDb) { - for(int32_t i = 0; i < pShow->numOfColumns; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + for (int32_t i = 0; i < pShow->numOfColumns; ++i) { + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i); if (i == 0) { colDataAppend(pColInfo, rows, buf, false); } else if (i == 3) { @@ -1404,8 +1430,11 @@ static void dumpDbInfoData(SSDataBlock* pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.replications, false); + const char *src = pDb->cfg.strict ? "strict" : "nostrict"; + char b[9 + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_SIZE_TO_VARSTR(b, src, strlen(src)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.quorum, false); + colDataAppend(pColInfo, rows, (const char *)b, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.daysPerFile, false); @@ -1484,7 +1513,6 @@ static void dumpDbInfoData(SSDataBlock* pBlock, SDbObj *pDb, SShowObj *pShow, in // pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); // *(int8_t *)pWrite = pDb->cfg.update; - } static void setInformationSchemaDbCfg(SDbObj *pDbObj) { @@ -1493,19 +1521,19 @@ static void setInformationSchemaDbCfg(SDbObj *pDbObj) { pDbObj->createdTime = 0; pDbObj->cfg.numOfVgroups = 0; - pDbObj->cfg.quorum = 1; + pDbObj->cfg.strict = 1; pDbObj->cfg.replications = 1; pDbObj->cfg.update = 1; pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI; } -static void setPerfSchemaDbCfg(SDbObj* pDbObj) { +static void setPerfSchemaDbCfg(SDbObj *pDbObj) { ASSERT(pDbObj != NULL); strncpy(pDbObj->name, TSDB_PERFORMANCE_SCHEMA_DB, tListLen(pDbObj->name)); pDbObj->createdTime = 0; pDbObj->cfg.numOfVgroups = 0; - pDbObj->cfg.quorum = 1; + pDbObj->cfg.strict = 1; pDbObj->cfg.replications = 1; pDbObj->cfg.update = 1; pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI; @@ -1519,7 +1547,7 @@ static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, v return true; } -static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rowsCapacity) { +static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -1567,10 +1595,10 @@ static void mndCancelGetNextDb(SMnode *pMnode, void *pIter) { static int32_t mndProcessGetIndexReq(SNodeMsg *pReq) { SUserIndexReq indexReq = {0}; - SMnode *pMnode = pReq->pNode; - int32_t code = -1; + SMnode *pMnode = pReq->pNode; + int32_t code = -1; SUserIndexRsp rsp = {0}; - bool exist = false; + bool exist = false; if (tDeserializeSUserIndexReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &indexReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -1583,7 +1611,7 @@ static int32_t mndProcessGetIndexReq(SNodeMsg *pReq) { } if (!exist) { - //TODO GET INDEX FROM FULLTEXT + // TODO GET INDEX FROM FULLTEXT code = -1; terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; } else { @@ -1594,9 +1622,9 @@ static int32_t mndProcessGetIndexReq(SNodeMsg *pReq) { code = -1; goto _OVER; } - + tSerializeSUserIndexRsp(pRsp, contLen, &rsp); - + pReq->pRsp = pRsp; pReq->rspLen = contLen; @@ -1610,4 +1638,3 @@ _OVER: return code; } - diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 6374b4cad202b513556e354cef18682a5dcc2259..78d54d273deaf8b4295b7f1ccf05224bb1e8e060 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -14,6 +14,354 @@ */ #include "mndDef.h" +#include "mndConsumer.h" + +SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]) { + SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); + if (pConsumer == NULL) { + return NULL; + } + + pConsumer->consumerId = consumerId; + memcpy(pConsumer->cgroup, cgroup, TSDB_CGROUP_LEN); + + pConsumer->epoch = 0; + pConsumer->status = MQ_CONSUMER_STATUS__MODIFY; + pConsumer->hbStatus = 0; + + taosInitRWLatch(&pConsumer->lock); + + pConsumer->currentTopics = taosArrayInit(0, sizeof(void *)); + pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *)); + pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *)); + + if (pConsumer->currentTopics == NULL || pConsumer->rebNewTopics == NULL || pConsumer->rebRemovedTopics == NULL) { + taosArrayDestroy(pConsumer->currentTopics); + taosArrayDestroy(pConsumer->rebNewTopics); + taosArrayDestroy(pConsumer->rebRemovedTopics); + taosMemoryFree(pConsumer); + return NULL; + } + + return pConsumer; +} + +void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer) { + if (pConsumer->currentTopics) { + taosArrayDestroyP(pConsumer->currentTopics, (FDelete)taosMemoryFree); + } + if (pConsumer->rebNewTopics) { + taosArrayDestroyP(pConsumer->rebNewTopics, (FDelete)taosMemoryFree); + } + if (pConsumer->rebRemovedTopics) { + taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree); + } +} + +int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { + int32_t tlen = 0; + int32_t sz; + tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); + tlen += taosEncodeString(buf, pConsumer->cgroup); + tlen += taosEncodeFixedI8(buf, pConsumer->updateType); + tlen += taosEncodeFixedI32(buf, pConsumer->epoch); + tlen += taosEncodeFixedI32(buf, pConsumer->status); + + // current topics + if (pConsumer->currentTopics) { + sz = taosArrayGetSize(pConsumer->currentTopics); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + char *topic = taosArrayGetP(pConsumer->currentTopics, i); + tlen += taosEncodeString(buf, topic); + } + } else { + tlen += taosEncodeFixedI32(buf, 0); + } + + // reb new topics + if (pConsumer->rebNewTopics) { + sz = taosArrayGetSize(pConsumer->rebNewTopics); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + char *topic = taosArrayGetP(pConsumer->rebNewTopics, i); + tlen += taosEncodeString(buf, topic); + } + } else { + tlen += taosEncodeFixedI32(buf, 0); + } + + // reb removed topics + if (pConsumer->rebRemovedTopics) { + sz = taosArrayGetSize(pConsumer->rebRemovedTopics); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + char *topic = taosArrayGetP(pConsumer->rebRemovedTopics, i); + tlen += taosEncodeString(buf, topic); + } + } else { + tlen += taosEncodeFixedI32(buf, 0); + } + + return tlen; +} + +void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) { + int32_t sz; + buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); + buf = taosDecodeStringTo(buf, pConsumer->cgroup); + buf = taosDecodeFixedI8(buf, &pConsumer->updateType); + buf = taosDecodeFixedI32(buf, &pConsumer->epoch); + buf = taosDecodeFixedI32(buf, &pConsumer->status); + + // current topics + buf = taosDecodeFixedI32(buf, &sz); + pConsumer->currentTopics = taosArrayInit(sz, sizeof(void *)); + for (int32_t i = 0; i < sz; i++) { + char *topic; + buf = taosDecodeString(buf, &topic); + taosArrayPush(pConsumer->currentTopics, &topic); + } + + // reb new topics + buf = taosDecodeFixedI32(buf, &sz); + pConsumer->rebNewTopics = taosArrayInit(sz, sizeof(void *)); + for (int32_t i = 0; i < sz; i++) { + char *topic; + buf = taosDecodeString(buf, &topic); + taosArrayPush(pConsumer->rebNewTopics, &topic); + } + + // reb removed topics + buf = taosDecodeFixedI32(buf, &sz); + pConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *)); + for (int32_t i = 0; i < sz; i++) { + char *topic; + buf = taosDecodeString(buf, &topic); + taosArrayPush(pConsumer->rebRemovedTopics, &topic); + } + + return (void *)buf; +} + +SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { + SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); + if (pVgEpNew == NULL) return NULL; + pVgEpNew->vgId = pVgEp->vgId; + pVgEpNew->qmsg = strdup(pVgEp->qmsg); + pVgEpNew->epSet = pVgEp->epSet; + return pVgEpNew; +} + +void tDeleteSMqVgEp(SMqVgEp *pVgEp) { taosMemoryFree(pVgEp->qmsg); } + +int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI32(buf, pVgEp->vgId); + tlen += taosEncodeString(buf, pVgEp->qmsg); + tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); + return tlen; +} + +void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) { + buf = taosDecodeFixedI32(buf, &pVgEp->vgId); + buf = taosDecodeString(buf, &pVgEp->qmsg); + buf = taosDecodeSEpSet(buf, &pVgEp->epSet); + return (void *)buf; +} + +SMqConsumerEpInSub *tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub *pEpInSub) { + SMqConsumerEpInSub *pEpInSubNew = taosMemoryMalloc(sizeof(SMqConsumerEpInSub)); + if (pEpInSubNew == NULL) return NULL; + pEpInSubNew->consumerId = pEpInSub->consumerId; + pEpInSubNew->vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp); + return pEpInSubNew; +} + +void tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub *pEpInSub) { + taosArrayDestroyEx(pEpInSub->vgs, (FDelete)tDeleteSMqVgEp); +} + +int32_t tEncodeSMqConsumerEpInSub(void **buf, const SMqConsumerEpInSub *pEpInSub) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pEpInSub->consumerId); + int32_t sz = taosArrayGetSize(pEpInSub->vgs); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, i); + tlen += tEncodeSMqVgEp(buf, pVgEp); + } + /*tlen += taosEncodeArray(buf, pEpInSub->vgs, (FEncode)tEncodeSMqVgEp);*/ + return tlen; +} + +void *tDecodeSMqConsumerEpInSub(const void *buf, SMqConsumerEpInSub *pEpInSub) { + buf = taosDecodeFixedI64(buf, &pEpInSub->consumerId); + /*buf = taosDecodeArray(buf, &pEpInSub->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));*/ + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pEpInSub->vgs = taosArrayInit(sz, sizeof(void *)); + for (int32_t i = 0; i < sz; i++) { + SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); + buf = tDecodeSMqVgEp(buf, pVgEp); + taosArrayPush(pEpInSub->vgs, &pVgEp); + } + + return (void *)buf; +} + +SMqSubscribeObj *tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]) { + SMqSubscribeObj *pSubNew = taosMemoryCalloc(1, sizeof(SMqSubscribeObj)); + if (pSubNew == NULL) return NULL; + memcpy(pSubNew->key, key, TSDB_SUBSCRIBE_KEY_LEN); + taosInitRWLatch(&pSubNew->lock); + pSubNew->vgNum = -1; + pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + // TODO set free fp + SMqConsumerEpInSub epInSub = { + .consumerId = -1, + .vgs = taosArrayInit(0, sizeof(void *)), + }; + int64_t unexistKey = -1; + taosHashPut(pSubNew->consumerHash, &unexistKey, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub)); + return pSubNew; +} + +SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { + SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj)); + if (pSubNew == NULL) return NULL; + memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); + taosInitRWLatch(&pSubNew->lock); + + pSubNew->subType = pSub->subType; + pSubNew->withTbName = pSub->withTbName; + pSubNew->withSchema = pSub->withSchema; + pSubNew->withTag = pSub->withTag; + pSubNew->withTagSchema = pSub->withTagSchema; + + pSubNew->vgNum = pSub->vgNum; + pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + /*taosHashSetFreeFp(pSubNew->consumerHash, taosArrayDestroy);*/ + void *pIter = NULL; + SMqConsumerEpInSub *pEpInSub = NULL; + while (1) { + pIter = taosHashIterate(pSub->consumerHash, pIter); + if (pIter == NULL) break; + pEpInSub = (SMqConsumerEpInSub *)pIter; + SMqConsumerEpInSub newEp = { + .consumerId = pEpInSub->consumerId, + .vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp), + }; + taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEpInSub)); + } + return pSubNew; +} + +void tDeleteSubscribeObj(SMqSubscribeObj *pSub) { + /*taosArrayDestroyEx(pSub->consumerEps, (FDelete)tDeleteSMqConsumerEpInSub);*/ + taosHashCleanup(pSub->consumerHash); +} + +int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pSub->key); + tlen += taosEncodeFixedI32(buf, pSub->vgNum); + tlen += taosEncodeFixedI8(buf, pSub->subType); + tlen += taosEncodeFixedI8(buf, pSub->withTbName); + tlen += taosEncodeFixedI8(buf, pSub->withSchema); + tlen += taosEncodeFixedI8(buf, pSub->withTag); + tlen += taosEncodeFixedI8(buf, pSub->withTagSchema); + + void *pIter = NULL; + int32_t sz = taosHashGetSize(pSub->consumerHash); + tlen += taosEncodeFixedI32(buf, sz); + + int32_t cnt = 0; + while (1) { + pIter = taosHashIterate(pSub->consumerHash, pIter); + if (pIter == NULL) break; + SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; + tlen += tEncodeSMqConsumerEpInSub(buf, pEpInSub); + cnt++; + } + ASSERT(cnt == sz); + /*tlen += taosEncodeArray(buf, pSub->consumerEps, (FEncode)tEncodeSMqConsumerEpInSub);*/ + return tlen; +} + +void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) { + // + buf = taosDecodeStringTo(buf, pSub->key); + buf = taosDecodeFixedI32(buf, &pSub->vgNum); + buf = taosDecodeFixedI8(buf, &pSub->subType); + buf = taosDecodeFixedI8(buf, &pSub->withTbName); + buf = taosDecodeFixedI8(buf, &pSub->withSchema); + buf = taosDecodeFixedI8(buf, &pSub->withTag); + buf = taosDecodeFixedI8(buf, &pSub->withTagSchema); + + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + + pSub->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + for (int32_t i = 0; i < sz; i++) { + /*SMqConsumerEpInSub* pEpInSub = taosMemoryMalloc(sizeof(SMqConsumerEpInSub));*/ + SMqConsumerEpInSub epInSub = {0}; + buf = tDecodeSMqConsumerEpInSub(buf, &epInSub); + taosHashPut(pSub->consumerHash, &epInSub.consumerId, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub)); + } + + /*buf = taosDecodeArray(buf, &pSub->consumerEps, (FDecode)tDecodeSMqConsumerEpInSub, sizeof(SMqConsumerEpInSub));*/ + return (void *)buf; +} + +SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { + SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry)); + if (pEntryNew == NULL) return NULL; + pEntryNew->epoch = pEntry->epoch; + pEntryNew->consumers = taosArrayDeepCopy(pEntry->consumers, (FCopy)tCloneSMqConsumerEpInSub); + return pEntryNew; +} + +void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { + taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEpInSub); +} + +int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) { + int32_t tlen = 0; + tlen += taosEncodeFixedI32(buf, pEntry->epoch); + tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry); + return tlen; +} +void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) { + buf = taosDecodeFixedI32(buf, &pEntry->epoch); + buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); + return (void *)buf; +} + +SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) { + SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj)); + if (pLogNew == NULL) return pLogNew; + memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN); + pLogNew->logs = taosArrayDeepCopy(pLog->logs, (FCopy)tCloneSMqConsumerEpInSub); + return pLogNew; +} + +void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) { + taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEpInSub); +} + +int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pLog->key); + tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry); + return tlen; +} + +void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) { + buf = taosDecodeStringTo(buf, pLog->key); + buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); + return (void *)buf; +} int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { int32_t sz = 0; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index f225f2fbe76c40d0e6ab1f5cbda43d6a28dc8449..5fa51820791b44ddb48cf1938fa07f9c5e011d8b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -326,7 +326,7 @@ static int32_t mndProcessStatusReq(SNodeMsg *pReq) { SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVload->vgId); if (pVgroup != NULL) { - if (pVload->role == TAOS_SYNC_STATE_LEADER) { + if (pVload->syncState == TAOS_SYNC_STATE_LEADER) { pVgroup->numOfTables = pVload->numOfTables; pVgroup->numOfTimeSeries = pVload->numOfTimeSeries; pVgroup->totalStorage = pVload->totalStorage; @@ -337,10 +337,10 @@ static int32_t mndProcessStatusReq(SNodeMsg *pReq) { for (int32_t vg = 0; vg < pVgroup->replica; ++vg) { // sync integration if (pVgroup->vnodeGid[vg].dnodeId == statusReq.dnodeId) { - if (pVgroup->vnodeGid[vg].role != pVload->role) { + if (pVgroup->vnodeGid[vg].role != pVload->syncState) { roleChanged = true; } - pVgroup->vnodeGid[vg].role = pVload->role; + pVgroup->vnodeGid[vg].role = pVload->syncState; } } if (roleChanged) { diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 26c7b4d20030557108ac998676f2fbc68757d359..156d894a442a5e48cd54eb7a4d652190100df47d 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -77,7 +77,9 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) { SDB_SET_INT64(pRaw, dataPos, pFunc->signature, _OVER) SDB_SET_INT32(pRaw, dataPos, pFunc->commentSize, _OVER) SDB_SET_INT32(pRaw, dataPos, pFunc->codeSize, _OVER) - SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, _OVER) + if (pFunc->commentSize > 0) { + SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, _OVER) + } SDB_SET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, _OVER) SDB_SET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER); @@ -125,13 +127,18 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pFunc->commentSize, _OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->codeSize, _OVER) - pFunc->pComment = taosMemoryCalloc(1, pFunc->commentSize); + if (pFunc->commentSize > 0) { + pFunc->pComment = taosMemoryCalloc(1, pFunc->commentSize); + if (pFunc->pComment == NULL) { + goto _OVER; + } + SDB_GET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, _OVER) + } + pFunc->pCode = taosMemoryCalloc(1, pFunc->codeSize); - if (pFunc->pComment == NULL || pFunc->pCode == NULL) { + if (pFunc->pCode == NULL) { goto _OVER; } - - SDB_GET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, _OVER) SDB_GET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, _OVER) SDB_GET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, _OVER) @@ -192,16 +199,20 @@ static int32_t mndCreateFunc(SMnode *pMnode, SNodeMsg *pReq, SCreateFuncReq *pCr func.outputLen = pCreate->outputLen; func.bufSize = pCreate->bufSize; func.signature = pCreate->signature; - func.commentSize = strlen(pCreate->pComment) + 1; - func.codeSize = strlen(pCreate->pCode) + 1; - func.pComment = taosMemoryMalloc(func.commentSize); + if (NULL != pCreate->pComment) { + func.commentSize = strlen(pCreate->pComment) + 1; + func.pComment = taosMemoryMalloc(func.commentSize); + } + func.codeSize = pCreate->codeLen; func.pCode = taosMemoryMalloc(func.codeSize); if (func.pCode == NULL || func.pCode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } - memcpy(func.pComment, pCreate->pComment, func.commentSize); + if (func.commentSize > 0) { + memcpy(func.pComment, pCreate->pComment, func.commentSize); + } memcpy(func.pCode, pCreate->pCode, func.codeSize); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_FUNC, &pReq->rpcMsg); @@ -293,16 +304,6 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) { goto _OVER; } - if (createReq.pComment == NULL) { - terrno = TSDB_CODE_MND_INVALID_FUNC_COMMENT; - goto _OVER; - } - - if (createReq.pComment[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_FUNC_COMMENT; - goto _OVER; - } - if (createReq.pCode == NULL) { terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; goto _OVER; @@ -426,7 +427,6 @@ static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) { SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName); if (pFunc == NULL) { - terrno = TSDB_CODE_MND_INVALID_FUNC; goto RETRIEVE_FUNC_OVER; } @@ -438,16 +438,27 @@ static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) { funcInfo.outputLen = pFunc->outputLen; funcInfo.bufSize = pFunc->bufSize; funcInfo.signature = pFunc->signature; - funcInfo.commentSize = pFunc->commentSize; - funcInfo.codeSize = pFunc->codeSize; - funcInfo.pCode = taosMemoryCalloc(1, sizeof(funcInfo.codeSize)); - funcInfo.pComment = taosMemoryCalloc(1, sizeof(funcInfo.commentSize)); - if (funcInfo.pCode == NULL || funcInfo.pComment == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto RETRIEVE_FUNC_OVER; + if (retrieveReq.ignoreCodeComment) { + funcInfo.commentSize = 0; + funcInfo.codeSize = 0; + } else { + funcInfo.commentSize = pFunc->commentSize; + funcInfo.codeSize = pFunc->codeSize; + funcInfo.pCode = taosMemoryCalloc(1, funcInfo.codeSize); + if (funcInfo.pCode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto RETRIEVE_FUNC_OVER; + } + memcpy(funcInfo.pCode, pFunc->pCode, pFunc->codeSize); + if (funcInfo.commentSize > 0) { + funcInfo.pComment = taosMemoryCalloc(1, funcInfo.commentSize); + if (funcInfo.pComment == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto RETRIEVE_FUNC_OVER; + } + memcpy(funcInfo.pComment, pFunc->pComment, pFunc->commentSize); + } } - memcpy(funcInfo.pComment, pFunc->pComment, pFunc->commentSize); - memcpy(funcInfo.pCode, pFunc->pCode, pFunc->codeSize); taosArrayPush(retrieveRsp.pFuncInfos, &funcInfo); mndReleaseFunc(pMnode, pFunc); } @@ -511,11 +522,16 @@ static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)b1, false); - char *b2 = taosMemoryCalloc(1, pShow->bytes[cols]); - STR_WITH_MAXSIZE_TO_VARSTR(b2, pFunc->pComment, pShow->bytes[cols]); + if (pFunc->pComment) { + char *b2 = taosMemoryCalloc(1, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(b2, pFunc->pComment, pShow->bytes[cols]); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)b2, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)b2, false); + } else { + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, NULL, true); + } int32_t isAgg = (pFunc->funcType == TSDB_FUNC_TYPE_AGGREGATE) ? 1 : 0; @@ -549,4 +565,4 @@ static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index d84c5f84cf11cfba6794f98114ede7a07404ae70..32c667faf90fb7616d2acccbcd7daac8c1dc4a13 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -75,7 +75,7 @@ static const SInfosTableSchema userDBSchema[] = { {.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "quorum", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "strict", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, @@ -98,7 +98,7 @@ static const SInfosTableSchema userFuncSchema[] = { {.name = "name", .bytes = TSDB_FUNC_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "comment", .bytes = PATH_MAX - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "aggregate", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "comment", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "output_type", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "code_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "bufsize", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index f403da2f2d9d698421a78a36418dcf8deb4ed9fb..b51c545a6d2739014275a58396054c5b96945836 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -75,19 +75,6 @@ void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj) { sdbRelease(pMnode->pSdb, pObj); } -const char *mndGetRoleStr(int32_t showType) { - switch (showType) { - case TAOS_SYNC_STATE_FOLLOWER: - return "FOLLOWER"; - case TAOS_SYNC_STATE_CANDIDATE: - return "CANDIDATE"; - case TAOS_SYNC_STATE_LEADER: - return "LEADER"; - default: - return "ERROR"; - } -} - void mndUpdateMnodeRole(SMnode *pMnode) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; @@ -637,7 +624,7 @@ static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *p pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, b1, false); - const char *roles = mndGetRoleStr(pObj->role); + const char *roles = syncStr(pObj->role); char *b2 = taosMemoryCalloc(1, strlen(roles) + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(b2, roles, pShow->bytes[cols]); diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index 31c16e1e1d1349a11ce89c2ee325f99af9735156..f5433e8f9e06a30a14eb5fe4a867f28eadd38f05 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -130,13 +130,12 @@ OFFSET_DECODE_OVER: return pRow; } -int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) { - int32_t code = 0; +int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) { int32_t sz = taosArrayGetSize(vgs); for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp *pConsumerEp = taosArrayGet(vgs, i); - SMqOffsetObj offsetObj; - if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, pConsumerEp->vgId) < 0) { + int32_t vgId = *(int32_t *)taosArrayGet(vgs, i); + SMqOffsetObj offsetObj; + if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, vgId) < 0) { return -1; } offsetObj.offset = -1; @@ -170,13 +169,22 @@ static int32_t mndProcessCommitOffsetReq(SNodeMsg *pMsg) { if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) { return -1; } + bool create = false; SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key); - ASSERT(pOffsetObj); + if (pOffsetObj == NULL) { + pOffsetObj = taosMemoryMalloc(sizeof(SMqOffset)); + memcpy(pOffsetObj->key, key, TSDB_PARTITION_KEY_LEN); + create = true; + } pOffsetObj->offset = pOffset->offset; SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj); sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY); mndTransAppendRedolog(pTrans, pOffsetRaw); - mndReleaseOffset(pMnode, pOffsetObj); + if (create) { + taosMemoryFree(pOffsetObj); + } else { + mndReleaseOffset(pMnode, pOffsetObj); + } } if (mndTransPrepare(pMnode, pTrans) != 0) { @@ -201,7 +209,7 @@ static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset) { static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOldOffset, SMqOffsetObj *pNewOffset) { mTrace("offset:%s, perform update action", pOldOffset->key); - pOldOffset->offset = pNewOffset->offset; + atomic_store_64(&pOldOffset->offset, pNewOffset->offset); return 0; } diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c index 38e9e792286d41bfcc53f5559df0ade2c264de3b..fe96bcb709dcfd7fc465a6d8a1594f490e32cba4 100644 --- a/source/dnode/mnode/impl/src/mndPerfSchema.c +++ b/source/dnode/mnode/impl/src/mndPerfSchema.c @@ -41,10 +41,10 @@ static const SPerfsTableSchema queriesSchema[] = { static const SPerfsTableSchema topicSchema[] = { {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + /*{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},*/ {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + /*{.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},*/ }; static const SPerfsTableSchema consumerSchema[] = { diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 826a73afc68915c3b161b9896a2bc58677bd409d..ec4be8cba3ad83fb448676de3ff892247eb58ce9 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -24,20 +24,20 @@ #include "version.h" typedef struct { - uint32_t id; - int8_t connType; - char user[TSDB_USER_LEN]; - char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc - int64_t appStartTimeMs; // app start time - int32_t pid; // pid of app that invokes taosc - uint32_t ip; - uint16_t port; - int8_t killed; - int64_t loginTimeMs; - int64_t lastAccessTimeMs; - uint64_t killId; - int32_t numOfQueries; - SArray *pQueries; //SArray + uint32_t id; + int8_t connType; + char user[TSDB_USER_LEN]; + char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc + int64_t appStartTimeMs; // app start time + int32_t pid; // pid of app that invokes taosc + uint32_t ip; + uint16_t port; + int8_t killed; + int64_t loginTimeMs; + int64_t lastAccessTimeMs; + uint64_t killId; + int32_t numOfQueries; + SArray * pQueries; // SArray } SConnObj; static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType, uint32_t ip, uint16_t port, @@ -45,7 +45,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType static void mndFreeConn(SConnObj *pConn); static SConnObj *mndAcquireConn(SMnode *pMnode, uint32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); -static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter); +static void * mndGetNextConn(SMnode *pMnode, SCacheIter *pIter); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq); static int32_t mndProcessConnectReq(SNodeMsg *pReq); @@ -71,9 +71,9 @@ int32_t mndInitProfile(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_KILL_QUERY, mndProcessKillQueryReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_CONN, mndProcessKillConnReq); -// mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); + // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndCancelGetNextConn); -// mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries); + // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndCancelGetNextQuery); return 0; @@ -91,7 +91,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType int32_t pid, const char *app, int64_t startTime) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - char connStr[255] = {0}; + char connStr[255] = {0}; int32_t len = snprintf(connStr, sizeof(connStr), "%s%d%d%d%s", user, ip, port, pid, app); int32_t connId = mndGenerateUid(connStr, len); if (startTime == 0) startTime = taosGetTimestampMs(); @@ -174,10 +174,10 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { } static int32_t mndProcessConnectReq(SNodeMsg *pReq) { - SMnode *pMnode = pReq->pNode; - SUserObj *pUser = NULL; - SDbObj *pDb = NULL; - SConnObj *pConn = NULL; + SMnode * pMnode = pReq->pNode; + SUserObj * pUser = NULL; + SDbObj * pDb = NULL; + SConnObj * pConn = NULL; int32_t code = -1; SConnectReq connReq = {0}; char ip[30] = {0}; @@ -194,6 +194,11 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) { mError("user:%s, failed to login while acquire user since %s", pReq->user, terrstr()); goto CONN_OVER; } + if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) { + mError("user:%s, failed to auth while acquire user\n %s \r\n %s", pReq->user, connReq.passwd, pUser->pass); + code = TSDB_CODE_RPC_AUTH_FAILURE; + goto CONN_OVER; + } if (connReq.db[0]) { char db[TSDB_DB_FNAME_LEN]; @@ -253,8 +258,8 @@ static int32_t mndSaveQueryList(SConnObj *pConn, SQueryHbReqBasic *pBasic) { pConn->pQueries = pBasic->queryDesc; pBasic->queryDesc = NULL; - - pConn->numOfQueries = pBasic->queryDesc ? taosArrayGetSize(pBasic->queryDesc) : 0; + + pConn->numOfQueries = pBasic->queryDesc ? taosArrayGetSize(pBasic->queryDesc) : 0; return TSDB_CODE_SUCCESS; } @@ -324,9 +329,10 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { return NULL; } -static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq, SClientHbBatchRsp *pBatchRsp) { +static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq, + SClientHbBatchRsp *pBatchRsp) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL}; + SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL}; if (pHbReq->query) { SQueryHbReqBasic *pBasic = pHbReq->query; @@ -335,8 +341,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb rpcGetConnInfo(pMsg->handle, &connInfo); SConnObj *pConn = mndAcquireConn(pMnode, pBasic->connId); - if (pConn == NULL) { - pConn = mndCreateConn(pMnode, connInfo.user, CONN_TYPE__QUERY, connInfo.clientIp, connInfo.clientPort, pBasic->pid, pBasic->app, 0); + if (pConn == NULL) { + pConn = mndCreateConn(pMnode, connInfo.user, CONN_TYPE__QUERY, connInfo.clientIp, connInfo.clientPort, + pBasic->pid, pBasic->app, 0); if (pConn == NULL) { mError("user:%s, conn:%u is freed and failed to create new since %s", connInfo.user, pBasic->connId, terrstr()); return -1; @@ -345,7 +352,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } } else if (pConn->killed) { mError("user:%s, conn:%u is already killed", connInfo.user, pConn->id); - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn); terrno = TSDB_CODE_MND_INVALID_CONNECTION; return -1; } @@ -369,8 +376,8 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } rspBasic->connId = pConn->id; - rspBasic->totalDnodes = 1; //TODO - rspBasic->onlineDnodes = 1; //TODO + rspBasic->totalDnodes = 1; // TODO + rspBasic->onlineDnodes = 1; // TODO mndGetMnodeEpSet(pMnode, &rspBasic->epSet); mndReleaseConn(pMnode, pConn); @@ -379,7 +386,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb int32_t kvNum = taosHashGetSize(pHbReq->info); if (NULL == pHbReq->info || kvNum <= 0) { - taosArrayPush(pBatchRsp->rsps, &hbRsp); + taosArrayPush(pBatchRsp->rsps, &hbRsp); return TSDB_CODE_SUCCESS; } @@ -396,7 +403,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb switch (kv->key) { case HEARTBEAT_KEY_DBINFO: { - void *rspMsg = NULL; + void * rspMsg = NULL; int32_t rspLen = 0; mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { @@ -406,7 +413,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb break; } case HEARTBEAT_KEY_STBINFO: { - void *rspMsg = NULL; + void * rspMsg = NULL; int32_t rspLen = 0; mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { @@ -457,7 +464,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) { taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq); int32_t tlen = tSerializeSClientHbBatchRsp(NULL, 0, &batchRsp); - void *buf = rpcMallocCont(tlen); + void * buf = rpcMallocCont(tlen); tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp); int32_t rspNum = (int32_t)taosArrayGetSize(batchRsp.rsps); @@ -479,7 +486,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) { } static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) { - SMnode *pMnode = pReq->pNode; + SMnode * pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -513,7 +520,7 @@ static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) { } static int32_t mndProcessKillConnReq(SNodeMsg *pReq) { - SMnode *pMnode = pReq->pNode; + SMnode * pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -545,11 +552,11 @@ static int32_t mndProcessKillConnReq(SNodeMsg *pReq) { } static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pNode; + SMnode * pMnode = pReq->pNode; int32_t numOfRows = 0; SConnObj *pConn = NULL; int32_t cols = 0; - char *pWrite; + char * pWrite; char ipStr[TSDB_IPv4ADDR_LEN + 6]; if (pShow->pIter == NULL) { @@ -604,8 +611,8 @@ static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int } static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pNode; - int32_t numOfRows = 0; + SMnode *pMnode = pReq->pNode; + int32_t numOfRows = 0; #if 0 SConnObj *pConn = NULL; int32_t cols = 0; @@ -703,7 +710,7 @@ static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, i } pShow->numOfRows += numOfRows; -#endif +#endif return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index df7946a0c12a19e2f4684f3714add4c53d87eff5..73583058f1fd599dba0608a1eff99d6702cb99cc 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -34,6 +34,54 @@ extern bool tsStreamSchedV; +int32_t mndConvertRSmaTask(const char* ast, int8_t triggerType, int64_t watermark, char** pStr, int32_t* pLen) { + SNode* pAst = NULL; + SQueryPlan* pPlan = NULL; + terrno = TSDB_CODE_SUCCESS; + + if (nodesStringToNode(ast, &pAst) < 0) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + goto END; + } + + SPlanContext cxt = { + .pAstRoot = pAst, + .topicQuery = false, + .streamQuery = true, + .rSmaQuery = true, + .triggerType = triggerType, + .watermark = watermark, + }; + if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + goto END; + } + + int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); + if (levelNum != 1) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + goto END; + } + SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, 0); + + int32_t opNum = LIST_LENGTH(inner->pNodeList); + if (opNum != 1) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + goto END; + } + + SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); + if (qSubPlanToString(plan, pStr, pLen) < 0) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + goto END; + } + +END: + if (pAst) nodesDestroyNode(pAst); + if (pPlan) nodesDestroyNode(pPlan); + return terrno; +} + int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet, tmsg_t type, int32_t nodeId) { SCoder encoder; tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); @@ -434,7 +482,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib return -1; } - ASSERT(pSub->vgNum == 0); + ASSERT(pSub->vgNum == -1); + + pSub->vgNum = 0; int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); if (levelNum != 1) { @@ -453,6 +503,12 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib } SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); + int64_t unexistKey = -1; + SMqConsumerEpInSub* pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t)); + ASSERT(pEpInSub); + + ASSERT(taosHashGetSize(pSub->consumerHash) == 1); + void* pIter = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); @@ -466,24 +522,41 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib plan->execNode.nodeId = pVgroup->vgId; plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); + SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); + pVgEp->epSet = plan->execNode.epSet; + pVgEp->vgId = plan->execNode.nodeId; + +#if 0 SMqConsumerEp consumerEp = {0}; consumerEp.status = 0; consumerEp.consumerId = -1; consumerEp.epSet = plan->execNode.epSet; consumerEp.vgId = plan->execNode.nodeId; - - mDebug("init subscribption %s, assign vg: %d", pSub->key, consumerEp.vgId); +#endif + + mDebug("init subscribption %s, assign vg: %d", pSub->key, pVgEp->vgId); int32_t msgLen; - if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) { + if (qSubPlanToString(plan, &pVgEp->qmsg, &msgLen) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - taosArrayPush(pSub->unassignedVg, &consumerEp); + taosArrayPush(pEpInSub->vgs, &pVgEp); + + ASSERT(taosHashGetSize(pSub->consumerHash) == 1); + + /*taosArrayPush(pSub->unassignedVg, &consumerEp);*/ } + ASSERT(pEpInSub->vgs->size > 0); + pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t)); + + ASSERT(pEpInSub->vgs->size > 0); + + ASSERT(taosHashGetSize(pSub->consumerHash) == 1); + qDestroyQueryPlan(pPlan); return 0; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 1a14c94640f5dd8b690f4d4d3506f6baaee16f49..f81179201c405c122f41715dfab3e690670c5afc 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -18,7 +18,7 @@ #define SHOW_STEP_SIZE 100 -static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq); +static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq); static void mndFreeShowObj(SShowObj *pShow); static SShowObj *mndAcquireShowObj(SMnode *pMnode, int64_t showId); static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); @@ -47,18 +47,80 @@ void mndCleanupShow(SMnode *pMnode) { } } -static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq) { +static int32_t convertToRetrieveType(char* name, int32_t len) { + int32_t type = -1; + + if (strncasecmp(name, TSDB_INS_TABLE_DNODES, len) == 0) { + type = TSDB_MGMT_TABLE_DNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_MNODES, len) == 0) { + type = TSDB_MGMT_TABLE_MNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_MODULES, len) == 0) { + type = TSDB_MGMT_TABLE_MODULE; + } else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, len) == 0) { + type = TSDB_MGMT_TABLE_QNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_BNODES, len) == 0) { + type = TSDB_MGMT_TABLE_BNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, len) == 0) { + type = TSDB_MGMT_TABLE_SNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, len) == 0) { + type = TSDB_MGMT_TABLE_CLUSTER; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_DATABASES, len) == 0) { + type = TSDB_MGMT_TABLE_DB; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, len) == 0) { + type = TSDB_MGMT_TABLE_FUNC; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, len) == 0) { + // type = TSDB_MGMT_TABLE_INDEX; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, len) == 0) { + type = TSDB_MGMT_TABLE_STB; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, len) == 0) { + type = TSDB_MGMT_TABLE_STREAMS; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) { + type = TSDB_MGMT_TABLE_TABLE; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, len) == 0) { + // type = TSDB_MGMT_TABLE_DIST; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, len) == 0) { + type = TSDB_MGMT_TABLE_USER; + } else if (strncasecmp(name, TSDB_INS_TABLE_LICENCES, len) == 0) { + type = TSDB_MGMT_TABLE_GRANTS; + } else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, len) == 0) { + type = TSDB_MGMT_TABLE_VGROUP; + } else if (strncasecmp(name, TSDB_INS_TABLE_TOPICS, len) == 0) { + type = TSDB_MGMT_TABLE_TOPICS; + } else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, len) == 0) { + type = TSDB_MGMT_TABLE_CONSUMERS; + } else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, len) == 0) { + type = TSDB_MGMT_TABLE_SUBSCRIBES; + } else if (strncasecmp(name, TSDB_INS_TABLE_TRANS, len) == 0) { + type = TSDB_MGMT_TABLE_TRANS; + } else if (strncasecmp(name, TSDB_INS_TABLE_SMAS, len) == 0) { + type = TSDB_MGMT_TABLE_SMAS; + } else if (strncasecmp(name, TSDB_INS_TABLE_CONFIGS, len) == 0) { + type = TSDB_MGMT_TABLE_CONFIGS; + } else if (strncasecmp(name, TSDB_INS_TABLE_CONNS, len) == 0) { + type = TSDB_MGMT_TABLE_CONNS; + } else if (strncasecmp(name, TSDB_INS_TABLE_QUERIES, len) == 0) { + type = TSDB_MGMT_TABLE_QUERIES; + } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, len) == 0) { + type = TSDB_MGMT_TABLE_VNODES; + } else { +// ASSERT(0); + } + + return type; +} + +static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) { SShowMgmt *pMgmt = &pMnode->showMgmt; int64_t showId = atomic_add_fetch_64(&pMgmt->showId, 1); if (showId == 0) atomic_add_fetch_64(&pMgmt->showId, 1); - int32_t size = sizeof(SShowObj) + pReq->payloadLen; + int32_t size = sizeof(SShowObj); + SShowObj showObj = {0}; - showObj.id = showId; + showObj.id = showId; showObj.pMnode = pMnode; - showObj.type = pReq->type; - showObj.payloadLen = pReq->payloadLen; + showObj.type = convertToRetrieveType(pReq->tb, tListLen(pReq->tb)); memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN); int32_t keepTime = tsShellActivityTimer * 6 * 1000; @@ -127,10 +189,6 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) { } if (retrieveReq.showId == 0) { - SShowReq req = {0}; - req.type = retrieveReq.type; - strncpy(req.db, retrieveReq.db, tListLen(req.db)); - STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb) + 1); if (pMeta == NULL) { terrno = TSDB_CODE_MND_INVALID_INFOS_TBL; @@ -138,7 +196,7 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) { return -1; } - pShow = mndCreateShowObj(pMnode, &req); + pShow = mndCreateShowObj(pMnode, &retrieveReq); if (pShow == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to process show-meta req since %s", terrstr()); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 3ead2f26a3ea2f09dcca222504e671dd90a8a0e8..f304e3153ded7f26c586bffcd7f2f249480662aa 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -20,6 +20,7 @@ #include "mndInfoSchema.h" #include "mndMnode.h" #include "mndPerfSchema.h" +#include "mndScheduler.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" @@ -72,7 +73,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags + pStb->numOfSmas) * sizeof(SSchema) + - TSDB_STB_RESERVE_SIZE; + + pStb->commentLen + pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; @@ -93,6 +94,8 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfSmas, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->ast1Len, _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->ast2Len, _OVER) for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; @@ -121,6 +124,12 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { if (pStb->commentLen > 0) { SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, _OVER) } + if (pStb->ast1Len > 0) { + SDB_SET_BINARY(pRaw, dataPos, pStb->pAst1, pStb->ast1Len, _OVER) + } + if (pStb->ast2Len > 0) { + SDB_SET_BINARY(pRaw, dataPos, pStb->pAst2, pStb->ast2Len, _OVER) + } SDB_SET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -173,6 +182,8 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfSmas, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->ast1Len, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->ast2Len, _OVER) pStb->pColumns = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema)); pStb->pTags = taosMemoryCalloc(pStb->numOfTags, sizeof(SSchema)); @@ -210,6 +221,16 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { if (pStb->comment == NULL) goto _OVER; SDB_GET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, _OVER) } + if (pStb->ast1Len > 0) { + pStb->pAst1 = taosMemoryCalloc(pStb->ast1Len, 1); + if (pStb->pAst1 == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pStb->pAst1, pStb->ast1Len, _OVER) + } + if (pStb->ast2Len > 0) { + pStb->pAst2 = taosMemoryCalloc(pStb->ast2Len, 1); + if (pStb->pAst2 == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pStb->pAst2, pStb->ast2Len, _OVER) + } SDB_GET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, _OVER) terrno = 0; @@ -238,6 +259,8 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { taosMemoryFreeClear(pStb->pColumns); taosMemoryFreeClear(pStb->pTags); taosMemoryFreeClear(pStb->comment); + taosMemoryFreeClear(pStb->pAst1); + taosMemoryFreeClear(pStb->pAst2); return 0; } @@ -294,6 +317,30 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { } } + if (pOld->ast1Len < pNew->ast1Len) { + void *pAst1 = taosMemoryMalloc(pNew->ast1Len); + if (pAst1 != NULL) { + taosMemoryFree(pOld->pAst1); + pOld->pAst1 = pAst1; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); + taosWUnLockLatch(&pOld->lock); + } + } + + if (pOld->ast2Len < pNew->ast2Len) { + void *pAst2 = taosMemoryMalloc(pNew->ast2Len); + if (pAst2 != NULL) { + taosMemoryFree(pOld->pAst2); + pOld->pAst2 = pAst2; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); + taosWUnLockLatch(&pOld->lock); + } + } + pOld->updateTime = pNew->updateTime; pOld->version = pNew->version; pOld->nextColId = pNew->nextColId; @@ -302,7 +349,13 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { memcpy(pOld->pColumns, pNew->pColumns, pOld->numOfColumns * sizeof(SSchema)); memcpy(pOld->pTags, pNew->pTags, pOld->numOfTags * sizeof(SSchema)); if (pNew->commentLen != 0) { - memcpy(pOld->comment, pNew->comment, TSDB_STB_COMMENT_LEN); + memcpy(pOld->comment, pNew->comment, pNew->commentLen); + } + if (pNew->ast1Len != 0) { + memcpy(pOld->pAst1, pNew->pAst1, pNew->ast1Len); + } + if (pNew->ast2Len != 0) { + memcpy(pOld->pAst2, pNew->pAst2, pNew->ast2Len); } taosWUnLockLatch(&pOld->lock); return 0; @@ -391,6 +444,25 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt for (int32_t f = 0; f < pRSmaParam->nFuncIds; ++f) { *(pRSmaParam->pFuncIds + f) = pStb->aggregationMethod; } + if (pStb->ast1Len > 0) { + if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &pRSmaParam->qmsg1, &pRSmaParam->qmsg1Len) != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pRSmaParam->pFuncIds); + taosMemoryFreeClear(req.stbCfg.pRSmaParam); + taosMemoryFreeClear(req.stbCfg.pSchema); + return NULL; + } + } + if (pStb->ast2Len > 0) { + int32_t qmsgLen2 = 0; + if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &pRSmaParam->qmsg2, &pRSmaParam->qmsg2Len) != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pRSmaParam->pFuncIds); + taosMemoryFreeClear(pRSmaParam->qmsg1); + taosMemoryFreeClear(req.stbCfg.pRSmaParam); + taosMemoryFreeClear(req.stbCfg.pSchema); + return NULL; + } + } + req.stbCfg.pRSmaParam = pRSmaParam; } @@ -399,6 +471,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt if (pHead == NULL) { if (pRSmaParam) { taosMemoryFreeClear(pRSmaParam->pFuncIds); + taosMemoryFreeClear(pRSmaParam->qmsg1); + taosMemoryFreeClear(pRSmaParam->qmsg2); taosMemoryFreeClear(pRSmaParam); } taosMemoryFreeClear(req.stbCfg.pSchema); @@ -415,6 +489,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt *pContLen = contLen; if (pRSmaParam) { taosMemoryFreeClear(pRSmaParam->pFuncIds); + taosMemoryFreeClear(pRSmaParam->qmsg1); + taosMemoryFreeClear(pRSmaParam->qmsg2); taosMemoryFreeClear(pRSmaParam); } taosMemoryFreeClear(req.stbCfg.pSchema); @@ -646,6 +722,26 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre memcpy(stbObj.comment, pCreate->comment, stbObj.commentLen); } + stbObj.ast1Len = pCreate->ast1Len; + if (stbObj.ast1Len > 0) { + stbObj.pAst1 = taosMemoryCalloc(stbObj.ast1Len, 1); + if (stbObj.pAst1 == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + memcpy(stbObj.pAst1, pCreate->pAst1, stbObj.ast1Len); + } + + stbObj.ast2Len = pCreate->ast2Len; + if (stbObj.ast2Len > 0) { + stbObj.pAst2 = taosMemoryCalloc(stbObj.ast2Len, 1); + if (stbObj.pAst2 == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + memcpy(stbObj.pAst2, pCreate->pAst2, stbObj.ast2Len); + } + stbObj.pColumns = taosMemoryMalloc(stbObj.numOfColumns * sizeof(SSchema)); stbObj.pTags = taosMemoryMalloc(stbObj.numOfTags * sizeof(SSchema)); stbObj.pSmas = taosMemoryMalloc(stbObj.numOfSmas * sizeof(SSchema)); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 58e5b6c65b87ae037e00b32403581b5e5f57734a..1062154e023432f0af070d12efa59ec725a431eb 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -59,7 +59,7 @@ int32_t mndInitStream(SMnode *pMnode) { /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/ // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveStream); - mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextStream); + /*mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextStream);*/ return sdbSetTable(pMnode->pSdb, table); } @@ -247,7 +247,8 @@ static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64 return code; } -int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, int8_t triggerType, int64_t watermark, STrans *pTrans) { +int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, int8_t triggerType, int64_t watermark, + STrans *pTrans) { SNode *pAst = NULL; if (nodesStringToNode(ast, &pAst) < 0) { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 3a31b0abb9ef47e78772d9a8612155b7ed344382..e37bd60e125611c596fc49a9ccf335b375dbe662 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -40,30 +40,30 @@ enum { MQ_SUBSCRIBE_STATUS__DELETED, }; -static int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName); - static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw); static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub); -static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg); -static int32_t mndProcessSubscribeRsp(SNodeMsg *pMsg); -static int32_t mndProcessSubscribeInternalReq(SNodeMsg *pMsg); +static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg); static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pMsg); -static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg); -static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg); -static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg); -static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg); -static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, - const SMqConsumerEp *pConsumerEp); +static int32_t mndSetSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { + SSdbRaw *pRedoRaw = mndSubActionEncode(pSub); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} -static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, - const char *topicName); -static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, - const char *oldTopicName); +static int32_t mndSetSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { + SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} int32_t mndInitSubscribe(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_SUBSCRIBE, @@ -74,242 +74,93 @@ int32_t mndInitSubscribe(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndSubActionUpdate, .deleteFp = (SdbDeleteFp)mndSubActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); - mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp); - mndSetMsgHandle(pMnode, TDMT_VND_MQ_REB_RSP, mndProcessSubscribeInternalRsp); - mndSetMsgHandle(pMnode, TDMT_VND_MQ_CANCEL_CONN_RSP, mndProcessSubscribeInternalRsp); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); - mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessDoRebalanceMsg); + mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndProcessSubscribeInternalRsp); + mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq); return sdbSetTable(pMnode->pSdb, table); } -static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *cgroup) { - SMqSubscribeObj *pSub = tNewSubscribeObj(); +static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey) { + SMqSubscribeObj *pSub = tNewSubscribeObj(subKey); if (pSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - char key[TSDB_SUBSCRIBE_KEY_LEN]; - mndMakeSubscribeKey(key, cgroup, pTopic->name); - strcpy(pSub->key, key); + pSub->subType = pTopic->subType; + pSub->withTbName = pTopic->withTbName; + pSub->withSchema = pTopic->withSchema; + pSub->withTag = pTopic->withTag; + pSub->withTagSchema = pTopic->withTagSchema; + + ASSERT(taosHashGetSize(pSub->consumerHash) == 1); if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { - tDeleteSMqSubscribeObj(pSub); + tDeleteSubscribeObj(pSub); taosMemoryFree(pSub); return NULL; } - // TODO: disable alter subscribed table + ASSERT(taosHashGetSize(pSub->consumerHash) == 1); + return pSub; } -static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp, - const char *topicName) { - SMqMVRebReq req = { - .vgId = pConsumerEp->vgId, - .oldConsumerId = pConsumerEp->oldConsumerId, - .newConsumerId = pConsumerEp->consumerId, - }; - req.topic = strdup(topicName); - - int32_t tlen = tEncodeSMqMVRebReq(NULL, &req); - void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); +static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscribeObj *pSub, + const SMqRebOutputVg *pRebVg) { + SMqRebVgReq req = {0}; + req.oldConsumerId = pRebVg->oldConsumerId; + req.newConsumerId = pRebVg->newConsumerId; + req.vgId = pRebVg->pVgEp->vgId; + req.qmsg = pRebVg->pVgEp->qmsg; + req.subType = pSub->subType; + req.withTbName = pSub->withTbName; + req.withSchema = pSub->withSchema; + req.withTag = pSub->withTag; + req.withTagSchema = pSub->withTagSchema; + strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); + + int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); + void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } SMsgHead *pMsgHead = (SMsgHead *)buf; - pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen); - pMsgHead->vgId = htonl(pConsumerEp->vgId); + pMsgHead->contLen = htonl(tlen); + pMsgHead->vgId = htonl(pRebVg->pVgEp->vgId); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncodeSMqMVRebReq(&abuf, &req); - taosMemoryFree(req.topic); - + tEncodeSMqRebVgReq(&abuf, &req); *pBuf = buf; *pLen = tlen; return 0; } -static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, - const char *topicName) { - ASSERT(pConsumerEp->oldConsumerId != -1); +static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub, + const SMqRebOutputVg *pRebVg) { + ASSERT(pRebVg->oldConsumerId != pRebVg->newConsumerId); void *buf; int32_t tlen; - if (mndBuildRebalanceMsg(&buf, &tlen, pConsumerEp, topicName) < 0) { + if (mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg) < 0) { return -1; } - int32_t vgId = pConsumerEp->vgId; + int32_t vgId = pRebVg->pVgEp->vgId; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgObj); action.pCont = buf; - action.contLen = sizeof(SMsgHead) + tlen; - action.msgType = TDMT_VND_MQ_REB; + action.contLen = tlen; + action.msgType = TDMT_VND_MQ_VG_CHANGE; mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(buf); return -1; } - - return 0; -} - -static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp, - const char *oldTopicName) { - SMqCancelConnReq req = {0}; - req.consumerId = pConsumerEp->consumerId; - req.vgId = pConsumerEp->vgId; - req.epoch = pConsumerEp->epoch; - strcpy(req.topicName, oldTopicName); - - int32_t tlen = tEncodeSMqCancelConnReq(NULL, &req); - void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - SMsgHead *pMsgHead = (SMsgHead *)buf; - - pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen); - pMsgHead->vgId = htonl(pConsumerEp->vgId); - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncodeSMqCancelConnReq(&abuf, &req); - *pBuf = buf; - *pLen = tlen; - return 0; -} - -static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, - const char *oldTopicName) { - void *buf; - int32_t tlen; - if (mndBuildCancelConnReq(&buf, &tlen, pConsumerEp, oldTopicName) < 0) { - return -1; - } - - int32_t vgId = pConsumerEp->vgId; - SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); - - STransAction action = {0}; - action.epSet = mndGetVgroupEpset(pMnode, pVgObj); - action.pCont = buf; - action.contLen = sizeof(SMsgHead) + tlen; - action.msgType = TDMT_VND_MQ_CANCEL_CONN; - - mndReleaseVgroup(pMnode, pVgObj); - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(buf); - return -1; - } - - return 0; -} - -static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { - SMnode *pMnode = pMsg->pNode; - SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; - SMqCMGetSubEpRsp rsp = {0}; - int64_t consumerId = be64toh(pReq->consumerId); - int32_t epoch = ntohl(pReq->epoch); - - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pNode, consumerId); - if (pConsumer == NULL) { - terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; - return -1; - } - // TODO add lock - ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); - int32_t serverEpoch = pConsumer->epoch; - - // TODO - int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus); - mDebug("consumer %ld epoch(%d) try to get sub ep, server epoch %d, old val: %d", consumerId, epoch, serverEpoch, - hbStatus); - atomic_store_32(&pConsumer->hbStatus, 0); - /*SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);*/ - /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/ - /*sdbWrite(pMnode->pSdb, pConsumerRaw);*/ - - strcpy(rsp.cgroup, pReq->cgroup); - if (epoch != serverEpoch) { - mInfo("send new assignment to consumer %ld, consumer epoch %d, server epoch %d", pConsumer->consumerId, epoch, - serverEpoch); - mDebug("consumer %ld try r lock", consumerId); - taosRLockLatch(&pConsumer->lock); - mDebug("consumer %ld r locked", consumerId); - SArray *pTopics = pConsumer->currentTopics; - int32_t sz = taosArrayGetSize(pTopics); - rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); - for (int32_t i = 0; i < sz; i++) { - char *topicName = taosArrayGetP(pTopics, i); - SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topicName); - ASSERT(pSub); - int32_t csz = taosArrayGetSize(pSub->consumers); - // TODO: change to bsearch - for (int32_t j = 0; j < csz; j++) { - SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j); - if (consumerId == pSubConsumer->consumerId) { - int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo); - mInfo("topic %s has %d vg", topicName, serverEpoch); - - SMqSubTopicEp topicEp; - strcpy(topicEp.topic, topicName); - - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topicName); - ASSERT(pTopic != NULL); - topicEp.schema = pTopic->schema; - mndReleaseTopic(pMnode, pTopic); - - topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp)); - for (int32_t k = 0; k < vgsz; k++) { - char offsetKey[TSDB_PARTITION_KEY_LEN]; - SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k); - SMqSubVgEp vgEp = { - .epSet = pConsumerEp->epSet, - .vgId = pConsumerEp->vgId, - .offset = -1, - }; - mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId); - SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey); - if (pOffsetObj != NULL) { - vgEp.offset = pOffsetObj->offset; - mndReleaseOffset(pMnode, pOffsetObj); - } - taosArrayPush(topicEp.vgs, &vgEp); - } - taosArrayPush(rsp.topics, &topicEp); - break; - } - } - mndReleaseSubscribe(pMnode, pSub); - } - taosRUnLockLatch(&pConsumer->lock); - mDebug("consumer %ld r unlock", consumerId); - } - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp); - void *buf = rpcMallocCont(tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP; - ((SMqRspHead *)buf)->epoch = serverEpoch; - ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId; - - void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); - tDeleteSMqCMGetSubEpRsp(&rsp); - mndReleaseConsumer(pMnode, pConsumer); - pMsg->pRsp = buf; - pMsg->rspLen = tlen; return 0; } @@ -337,275 +188,351 @@ static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { return pRebSub; } -static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { - SMnode *pMnode = pMsg->pNode; - SSdb *pSdb = pMnode->pSdb; - SMqConsumerObj *pConsumer; - void *pIter = NULL; - SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg)); - pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); +static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { + if (pInput->pTopic != NULL) { + // create subscribe + pOutput->pSub = mndCreateSub(pMnode, pInput->pTopic, pInput->pRebInfo->key); + ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) == 1); + } else { + pOutput->pSub = tCloneSubscribeObj(pInput->pOldSub); + } + int32_t totalVgNum = pOutput->pSub->vgNum; + + mInfo("mq rebalance subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum); + + // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg + SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + + ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0); + // 2. check and get actual removed consumers, put their vg into hash + int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers); + int32_t actualRemoved = 0; + for (int32_t i = 0; i < removedNum; i++) { + int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); + ASSERT(consumerId > 0); + SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); + ASSERT(pEpInSub); + if (pEpInSub) { + ASSERT(consumerId == pEpInSub->consumerId); + actualRemoved++; + int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs); + for (int32_t j = 0; j < consumerVgNum; j++) { + SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, j); + SMqRebOutputVg outputVg = { + .oldConsumerId = consumerId, + .newConsumerId = -1, + .pVgEp = pVgEp, + }; + taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); + } + taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); + // put into removed + taosArrayPush(pOutput->removedConsumers, &consumerId); + } + } + ASSERT(removedNum == actualRemoved); + ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0); + + // if previously no consumer, there are vgs not assigned + { + int64_t unexistKey = -1; + SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t)); + ASSERT(pEpInSub); + int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs); + for (int32_t i = 0; i < consumerVgNum; i++) { + SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs); + SMqRebOutputVg rebOutput = { + .oldConsumerId = -1, + .newConsumerId = -1, + .pVgEp = pVgEp, + }; + taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); + } + } + // 3. calc vg number of each consumer + int32_t oldSz = 0; + if (pInput->pOldSub) { + oldSz = taosHashGetSize(pInput->pOldSub->consumerHash) - 1; + } + int32_t afterRebConsumerNum = + oldSz + taosArrayGetSize(pInput->pRebInfo->newConsumers) - taosArrayGetSize(pInput->pRebInfo->removedConsumers); + int32_t minVgCnt = 0; + int32_t imbConsumerNum = 0; + // calc num + if (afterRebConsumerNum) { + minVgCnt = totalVgNum / afterRebConsumerNum; + imbConsumerNum = totalVgNum % afterRebConsumerNum; + } + + // 4. first scan: remove consumer more than wanted, put to remove hash + int32_t imbCnt = 0; + void *pIter = NULL; while (1) { - pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); + pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); if (pIter == NULL) break; - int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); - if (hbStatus > MND_SUBSCRIBE_REBALANCE_CNT) { - int32_t old = - atomic_val_compare_exchange_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE, MQ_CONSUMER_STATUS__LOST); - if (old == MQ_CONSUMER_STATUS__ACTIVE) { - // get all topics of that topic - int32_t sz = taosArrayGetSize(pConsumer->currentTopics); - for (int32_t i = 0; i < sz; i++) { - char *topic = taosArrayGetP(pConsumer->currentTopics, i); - char key[TSDB_SUBSCRIBE_KEY_LEN]; - mndMakeSubscribeKey(key, pConsumer->cgroup, topic); - SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); - taosArrayPush(pRebSub->lostConsumers, &pConsumer->consumerId); + SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; + if (pEpInSub->consumerId == -1) continue; + ASSERT(pEpInSub->consumerId > 0); + int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs); + // all old consumers still existing are touched + // TODO optimize: touch only consumer whose vgs changed + taosArrayPush(pOutput->touchedConsumers, &pEpInSub->consumerId); + if (consumerVgNum > minVgCnt) { + if (imbCnt < imbConsumerNum) { + if (consumerVgNum == minVgCnt + 1) { + continue; + } else { + // pop until equal minVg + 1 + while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt + 1) { + SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs); + SMqRebOutputVg outputVg = { + .oldConsumerId = pEpInSub->consumerId, + .newConsumerId = -1, + .pVgEp = pVgEp, + }; + taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); + } + imbCnt++; } - } - } - int32_t status = atomic_load_32(&pConsumer->status); - if (status == MQ_CONSUMER_STATUS__INIT || status == MQ_CONSUMER_STATUS__MODIFY) { - SArray *rebSubs; - if (status == MQ_CONSUMER_STATUS__INIT) { - rebSubs = pConsumer->currentTopics; } else { - rebSubs = pConsumer->recentRemovedTopics; - } - int32_t sz = taosArrayGetSize(rebSubs); - for (int32_t i = 0; i < sz; i++) { - char *topic = taosArrayGetP(rebSubs, i); - char key[TSDB_SUBSCRIBE_KEY_LEN]; - mndMakeSubscribeKey(key, pConsumer->cgroup, topic); - SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); - if (status == MQ_CONSUMER_STATUS__INIT) { - taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId); - } else if (status == MQ_CONSUMER_STATUS__MODIFY) { - taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); - } - } - if (status == MQ_CONSUMER_STATUS__MODIFY) { - int32_t removeSz = taosArrayGetSize(pConsumer->recentRemovedTopics); - for (int32_t i = 0; i < removeSz; i++) { - char *topicName = taosArrayGetP(pConsumer->recentRemovedTopics, i); - taosMemoryFree(topicName); + // pop until equal minVg + while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt) { + SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs); + SMqRebOutputVg outputVg = { + .oldConsumerId = pEpInSub->consumerId, + .newConsumerId = -1, + .pVgEp = pVgEp, + }; + taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); } - taosArrayClear(pConsumer->recentRemovedTopics); } } } - if (taosHashGetSize(pRebMsg->rebSubHash) != 0) { - mInfo("mq rebalance will be triggered"); - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_MQ_DO_REBALANCE, - .pCont = pRebMsg, - .contLen = sizeof(SMqDoRebalanceMsg), - }; - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); - } else { - taosHashCleanup(pRebMsg->rebSubHash); - rpcFreeCont(pRebMsg); - } - return 0; -} -static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { - SMnode *pMnode = pMsg->pNode; - SMqDoRebalanceMsg *pReq = pMsg->rpcMsg.pCont; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_REBALANCE, &pMsg->rpcMsg); - void *pIter = NULL; - - mInfo("mq rebalance start"); + // 5. add new consumer into sub + { + int32_t consumerNum = taosArrayGetSize(pInput->pRebInfo->newConsumers); + for (int32_t i = 0; i < consumerNum; i++) { + int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); + ASSERT(consumerId > 0); + SMqConsumerEpInSub newConsumerEp; + newConsumerEp.consumerId = consumerId; + newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); + taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, + sizeof(SMqConsumerEpInSub)); + /*SMqConsumerEpInSub *pTestNew = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));*/ + /*ASSERT(pTestNew->consumerId == consumerId);*/ + /*ASSERT(pTestNew->vgs == newConsumerEp.vgs);*/ + taosArrayPush(pOutput->newConsumers, &consumerId); + } + } + // 6. second scan: find consumer do not have enough vg, extract from temporary hash and assign to new consumer. + // All related vg should be put into rebVgs + SMqRebOutputVg *pRebVg = NULL; + void *pRemovedIter = NULL; + pIter = NULL; while (1) { - pIter = taosHashIterate(pReq->rebSubHash, pIter); + pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); if (pIter == NULL) break; - SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter; - SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); - taosMemoryFreeClear(pRebSub->key); - - mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum, - (int32_t)taosArrayGetSize(pSub->unassignedVg)); - - // remove lost consumer - for (int32_t i = 0; i < taosArrayGetSize(pRebSub->lostConsumers); i++) { - int64_t lostConsumerId = *(int64_t *)taosArrayGet(pRebSub->lostConsumers, i); - - mInfo("mq remove lost consumer %" PRId64 "", lostConsumerId); + SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; + if (pEpInSub->consumerId == -1) continue; + ASSERT(pEpInSub->consumerId > 0); + + // push until equal minVg + while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt) { + // iter hash and find one vg + pRemovedIter = taosHashIterate(pHash, pRemovedIter); + ASSERT(pRemovedIter); + pRebVg = (SMqRebOutputVg *)pRemovedIter; + // push + taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp); + pRebVg->newConsumerId = pEpInSub->consumerId; + taosArrayPush(pOutput->rebVgs, pRebVg); + } + } - for (int32_t j = 0; j < taosArrayGetSize(pSub->consumers); j++) { - SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j); - if (pSubConsumer->consumerId == lostConsumerId) { - taosArrayAddAll(pSub->unassignedVg, pSubConsumer->vgInfo); - taosArrayPush(pSub->lostConsumers, pSubConsumer); - taosArrayRemove(pSub->consumers, j); - break; - } - } + // 7. handle unassigned vg + if (taosHashGetSize(pOutput->pSub->consumerHash) != 1) { + // if has consumer, assign all left vg + while (1) { + pRemovedIter = taosHashIterate(pHash, pRemovedIter); + if (pRemovedIter == NULL) break; + pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); + ASSERT(pIter); + pRebVg = (SMqRebOutputVg *)pRemovedIter; + SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; + if (pEpInSub->consumerId == -1) continue; + ASSERT(pEpInSub->consumerId > 0); + taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp); + pRebVg->newConsumerId = pEpInSub->consumerId; + taosArrayPush(pOutput->rebVgs, pRebVg); } + } else { + // if all consumer is removed, put all vg into unassigned + int64_t unexistKey = -1; + SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t)); + ASSERT(pEpInSub); + ASSERT(pEpInSub->consumerId == -1); + + pIter = NULL; + SMqRebOutputVg *pRebOutput = NULL; + while (1) { + pIter = taosHashIterate(pHash, pIter); + if (pIter == NULL) break; + pRebOutput = (SMqRebOutputVg *)pIter; + ASSERT(pRebOutput->newConsumerId == -1); + taosArrayPush(pEpInSub->vgs, &pRebOutput->pVgEp); + taosArrayPush(pOutput->rebVgs, pRebOutput); + } + } - // calculate rebalance - int32_t consumerNum = taosArrayGetSize(pSub->consumers); - if (consumerNum != 0) { - int32_t vgNum = pSub->vgNum; - int32_t vgEachConsumer = vgNum / consumerNum; - int32_t imbalanceVg = vgNum % consumerNum; - - // iterate all consumers, set unassignedVgStash - for (int32_t i = 0; i < consumerNum; i++) { - SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i); - int32_t vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo); - int32_t vgThisConsumerAfterRb; - if (i < imbalanceVg) - vgThisConsumerAfterRb = vgEachConsumer + 1; - else - vgThisConsumerAfterRb = vgEachConsumer; - - mInfo("mq consumer:%" PRId64 ", connectted vgroup number change from %d to %d", pSubConsumer->consumerId, - vgThisConsumerBeforeRb, vgThisConsumerAfterRb); - - while (taosArrayGetSize(pSubConsumer->vgInfo) > vgThisConsumerAfterRb) { - SMqConsumerEp *pConsumerEp = taosArrayPop(pSubConsumer->vgInfo); - ASSERT(pConsumerEp != NULL); - ASSERT(pConsumerEp->consumerId == pSubConsumer->consumerId); - taosArrayPush(pSub->unassignedVg, pConsumerEp); - mDebug("mq rebalance: vg %d push to unassignedVg", pConsumerEp->vgId); - } + // 8. generate logs - SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId); - mDebug("consumer %ld try w lock", pRebConsumer->consumerId); - taosWLockLatch(&pRebConsumer->lock); - mDebug("consumer %ld w locked", pRebConsumer->consumerId); - int32_t status = atomic_load_32(&pRebConsumer->status); - if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb || - (vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) || - (vgThisConsumerAfterRb == 0 && status != MQ_CONSUMER_STATUS__LOST)) { - /*if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb) {*/ - /*pRebConsumer->epoch++;*/ - /*}*/ - if (vgThisConsumerAfterRb != 0) { - atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE); - } else { - atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE); - } + // 9. clear + taosHashCleanup(pHash); - mInfo("mq consumer:%" PRId64 ", status change from %d to %d", pRebConsumer->consumerId, status, - pRebConsumer->status); + return 0; +} - SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer); - sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); - mndTransAppendCommitlog(pTrans, pConsumerRaw); - } - taosWUnLockLatch(&pRebConsumer->lock); - mDebug("consumer %ld w unlock", pRebConsumer->consumerId); - mndReleaseConsumer(pMnode, pRebConsumer); - } +static int32_t mndPersistRebResult(SMnode *pMnode, SNodeMsg *pMsg, const SMqRebOutputObj *pOutput) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_REBALANCE, &pMsg->rpcMsg); + if (pTrans == NULL) { + return -1; + } + // make txn: + // 1. redo action: action to all vg + const SArray *rebVgs = pOutput->rebVgs; + int32_t vgNum = taosArrayGetSize(rebVgs); + for (int32_t i = 0; i < vgNum; i++) { + SMqRebOutputVg *pRebVg = taosArrayGet(rebVgs, i); + if (mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub, pRebVg) < 0) { + goto REB_FAIL; + } + } - // assign to vgroup - if (taosArrayGetSize(pSub->unassignedVg) != 0) { - for (int32_t i = 0; i < consumerNum; i++) { - SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i); - int32_t vgThisConsumerAfterRb; - if (i < imbalanceVg) - vgThisConsumerAfterRb = vgEachConsumer + 1; - else - vgThisConsumerAfterRb = vgEachConsumer; - - while (taosArrayGetSize(pSubConsumer->vgInfo) < vgThisConsumerAfterRb) { - SMqConsumerEp *pConsumerEp = taosArrayPop(pSub->unassignedVg); - mDebug("mq rebalance: vg %d pop from unassignedVg", pConsumerEp->vgId); - ASSERT(pConsumerEp != NULL); - - pConsumerEp->oldConsumerId = pConsumerEp->consumerId; - pConsumerEp->consumerId = pSubConsumer->consumerId; - // TODO - pConsumerEp->epoch = 0; - taosArrayPush(pSubConsumer->vgInfo, pConsumerEp); - - char topic[TSDB_TOPIC_FNAME_LEN]; - char cgroup[TSDB_CGROUP_LEN]; - mndSplitSubscribeKey(pSub->key, topic, cgroup); - if (pConsumerEp->oldConsumerId == -1) { - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - - mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 " cgroup: %s", pConsumerEp->vgId, - topic, pConsumerEp->consumerId, cgroup); - - mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); - mndReleaseTopic(pMnode, pTopic); - } else { - mInfo("mq rebalance: assign vgroup %d, from consumer %" PRId64 " to consumer %" PRId64 "", - pConsumerEp->vgId, pConsumerEp->oldConsumerId, pConsumerEp->consumerId); - - mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp, topic); - } - } - } - } - ASSERT(taosArrayGetSize(pSub->unassignedVg) == 0); + // 2. redo log: subscribe and vg assignment + // subscribe + if (mndSetSubRedoLogs(pMnode, pTrans, pOutput->pSub) != 0) { + goto REB_FAIL; + } - // TODO: log rebalance statistics - SSdbRaw *pSubRaw = mndSubActionEncode(pSub); - sdbSetRawStatus(pSubRaw, SDB_STATUS_READY); - mndTransAppendRedolog(pTrans, pSubRaw); + // 3. commit log: consumer to update status and epoch + // 3.1 set touched consumer + int32_t consumerNum = taosArrayGetSize(pOutput->touchedConsumers); + for (int32_t i = 0; i < consumerNum; i++) { + int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->touchedConsumers, i); + SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); + SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); + pConsumerNew->updateType = CONSUMER_UPDATE__TOUCH; + mndReleaseConsumer(pMnode, pConsumerOld); + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { + goto REB_FAIL; } - mndReleaseSubscribe(pMnode, pSub); } - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("mq-rebalance-trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - taosHashCleanup(pReq->rebSubHash); - mndTransDrop(pTrans); - return -1; + // 3.2 set new consumer + consumerNum = taosArrayGetSize(pOutput->newConsumers); + for (int32_t i = 0; i < consumerNum; i++) { + int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->newConsumers, i); + ASSERT(consumerId > 0); + SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); + SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); + pConsumerNew->updateType = CONSUMER_UPDATE__ADD; + char *topic = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); + char cgroup[TSDB_CGROUP_LEN]; + mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup); + taosArrayPush(pConsumerNew->rebNewTopics, &topic); + mndReleaseConsumer(pMnode, pConsumerOld); + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { + goto REB_FAIL; + } } - taosHashCleanup(pReq->rebSubHash); + // 3.3 set removed consumer + consumerNum = taosArrayGetSize(pOutput->removedConsumers); + for (int32_t i = 0; i < consumerNum; i++) { + int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->removedConsumers, i); + ASSERT(consumerId > 0); + SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); + SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); + pConsumerNew->updateType = CONSUMER_UPDATE__REMOVE; + char *topic = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); + char cgroup[TSDB_CGROUP_LEN]; + mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup); + taosArrayPush(pConsumerNew->rebRemovedTopics, &topic); + mndReleaseConsumer(pMnode, pConsumerOld); + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { + goto REB_FAIL; + } + } + // 4. commit log: modification log + if (mndTransPrepare(pMnode, pTrans) != 0) goto REB_FAIL; + mndTransDrop(pTrans); return 0; + +REB_FAIL: + mndTransDrop(pTrans); + return -1; } -static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, - const SMqConsumerEp *pConsumerEp) { - ASSERT(pConsumerEp->oldConsumerId == -1); - int32_t vgId = pConsumerEp->vgId; - - SMqSetCVgReq req = { - .vgId = vgId, - .consumerId = pConsumerEp->consumerId, - .sql = pTopic->sql, - .physicalPlan = pTopic->physicalPlan, - .qmsg = pConsumerEp->qmsg, - }; - - strcpy(req.cgroup, cgroup); - strcpy(req.topicName, pTopic->name); - int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); - void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } +static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; + SMqDoRebalanceMsg *pReq = pMsg->rpcMsg.pCont; + void *pIter = NULL; - SMsgHead *pMsgHead = (SMsgHead *)buf; + mInfo("mq rebalance start"); - pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen); - pMsgHead->vgId = htonl(vgId); + while (1) { + pIter = taosHashIterate(pReq->rebSubHash, pIter); + if (pIter == NULL) break; + SMqRebInputObj rebInput = {0}; - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncodeSMqSetCVgReq(&abuf, &req); + SMqRebOutputObj rebOutput = {0}; + rebOutput.newConsumers = taosArrayInit(0, sizeof(void *)); + rebOutput.removedConsumers = taosArrayInit(0, sizeof(void *)); + rebOutput.touchedConsumers = taosArrayInit(0, sizeof(void *)); + rebOutput.rebVgs = taosArrayInit(0, sizeof(SMqRebOutputVg)); - SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); + SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter; + SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); - STransAction action = {0}; - action.epSet = mndGetVgroupEpset(pMnode, pVgObj); - action.pCont = buf; - action.contLen = sizeof(SMsgHead) + tlen; - action.msgType = TDMT_VND_MQ_SET_CONN; + if (pSub == NULL) { + // split sub key and extract topic + char topic[TSDB_TOPIC_FNAME_LEN]; + char cgroup[TSDB_CGROUP_LEN]; + mndSplitSubscribeKey(pRebSub->key, topic, cgroup); + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); + ASSERT(pTopic); + taosRLockLatch(&pTopic->lock); + rebInput.pTopic = pTopic; + } - mndReleaseVgroup(pMnode, pVgObj); - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(buf); - return -1; + rebInput.pRebInfo = pRebSub; + rebInput.pOldSub = pSub; + + // TODO replace assert with error check + ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0); + // if add more consumer to balanced subscribe, + // possibly no vg is changed + /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/ + ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0); + + if (rebInput.pTopic) { + SMqTopicObj *pTopic = (SMqTopicObj *)rebInput.pTopic; + taosRUnLockLatch(&pTopic->lock); + mndReleaseTopic(pMnode, pTopic); + } else { + mndReleaseSubscribe(pMnode, pSub); + } } + + // reset flag + atomic_store_8(pReq->mqInReb, 0); + mInfo("mq rebalance completed successfully"); + taosHashCleanup(pReq->rebSubHash); + return 0; } @@ -697,16 +624,23 @@ static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *pSub) { static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *pSub) { mTrace("subscribe:%s, perform delete action", pSub->key); - tDeleteSMqSubscribeObj(pSub); + tDeleteSubscribeObj(pSub); return 0; } static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub) { mTrace("subscribe:%s, perform update action", pOldSub->key); + taosWLockLatch(&pOldSub->lock); + + SHashObj *tmp = pOldSub->consumerHash; + pOldSub->consumerHash = pNewSub->consumerHash; + pNewSub->consumerHash = tmp; + + taosWUnLockLatch(&pOldSub->lock); return 0; } -static int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName) { +int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName) { int32_t tlen = strlen(cgroup); memcpy(key, cgroup, tlen); key[tlen] = TMQ_SEPARATOR; @@ -739,6 +673,7 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) { sdbRelease(pSdb, pSub); } +#if 0 static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { SMnode *pMnode = pMsg->pNode; char *msgStr = pMsg->rpcMsg.pCont; @@ -901,6 +836,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { if (!createConsumer) mndReleaseConsumer(pMnode, pConsumer); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } +#endif static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index d6c1b6c94fae8bb5a1f098c1eb956579b0b6e7cb..e445022548711a4c201572c8617672e59af48cd4 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -21,23 +21,66 @@ #include "thttp.h" #include "tjson.h" -#define TELEMETRY_SERVER "telemetry.taosdata.com" -#define TELEMETRY_PORT 80 +typedef struct { + int64_t numOfDnode; + int64_t numOfMnode; + int64_t numOfVgroup; + int64_t numOfDatabase; + int64_t numOfSuperTable; + int64_t numOfChildTable; + int64_t numOfNormalTable; + int64_t numOfColumn; + int64_t totalPoints; + int64_t totalStorage; + int64_t compStorage; +} SMnodeStat; + +static void mndGetStat(SMnode* pMnode, SMnodeStat* pStat) { + memset(pStat, 0, sizeof(SMnodeStat)); + + SSdb* pSdb = pMnode->pSdb; + pStat->numOfDnode = sdbGetSize(pSdb, SDB_DNODE); + pStat->numOfMnode = sdbGetSize(pSdb, SDB_MNODE); + pStat->numOfVgroup = sdbGetSize(pSdb, SDB_VGROUP); + pStat->numOfDatabase = sdbGetSize(pSdb, SDB_DB); + pStat->numOfSuperTable = sdbGetSize(pSdb, SDB_STB); + + void* pIter = NULL; + while (1) { + SVgObj* pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); + if (pIter == NULL) break; + + pStat->numOfChildTable += pVgroup->numOfTables; + pStat->numOfColumn += pVgroup->numOfTimeSeries; + pStat->totalPoints += pVgroup->pointsWritten; + pStat->totalStorage += pVgroup->totalStorage; + pStat->compStorage += pVgroup->compStorage; + + sdbRelease(pSdb, pVgroup); + } + + pStat->numOfChildTable = 100; + pStat->numOfColumn = 200; + pStat->totalPoints = 300; + pStat->totalStorage = 400; + pStat->compStorage = 500; +} static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) { - SMnodeLoad load = {0}; - mndGetLoad(pMnode, &load); - - tjsonAddDoubleToObject(pJson, "numOfDnode", load.numOfDnode); - tjsonAddDoubleToObject(pJson, "numOfMnode", load.numOfMnode); - tjsonAddDoubleToObject(pJson, "numOfVgroup", load.numOfVgroup); - tjsonAddDoubleToObject(pJson, "numOfDatabase", load.numOfDatabase); - tjsonAddDoubleToObject(pJson, "numOfSuperTable", load.numOfSuperTable); - tjsonAddDoubleToObject(pJson, "numOfChildTable", load.numOfChildTable); - tjsonAddDoubleToObject(pJson, "numOfColumn", load.numOfColumn); - tjsonAddDoubleToObject(pJson, "numOfPoint", load.totalPoints); - tjsonAddDoubleToObject(pJson, "totalStorage", load.totalStorage); - tjsonAddDoubleToObject(pJson, "compStorage", load.compStorage); + SMnodeStat mstat = {0}; + mndGetStat(pMnode, &mstat); + + tjsonAddDoubleToObject(pJson, "numOfDnode", mstat.numOfDnode); + tjsonAddDoubleToObject(pJson, "numOfMnode", mstat.numOfMnode); + tjsonAddDoubleToObject(pJson, "numOfVgroup", mstat.numOfVgroup); + tjsonAddDoubleToObject(pJson, "numOfDatabase", mstat.numOfDatabase); + tjsonAddDoubleToObject(pJson, "numOfSuperTable", mstat.numOfSuperTable); + tjsonAddDoubleToObject(pJson, "numOfChildTable", mstat.numOfChildTable); + tjsonAddDoubleToObject(pJson, "numOfColumn", mstat.numOfColumn); + tjsonAddDoubleToObject(pJson, "numOfPoint", mstat.totalPoints); + tjsonAddDoubleToObject(pJson, "totalStorage", mstat.totalStorage); + tjsonAddDoubleToObject(pJson, "compStorage", mstat.compStorage); } static char* mndBuildTelemetryReport(SMnode* pMnode) { @@ -82,12 +125,14 @@ static char* mndBuildTelemetryReport(SMnode* pMnode) { static int32_t mndProcessTelemTimer(SNodeMsg* pReq) { SMnode* pMnode = pReq->pNode; STelemMgmt* pMgmt = &pMnode->telemMgmt; - if (!pMgmt->enable) return 0; + if (!tsEnableTelem) return 0; taosWLockLatch(&pMgmt->lock); char* pCont = mndBuildTelemetryReport(pMnode); if (pCont != NULL) { - taosSendHttpReport(TELEMETRY_SERVER, TELEMETRY_PORT, pCont, strlen(pCont), HTTP_FLAT); + if (taosSendHttpReport(tsTelemServer, tsTelemPort, pCont, strlen(pCont), HTTP_FLAT) != 0) { + mError("failed to send telemetry msg"); + } taosMemoryFree(pCont); } taosWUnLockLatch(&pMgmt->lock); @@ -98,7 +143,6 @@ int32_t mndInitTelem(SMnode* pMnode) { STelemMgmt* pMgmt = &pMnode->telemMgmt; taosInitRWLatch(&pMgmt->lock); - pMgmt->enable = tsEnableTelemetryReporting; taosGetEmail(pMgmt->email, sizeof(pMgmt->email)); mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index b9c42fe899a2bd798ba039b514483b3e38a8c907..f17b8a4d88e32fbb3ba2bbe67abd6c5efd0a6361 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -35,7 +35,7 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq); static int32_t mndProcessDropTopicReq(SNodeMsg *pReq); static int32_t mndProcessDropTopicInRsp(SNodeMsg *pRsp); -static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); int32_t mndInitTopic(SMnode *pMnode) { @@ -51,7 +51,7 @@ int32_t mndInitTopic(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicReq); mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndProcessDropTopicInRsp); - // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextTopic); return sdbSetTable(pMnode->pSdb, table); @@ -76,7 +76,13 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->uid, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, TOPIC_ENCODE_OVER); + SDB_SET_INT64(pRaw, dataPos, pTopic->subDbUid, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER); + SDB_SET_INT8(pRaw, dataPos, pTopic->subType, TOPIC_ENCODE_OVER); + SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER); + SDB_SET_INT8(pRaw, dataPos, pTopic->withSchema, TOPIC_ENCODE_OVER); + SDB_SET_INT8(pRaw, dataPos, pTopic->withTag, TOPIC_ENCODE_OVER); + SDB_SET_INT8(pRaw, dataPos, pTopic->withTagSchema, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->astLen, TOPIC_ENCODE_OVER); @@ -134,7 +140,13 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pTopic->updateTime, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->uid, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER); + SDB_GET_INT64(pRaw, dataPos, &pTopic->subDbUid, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER); + SDB_GET_INT8(pRaw, dataPos, &pTopic->subType, TOPIC_DECODE_OVER); + SDB_GET_INT8(pRaw, dataPos, &pTopic->withTbName, TOPIC_DECODE_OVER); + SDB_GET_INT8(pRaw, dataPos, &pTopic->withSchema, TOPIC_DECODE_OVER); + SDB_GET_INT8(pRaw, dataPos, &pTopic->withTag, TOPIC_DECODE_OVER); + SDB_GET_INT8(pRaw, dataPos, &pTopic->withTagSchema, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char)); @@ -254,33 +266,13 @@ static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; return -1; } - return 0; -} - -#if 0 -static int32_t mndGetPlanString(const SCMCreateTopicReq *pCreate, char **pStr) { - if (NULL == pCreate->ast) { - return TSDB_CODE_SUCCESS; - } - - SNode *pAst = NULL; - int32_t code = nodesStringToNode(pCreate->ast, &pAst); - SQueryPlan *pPlan = NULL; - if (TSDB_CODE_SUCCESS == code) { - SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; - code = qCreateQueryPlan(&cxt, &pPlan, NULL); - } - - if (TSDB_CODE_SUCCESS == code) { - code = nodesNodeToString(pPlan, false, pStr, NULL); + if ((pCreate->ast == NULL || pCreate->ast[0] == 0) && pCreate->subscribeDbName[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; + return -1; } - nodesDestroyNode(pAst); - nodesDestroyNode(pPlan); - terrno = code; - return code; + return 0; } -#endif static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) { mDebug("topic:%s to create", pCreate->name); @@ -297,28 +289,38 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq topicObj.ast = strdup(pCreate->ast); topicObj.astLen = strlen(pCreate->ast) + 1; - SNode *pAst = NULL; - if (nodesStringToNode(pCreate->ast, &pAst) != 0) { - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; - } + if (pCreate->ast && pCreate->ast[0]) { + topicObj.subType = TOPIC_SUB_TYPE__TABLE; + topicObj.withTbName = 0; + topicObj.withSchema = 0; - SQueryPlan *pPlan = NULL; + SNode *pAst = NULL; + if (nodesStringToNode(pCreate->ast, &pAst) != 0) { + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } - SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; - if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) { - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; - } + SQueryPlan *pPlan = NULL; - if (qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema) != 0) { - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; - } + SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; + if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) { + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } - if (nodesNodeToString(pPlan, false, &topicObj.physicalPlan, NULL) != 0) { - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + if (qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema) != 0) { + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + if (nodesNodeToString(pPlan, false, &topicObj.physicalPlan, NULL) != 0) { + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + } else { + topicObj.subType = TOPIC_SUB_TYPE__DB; + topicObj.withTbName = 1; + topicObj.withSchema = 1; } STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_TOPIC, &pReq->rpcMsg); @@ -509,56 +511,40 @@ static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTo return 0; } -static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { +static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SMqTopicObj *pTopic = NULL; - int32_t cols = 0; - char *pWrite; - char prefix[TSDB_DB_FNAME_LEN] = {0}; - SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); - if (pDb == NULL) return 0; - - tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN); - strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = (int32_t)strlen(prefix); - - while (numOfRows < rows) { + while (numOfRows < rowsCapacity) { pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic); if (pShow->pIter == NULL) break; - if (pTopic->dbUid != pDb->uid) { - if (strncmp(pTopic->name, prefix, prefixLen) != 0) { - mError("Inconsistent topic data, name:%s, db:%s, dbUid:%" PRIu64, pTopic->name, pDb->name, pDb->uid); - } + int32_t cols = 0; - sdbRelease(pSdb, pTopic); - continue; - } + char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tstrncpy(&topicName[VARSTR_HEADER_SIZE], pTopic->name, TSDB_TOPIC_NAME_LEN); + varDataSetLen(topicName, strlen(&topicName[VARSTR_HEADER_SIZE])); - cols = 0; + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)topicName, false); - char topicName[TSDB_TOPIC_NAME_LEN] = {0}; - tstrncpy(topicName, pTopic->name + prefixLen, TSDB_TOPIC_NAME_LEN); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, topicName); - cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pTopic->createTime, false); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = pTopic->createTime; - cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + char *sql = taosMemoryCalloc(1, strlen(pTopic->sql) + 1 + VARSTR_HEADER_SIZE); + strcpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql); + varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE])); + colDataAppend(pColInfo, numOfRows, (const char *)sql, false); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pTopic->sql, pShow->bytes[cols]); - cols++; + taosMemoryFree(sql); numOfRows++; sdbRelease(pSdb, pTopic); } - mndReleaseDb(pMnode, pDb); pShow->numOfRows += numOfRows; return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 75fe409d2a8a6b84dd0f83c23e35749dbfa5b920..3a4fde992fd999833321fcbe67714eeb27fb5919 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -32,7 +32,6 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); static int32_t mndProcessCreateVnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessAlterVnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessDropVnodeRsp(SNodeMsg *pRsp); -static int32_t mndProcessSyncVnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessCompactVnodeRsp(SNodeMsg *pRsp); static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); @@ -50,10 +49,9 @@ int32_t mndInitVgroup(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndVgroupActionUpdate}; mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp); - mndSetMsgHandle(pMnode, TDMT_DND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp); mndSetMsgHandle(pMnode, TDMT_DND_DROP_VNODE_RSP, mndProcessDropVnodeRsp); - mndSetMsgHandle(pMnode, TDMT_DND_SYNC_VNODE_RSP, mndProcessSyncVnodeRsp); - mndSetMsgHandle(pMnode, TDMT_DND_COMPACT_VNODE_RSP, mndProcessCompactVnodeRsp); + mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_VNODE_RSP, mndProcessCompactVnodeRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndCancelGetNextVgroup); @@ -205,7 +203,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.walLevel = pDb->cfg.walLevel; createReq.precision = pDb->cfg.precision; createReq.compression = pDb->cfg.compression; - createReq.quorum = pDb->cfg.quorum; + createReq.strict = pDb->cfg.strict; createReq.update = pDb->cfg.update; createReq.cacheLastRow = pDb->cfg.cacheLastRow; createReq.replica = pVgroup->replica; @@ -213,7 +211,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.streamMode = pVgroup->streamMode; createReq.hashBegin = pVgroup->hashBegin; createReq.hashEnd = pVgroup->hashEnd; - createReq.hashMethod = pDb->hashMethod; + createReq.hashMethod = pDb->cfg.hashMethod; createReq.numOfRetensions = pDb->cfg.numOfRetensions; createReq.pRetensions = pDb->cfg.pRetensions; @@ -465,8 +463,6 @@ static int32_t mndProcessDropVnodeRsp(SNodeMsg *pRsp) { return 0; } -static int32_t mndProcessSyncVnodeRsp(SNodeMsg *pRsp) { return 0; } - static int32_t mndProcessCompactVnodeRsp(SNodeMsg *pRsp) { return 0; } static bool mndGetVgroupMaxReplicaFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { @@ -545,7 +541,7 @@ static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vnodeGid[i].dnodeId, false); char buf1[20] = {0}; - const char *role = mndGetRoleStr(pVgroup->vnodeGid[i].role); + const char *role = syncStr(pVgroup->vnodeGid[i].role); STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->bytes[cols]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -636,7 +632,7 @@ static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p colDataAppend(pColInfo, numOfRows, (const char *)&val, false); char buf[20] = {0}; - STR_TO_VARSTR(buf, mndGetRoleStr(pVgid->role)); + STR_TO_VARSTR(buf, syncStr(pVgid->role)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)buf, false); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 3ac1c522b24b1f5581095ac0ab7e005346693d8c..daf8dd431fad5d85a39c0a64da260db43b35027b 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -22,12 +22,14 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndFunc.h" +#include "mndGrant.h" #include "mndInfoSchema.h" -#include "mndPerfSchema.h" #include "mndMnode.h" #include "mndOffset.h" +#include "mndPerfSchema.h" #include "mndProfile.h" #include "mndQnode.h" +#include "mndQuery.h" #include "mndShow.h" #include "mndSma.h" #include "mndSnode.h" @@ -40,12 +42,9 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" -#include "mndQuery.h" -#include "mndGrant.h" #define MQ_TIMER_MS 3000 #define TRNAS_TIMER_MS 6000 -#define TELEM_TIMER_MS 86400000 static void *mndBuildTimerMsg(int32_t *pContLen) { SMTimerReq timerReq = {0}; @@ -97,7 +96,7 @@ static void mndPullupTelem(void *param, void *tmrId) { tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } - taosTmrReset(mndPullupTelem, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer); + taosTmrReset(mndPullupTelem, tsTelemInterval * 1000, pMnode, pMnode->timer, &pMnode->telemTimer); } static int32_t mndInitTimer(SMnode *pMnode) { @@ -117,7 +116,8 @@ static int32_t mndInitTimer(SMnode *pMnode) { return -1; } - if (taosTmrReset(mndPullupTelem, 60000, pMnode, pMnode->timer, &pMnode->telemTimer)) { + int32_t interval = tsTelemInterval < 10 ? tsTelemInterval : 10; + if (taosTmrReset(mndPullupTelem, interval * 1000, pMnode, pMnode->timer, &pMnode->telemTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -262,6 +262,7 @@ static int32_t mndExecSteps(SMnode *pMnode) { return -1; } else { mDebug("%s is initialized", pStep->name); + tmsgReportStartup(pStep->name, "initialized"); } } @@ -413,31 +414,6 @@ int64_t mndGenerateUid(char *name, int32_t len) { } while (true); } -void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { - memset(pLoad, 0, sizeof(SMnodeLoad)); - - SSdb *pSdb = pMnode->pSdb; - pLoad->numOfDnode = sdbGetSize(pSdb, SDB_DNODE); - pLoad->numOfMnode = sdbGetSize(pSdb, SDB_MNODE); - pLoad->numOfVgroup = sdbGetSize(pSdb, SDB_VGROUP); - pLoad->numOfDatabase = sdbGetSize(pSdb, SDB_DB); - pLoad->numOfSuperTable = sdbGetSize(pSdb, SDB_STB); - - void *pIter = NULL; - while (1) { - SVgObj *pVgroup = NULL; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - if (pIter == NULL) break; - - pLoad->numOfChildTable += pVgroup->numOfTables; - pLoad->numOfColumn += pVgroup->numOfTimeSeries; - pLoad->totalPoints += pVgroup->pointsWritten; - pLoad->totalStorage += pVgroup->totalStorage; - pLoad->compStorage += pVgroup->compStorage; - - sdbRelease(pSdb, pVgroup); - } -} int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, SMonGrantInfo *pGrantInfo) { @@ -485,7 +461,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr SMonMnodeDesc desc = {0}; desc.mnode_id = pObj->id; tstrncpy(desc.mnode_ep, pObj->pDnode->ep, sizeof(desc.mnode_ep)); - tstrncpy(desc.role, mndGetRoleStr(pObj->role), sizeof(desc.role)); + tstrncpy(desc.role, syncStr(pObj->role), sizeof(desc.role)); taosArrayPush(pClusterInfo->mnodes, &desc); sdbRelease(pSdb, pObj); @@ -519,7 +495,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; SMonVnodeDesc *pVnDesc = &desc.vnodes[i]; pVnDesc->dnode_id = pVgid->dnodeId; - tstrncpy(pVnDesc->vnode_role, mndGetRoleStr(pVgid->role), sizeof(pVnDesc->vnode_role)); + tstrncpy(pVnDesc->vnode_role, syncStr(pVgid->role), sizeof(pVnDesc->vnode_role)); if (pVgid->role == TAOS_SYNC_STATE_LEADER) { tstrncpy(desc.status, "ready", sizeof(desc.status)); pClusterInfo->vgroups_alive++; @@ -544,3 +520,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr return 0; } + +int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { + pLoad->syncState = pMnode->syncMgmt.state; + return 0; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index adba6ca4344eca08a0f0729d215f0ae2e8965b6d..6e72ce89d3512a471b4038a836114866b6d6309d 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -45,12 +45,12 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { createReq.maxRows = 4096; createReq.commitTime = 3600; createReq.fsyncPeriod = 3000; - createReq.ttl = 0; + createReq.ttl = 1; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; createReq.replications = 1; - createReq.quorum = 1; + createReq.strict = 1; createReq.update = 0; createReq.cacheLastRow = 0; createReq.ignoreExist = 1; @@ -82,7 +82,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { alterdbReq.daysToKeep2 = 500; alterdbReq.fsyncPeriod = 4000; alterdbReq.walLevel = 2; - alterdbReq.quorum = 2; + alterdbReq.strict = 2; alterdbReq.cacheLastRow = 1; int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); @@ -139,12 +139,12 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { createReq.maxRows = 4096; createReq.commitTime = 3600; createReq.fsyncPeriod = 3000; - createReq.ttl = 0; + createReq.ttl = 1; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; createReq.replications = 1; - createReq.quorum = 1; + createReq.strict = 1; createReq.update = 0; createReq.cacheLastRow = 0; createReq.ignoreExist = 1; diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 22b0387d8b83c3498b0fd71434200f7da0e63d7f..4db9411e877e79e756c8ab754208f8610cbd9da3 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -32,6 +32,7 @@ void MndTestFunc::SetCode(SCreateFuncReq* pReq, const char* pCode) { int32_t len = strlen(pCode); pReq->pCode = (char*)taosMemoryCalloc(1, len + 1); strcpy(pReq->pCode, pCode); + pReq->codeLen = len; } void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) { @@ -60,21 +61,6 @@ TEST_F(MndTestFunc, 02_Create_Func) { ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_NAME); } - { - SCreateFuncReq createReq = {0}; - strcpy(createReq.name, "f1"); - SetCode(&createReq, "code1"); - - int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateFuncReq(pReq, contLen, &createReq); - tFreeSCreateFuncReq(&createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT); - } - { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); @@ -90,22 +76,6 @@ TEST_F(MndTestFunc, 02_Create_Func) { ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); } - { - SCreateFuncReq createReq = {0}; - strcpy(createReq.name, "f1"); - SetCode(&createReq, "code1"); - SetComment(&createReq, ""); - - int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateFuncReq(pReq, contLen, &createReq); - tFreeSCreateFuncReq(&createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT); - } - { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); @@ -270,7 +240,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST); } { @@ -329,7 +299,6 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->bufSize, 6); EXPECT_EQ(pFuncInfo->signature, 18); EXPECT_EQ(pFuncInfo->commentSize, strlen("comment2") + 1); - EXPECT_EQ(pFuncInfo->codeSize, strlen("code2") + 1); EXPECT_STREQ("comment2", pFuncInfo->pComment); EXPECT_STREQ("code2", pFuncInfo->pCode); @@ -368,7 +337,6 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->bufSize, 6); EXPECT_EQ(pFuncInfo->signature, 18); EXPECT_EQ(pFuncInfo->commentSize, strlen("comment2") + 1); - EXPECT_EQ(pFuncInfo->codeSize, strlen("code2") + 1); EXPECT_STREQ("comment2", pFuncInfo->pComment); EXPECT_STREQ("code2", pFuncInfo->pCode); } @@ -403,7 +371,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST); } } diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index 2c3be2135b6a679ddbc7231621279057767652b8..9c8e0298aa336ee5634480395a9e33ce3390d065 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -30,8 +30,15 @@ int32_t MndTestProfile::connId; TEST_F(MndTestProfile, 01_ConnectMsg) { SConnectReq connectReq = {0}; connectReq.pid = 1234; + + char passwd[] = "taosdata"; + char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; + taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt); + strcpy(connectReq.app, "mnode_test_profile"); strcpy(connectReq.db, ""); + strcpy(connectReq.user, "root"); + strcpy(connectReq.passwd, secretEncrypt); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); void* pReq = rpcMallocCont(contLen); @@ -58,10 +65,16 @@ TEST_F(MndTestProfile, 01_ConnectMsg) { } TEST_F(MndTestProfile, 02_ConnectMsg_InvalidDB) { + char passwd[] = "taosdata"; + char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; + taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt); + SConnectReq connectReq = {0}; connectReq.pid = 1234; strcpy(connectReq.app, "mnode_test_profile"); strcpy(connectReq.db, "invalid_db"); + strcpy(connectReq.user, "root"); + strcpy(connectReq.passwd, secretEncrypt); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); void* pReq = rpcMallocCont(contLen); diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index 201a42e3efd897dfb3ae44cf6705d03ac05817f3..5c431f65d3a5eb5663eb3a92005a4635e6f7f384 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -36,7 +36,7 @@ TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_MSG); + ASSERT_NE(pRsp->code, 0); } TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) { @@ -50,14 +50,20 @@ TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_MSG); + ASSERT_NE(pRsp->code, 0); } TEST_F(MndTestShow, 03_ShowMsg_Conn) { + char passwd[] = "taosdata"; + char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; + taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt); + SConnectReq connectReq = {0}; connectReq.pid = 1234; strcpy(connectReq.app, "mnode_test_show"); strcpy(connectReq.db, ""); + strcpy(connectReq.user, "root"); + strcpy(connectReq.passwd, secretEncrypt); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); void* pReq = rpcMallocCont(contLen); @@ -74,4 +80,4 @@ TEST_F(MndTestShow, 03_ShowMsg_Conn) { TEST_F(MndTestShow, 04_ShowMsg_Cluster) { test.SendShowReq(TSDB_MGMT_TABLE_CLUSTER, "cluster", ""); EXPECT_EQ(test.GetShowRows(), 1); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/test/sma/sma.cpp b/source/dnode/mnode/impl/test/sma/sma.cpp index 4f3b0d6e3769815e1f50d0e6e046b9fd69d4f1ee..0d41d5de203f36453c1ff168ae226f142ee3c23e 100644 --- a/source/dnode/mnode/impl/test/sma/sma.cpp +++ b/source/dnode/mnode/impl/test/sma/sma.cpp @@ -54,9 +54,10 @@ void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { createReq.precision = 0; createReq.compression = 2; createReq.replications = 1; - createReq.quorum = 1; + createReq.strict = 1; createReq.update = 0; createReq.cacheLastRow = 0; + createReq.ttl = 1; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index f8f8799c9f02828636fecfe5cb5812ee848c8555..9270d38e1148f925bfb6a7d754041d0f679ea05c 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -55,9 +55,10 @@ void* MndTestStb::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { createReq.precision = 0; createReq.compression = 2; createReq.replications = 1; - createReq.quorum = 1; + createReq.strict = 1; createReq.update = 0; createReq.cacheLastRow = 0; + createReq.ttl = 1; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); diff --git a/source/dnode/mnode/impl/test/topic/topic.cpp b/source/dnode/mnode/impl/test/topic/topic.cpp index ee47a3c8b43eb8fe03f3f9fbabe1a473ef0042c6..917cb23fc9b4cd2cec2204a2b0f6494a638f9b57 100644 --- a/source/dnode/mnode/impl/test/topic/topic.cpp +++ b/source/dnode/mnode/impl/test/topic/topic.cpp @@ -47,9 +47,10 @@ void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { createReq.precision = 0; createReq.compression = 2; createReq.replications = 1; - createReq.quorum = 1; + createReq.strict = 1; createReq.update = 0; createReq.cacheLastRow = 0; + createReq.ttl = 1; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index 4e7f3c021389eb4324f70326544e75696056cebe..79464db134c1b1b6f842a68a72351ddfabf92716 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -300,9 +300,10 @@ TEST_F(MndTestUser, 03_Alter_User) { createReq.precision = 0; createReq.compression = 2; createReq.replications = 1; - createReq.quorum = 1; + createReq.strict = 1; createReq.update = 0; createReq.cacheLastRow = 0; + createReq.ttl = 1; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 08b97ed6c498b888867dceeedd76693a2e889b38..f4975c183e26da5f7a198451e1df134fea6e657b 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -113,8 +113,8 @@ int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList int tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); -int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo); -SArray *tqRetrieveDataBlock(STqReadHandle *pHandle); +int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, int32_t *pNumOfRows, + int16_t *pNumOfCols); // need to reposition diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index ed33473b1620102b1eacc1d7786f7a8f610f9b36..6f929ce829f3d00715a128cfb208ab1af7a0ea0a 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -16,6 +16,13 @@ #ifndef _TD_VNODE_TQ_H_ #define _TD_VNODE_TQ_H_ +#include "executor.h" +#include "os.h" +#include "thash.h" +#include "tmsg.h" +#include "ttimer.h" +#include "wal.h" + #ifdef __cplusplus extern "C" { #endif @@ -30,12 +37,6 @@ extern "C" { #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) // clang-format on -enum { - TQ_STREAM_TOKEN__DATA = 1, - TQ_STREAM_TOKEN__WATERMARK, - TQ_STREAM_TOKEN__CHECKPOINT, -}; - #define TQ_BUFFER_SIZE 4 #define TQ_BUCKET_MASK 0xFF @@ -87,22 +88,12 @@ struct STqReadHandle { SSubmitMsgIter msgIter; SSubmitBlkIter blkIter; SMeta* pVnodeMeta; - SArray* pColIdList; // SArray + SArray* pColIdList; // SArray int32_t sver; SSchemaWrapper* pSchemaWrapper; STSchema* pSchema; }; -typedef struct { - int8_t type; - int8_t reserved[7]; - union { - void* data; - int64_t wmTs; - int64_t checkpointId; - }; -} STqStreamToken; - typedef struct { int16_t ver; int16_t action; @@ -151,22 +142,29 @@ typedef struct { } STqMetaStore; typedef struct { - SMemAllocatorFactory* pAllocatorFactory; - SMemAllocator* pAllocator; -} STqMemRef; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + int64_t consumerId; + int32_t epoch; + int8_t subType; + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + int8_t withTagSchema; + char* qmsg; + // SRWLatch lock; + SWalReadHandle* pWalReader; + // number should be identical to fetch thread num + STqReadHandle* pStreamReader[4]; + qTaskInfo_t task[4]; +} STqExec; struct STQ { - // the collection of groups - // the handle of meta kvstore - bool writeTrigger; - char* path; - STqMemRef tqMemRef; - STqMetaStore* tqMeta; - // STqPushMgr* tqPushMgr; + char* path; + // STqMetaStore* tqMeta; + SHashObj* execs; // subKey -> tqExec SHashObj* pStreamTasks; SVnode* pVnode; SWal* pWal; - SMeta* pVnodeMeta; }; typedef struct { @@ -230,10 +228,6 @@ typedef struct { // TODO sync function } STqStreamPusher; -typedef struct { - int8_t type; // mq or stream -} STqPusher; - typedef struct { SHashObj* pHash; // } STqPushMgr; @@ -250,16 +244,17 @@ int tqInit(); void tqCleanUp(); // open in each vnode -STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pMeta, SMemAllocatorFactory* allocFac); +STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal); void tqClose(STQ*); // required by vnode int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version); int tqCommit(STQ*); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); -int32_t tqProcessSetConnReq(STQ* pTq, char* msg); -int32_t tqProcessRebReq(STQ* pTq, char* msg); -int32_t tqProcessCancelConnReq(STQ* pTq, char* msg); +int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); +// int32_t tqProcessSetConnReq(STQ* pTq, char* msg); +// int32_t tqProcessRebReq(STQ* pTq, char* msg); +// int32_t tqProcessCancelConnReq(STQ* pTq, char* msg); int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId); @@ -314,4 +309,4 @@ STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet } #endif -#endif /*_TD_VNODE_TQ_H_*/ \ No newline at end of file +#endif /*_TD_VNODE_TQ_H_*/ diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ce9549af563106e38afa061ef781e452af47d97d..a84776abfd76df66348c2a9d1885a403b2352076 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -391,6 +391,7 @@ struct SReadH { #define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh)) #define TSDB_READ_FSET(rh) (&((rh)->rSet)) #define TSDB_READ_TABLE(rh) ((rh)->pTable) +#define TSDB_READ_TABLE_UID(rh) ((rh)->pTable->uid) #define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD) #define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA) #define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST) diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c index 9a566f788c8733ae0dd20065012ea0f3a30f1385..36057ae80626f37bd56a825fa58e2bef841f0276 100644 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -23,6 +23,9 @@ struct SMetaIdx { SIndex *pIdx; #endif /* data */ +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif }; int metaOpenIdx(SMeta *pMeta) { diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 9fd11222bfe12f007d13ed8296418a775334000c..8389286596007c779f2b225f896eec9dee660368 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -45,10 +45,12 @@ struct SMetaDB { #endif }; -typedef struct __attribute__((__packed__)) { +#pragma pack(push,1) +typedef struct { tb_uid_t uid; int32_t sver; } SSchemaDbKey; +#pragma pack(pop) typedef struct { char *name; @@ -626,14 +628,8 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { #ifdef META_TDB_SMA_TEST STSmaWrapper *pSW = NULL; - pSW = taosMemoryCalloc(1, sizeof(*pSW)); - if (pSW == NULL) { - return NULL; - } - SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid); if (pCur == NULL) { - taosMemoryFree(pSW); return NULL; } @@ -653,6 +649,12 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { continue; } + if ((pSW == NULL) && ((pSW = taosMemoryCalloc(1, sizeof(*pSW))) == NULL)) { + TDB_FREE(pSmaVal); + metaCloseSmaCursor(pCur); + return NULL; + } + ++pSW->number; STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); if (tptr == NULL) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0aa6023eaf39cedb77bd8344dea3c1bae2aac8d4..30f75218b42252bca478a9e30c6ad996d273ea00 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -19,7 +19,7 @@ int32_t tqInit() { return tqPushMgrInit(); } void tqCleanUp() { tqPushMgrCleanUp(); } -STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, SMemAllocatorFactory* allocFac) { +STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { STQ* pTq = taosMemoryMalloc(sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -28,33 +28,17 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, SMe pTq->path = strdup(path); pTq->pVnode = pVnode; pTq->pWal = pWal; - pTq->pVnodeMeta = pVnodeMeta; #if 0 - pTq->tqMemRef.pAllocatorFactory = allocFac; - pTq->tqMemRef.pAllocator = allocFac->create(allocFac); - if (pTq->tqMemRef.pAllocator == NULL) { - // TODO: error code of buffer pool - } -#endif pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, (FTqDelete)taosMemoryFree, 0); if (pTq->tqMeta == NULL) { taosMemoryFree(pTq); -#if 0 - allocFac->destroy(allocFac, pTq->tqMemRef.pAllocator); -#endif - return NULL; - } - -#if 0 - pTq->tqPushMgr = tqPushMgrOpen(); - if (pTq->tqPushMgr == NULL) { - // free store - taosMemoryFree(pTq); return NULL; } #endif + pTq->execs = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); + pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); return pTq; @@ -121,7 +105,11 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi return 0; } -int tqCommit(STQ* pTq) { return tqStorePersist(pTq->tqMeta); } +int tqCommit(STQ* pTq) { + // do nothing + /*return tqStorePersist(pTq->tqMeta);*/ + return 0; +} int32_t tqGetTopicHandleSize(const STqTopic* pTopic) { return strlen(pTopic->topicName) + strlen(pTopic->sql) + strlen(pTopic->physicalPlan) + strlen(pTopic->qmsg) + @@ -236,10 +224,10 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu } for (int j = 0; j < TQ_BUFFER_SIZE; j++) { pTopic->buffer.output[j].status = 0; - STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); + STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); SReadHandle handle = { .reader = pReadHandle, - .meta = pTq->pVnodeMeta, + .meta = pTq->pVnode->pMeta, }; pTopic->buffer.output[j].pReadHandle = pReadHandle; pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle); @@ -248,16 +236,16 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu return 0; } -#if 0 + int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { - SMqPollReq* pReq = pMsg->pCont; - int64_t consumerId = pReq->consumerId; - int64_t fetchOffset; - int64_t blockingTime = pReq->blockingTime; - int32_t reqEpoch = pReq->epoch; + SMqPollReqV2* pReq = pMsg->pCont; + int64_t consumerId = pReq->consumerId; + int32_t reqEpoch = pReq->epoch; + int64_t fetchOffset; + // get offset to fetch message if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { - fetchOffset = 0; + fetchOffset = walGetFirstVer(pTq->pWal); } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { fetchOffset = walGetLastVer(pTq->pWal); } else { @@ -267,65 +255,29 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); - SMqPollRsp rsp = { - /*.consumerId = consumerId,*/ - .numOfTopics = 0, - .pBlockData = NULL, - }; + STqExec* pExec = taosHashGet(pTq->execs, pReq->subKey, strlen(pReq->subKey)); + ASSERT(pExec); - STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); - if (pConsumer == NULL) { - vWarn("tmq poll: consumer %ld (epoch %d) not found in vg %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode)); - pMsg->pCont = NULL; - pMsg->contLen = 0; - pMsg->code = -1; - tmsgSendRsp(pMsg); - return 0; - } - - int32_t consumerEpoch = atomic_load_32(&pConsumer->epoch); + int32_t consumerEpoch = atomic_load_32(&pExec->epoch); while (consumerEpoch < reqEpoch) { - consumerEpoch = atomic_val_compare_exchange_32(&pConsumer->epoch, consumerEpoch, reqEpoch); + consumerEpoch = atomic_val_compare_exchange_32(&pExec->epoch, consumerEpoch, reqEpoch); } - STqTopic* pTopic = NULL; - int32_t sz = taosArrayGetSize(pConsumer->topics); - for (int32_t i = 0; i < sz; i++) { - STqTopic* topic = taosArrayGet(pConsumer->topics, i); - // TODO race condition - ASSERT(pConsumer->consumerId == consumerId); - if (strcmp(topic->topicName, pReq->topic) == 0) { - pTopic = topic; - break; - } - } - if (pTopic == NULL) { - vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic, - TD_VID(pTq->pVnode)); - pMsg->pCont = NULL; - pMsg->contLen = 0; - pMsg->code = -1; - tmsgSendRsp(pMsg); - return 0; - } - - vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch, - TD_VID(pTq->pVnode)); - + SMqDataBlkRsp rsp = {0}; rsp.reqOffset = pReq->currentOffset; - rsp.skipLogNum = 0; + rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); + rsp.blockData = taosArrayInit(0, sizeof(void*)); while (1) { - /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/ - // TODO - consumerEpoch = atomic_load_32(&pConsumer->epoch); + consumerEpoch = atomic_load_32(&pExec->epoch); if (consumerEpoch > reqEpoch) { vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch); break; } + SWalReadHead* pHead; - if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) { + if (walReadWithHandle_s(pExec->pWalReader, fetchOffset, &pHead) < 0) { // TODO: no more log, set timer to wait blocking time // if data inserted during waiting, launch query and // response to user @@ -333,101 +285,120 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { TD_VID(pTq->pVnode), fetchOffset); break; } + vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, pHead->msgType); - /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/ - /*pHead = pTopic->pReadhandle->pHead;*/ + if (pHead->msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->body; - qTaskInfo_t task = pTopic->buffer.output[workerId].task; - ASSERT(task); - qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK); - SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); - while (1) { - SSDataBlock* pDataBlock = NULL; - uint64_t ts; - if (qExecTask(task, &pDataBlock, &ts) < 0) { - ASSERT(false); + if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { + qTaskInfo_t task = pExec->task[workerId]; + ASSERT(task); + qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK); + while (1) { + SSDataBlock* pDataBlock = NULL; + uint64_t ts = 0; + if (qExecTask(task, &pDataBlock, &ts) < 0) { + ASSERT(0); + } + if (pDataBlock == NULL) break; + + ASSERT(pDataBlock->info.rows != 0); + ASSERT(pDataBlock->info.numOfCols != 0); + + int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock); + void* buf = taosMemoryCalloc(1, dataStrLen); + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; + pRetrieve->useconds = ts; + pRetrieve->precision = TSDB_DEFAULT_PRECISION; + pRetrieve->compressed = 0; + pRetrieve->completed = 1; + pRetrieve->numOfRows = htonl(pDataBlock->info.rows); + + // TODO enable compress + int32_t actualLen = 0; + blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false); + actualLen += sizeof(SRetrieveTableRsp); + ASSERT(actualLen <= dataStrLen); + taosArrayPush(rsp.blockDataLen, &actualLen); + taosArrayPush(rsp.blockData, &buf); + rsp.blockNum++; } - if (pDataBlock == NULL) { - /*pos = fetchOffset % TQ_BUFFER_SIZE;*/ - break; + } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { + STqReadHandle* pReader = pExec->pStreamReader[workerId]; + tqReadHandleSetMsg(pReader, pCont, 0); + while (tqNextDataBlock(pReader)) { + SSDataBlock block = {0}; + if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.rows, + &block.info.numOfCols) < 0) { + ASSERT(0); + } + int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(&block); + void* buf = taosMemoryCalloc(1, dataStrLen); + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; + /*pRetrieve->useconds = 0;*/ + pRetrieve->precision = TSDB_DEFAULT_PRECISION; + pRetrieve->compressed = 0; + pRetrieve->completed = 1; + pRetrieve->numOfRows = htonl(block.info.rows); + + // TODO enable compress + int32_t actualLen = 0; + blockCompressEncode(&block, pRetrieve->data, &actualLen, block.info.numOfCols, false); + actualLen += sizeof(SRetrieveTableRsp); + ASSERT(actualLen <= dataStrLen); + taosArrayPush(rsp.blockDataLen, &actualLen); + taosArrayPush(rsp.blockData, &buf); + rsp.blockNum++; } - - taosArrayPush(pRes, pDataBlock); + } else { + ASSERT(0); } + } - if (taosArrayGetSize(pRes) == 0) { - vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId, - pReq->epoch, TD_VID(pTq->pVnode), fetchOffset); - fetchOffset++; - rsp.skipLogNum++; - taosArrayDestroy(pRes); - continue; - } - rsp.schema = pTopic->buffer.output[workerId].pReadHandle->pSchemaWrapper; - rsp.rspOffset = fetchOffset; - - rsp.numOfTopics = 1; - rsp.pBlockData = pRes; + // TODO batch optimization: + // TODO continue scan until meeting batch requirement + if (rsp.blockNum != 0) break; + rsp.skipLogNum++; + fetchOffset++; + } - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - pMsg->code = -1; - taosMemoryFree(pHead); - return -1; - } - ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; - ((SMqRspHead*)buf)->epoch = pReq->epoch; - ((SMqRspHead*)buf)->consumerId = consumerId; + ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum); + ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum); - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqPollRsp(&abuf, &rsp); - /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/ - pMsg->pCont = buf; - pMsg->contLen = tlen; - pMsg->code = 0; - vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset, - pHead->msgType, consumerId, pReq->epoch); - tmsgSendRsp(pMsg); - taosMemoryFree(pHead); - return 0; - } else { - taosMemoryFree(pHead); - fetchOffset++; - rsp.skipLogNum++; - } - } + if (rsp.blockNum != 0) + rsp.rspOffset = fetchOffset; + else + rsp.rspOffset = fetchOffset - 1; - /*if (blockingTime != 0) {*/ - /*tqAddClientPusher(pTq->tqPushMgr, pMsg, consumerId, blockingTime);*/ - /*} else {*/ - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp); + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp); void* buf = rpcMallocCont(tlen); if (buf == NULL) { pMsg->code = -1; return -1; } + ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; ((SMqRspHead*)buf)->epoch = pReq->epoch; - rsp.rspOffset = fetchOffset - 1; + ((SMqRspHead*)buf)->consumerId = consumerId; void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqPollRsp(&abuf, &rsp); - rsp.pBlockData = NULL; + tEncodeSMqDataBlkRsp(&abuf, &rsp); pMsg->pCont = buf; pMsg->contLen = tlen; pMsg->code = 0; tmsgSendRsp(pMsg); - vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId, - pReq->epoch); - /*}*/ + vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", + TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset); + + // TODO destroy + taosArrayDestroy(rsp.blockData); + taosArrayDestroy(rsp.blockDataLen); return 0; } -#endif +#if 0 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqPollReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; @@ -436,7 +407,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { int32_t reqEpoch = pReq->epoch; if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { - fetchOffset = 0; + fetchOffset = walGetFirstVer(pTq->pWal); } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { fetchOffset = walGetLastVer(pTq->pWal); } else { @@ -635,124 +606,62 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { return 0; } +#endif -int32_t tqProcessRebReq(STQ* pTq, char* msg) { - SMqMVRebReq req = {0}; - terrno = TSDB_CODE_SUCCESS; - tDecodeSMqMVRebReq(msg, &req); - - vDebug("vg %d set from consumer %ld to consumer %ld", req.vgId, req.oldConsumerId, req.newConsumerId); - STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId); - ASSERT(pConsumer); - ASSERT(pConsumer->consumerId == req.oldConsumerId); - int32_t numOfTopics = taosArrayGetSize(pConsumer->topics); - if (numOfTopics == 1) { - STqTopic* pTopic = taosArrayGet(pConsumer->topics, 0); - ASSERT(strcmp(pTopic->topicName, req.topic) == 0); - STqConsumer* pNewConsumer = tqHandleGet(pTq->tqMeta, req.newConsumerId); - if (pNewConsumer == NULL) { - pConsumer->consumerId = req.newConsumerId; - tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); - tqHandleCommit(pTq->tqMeta, req.newConsumerId); - tqHandlePurge(pTq->tqMeta, req.oldConsumerId); - return 0; - } else { - taosArrayPush(pNewConsumer->topics, pTopic); +// TODO: persist meta into tdb +int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { + SMqRebVgReq req = {0}; + tDecodeSMqRebVgReq(msg, &req); + // todo lock + STqExec* pExec = taosHashGet(pTq->execs, req.subKey, strlen(req.subKey)); + if (pExec == NULL) { + ASSERT(req.oldConsumerId == -1); + ASSERT(req.newConsumerId != -1); + STqExec exec = {0}; + pExec = &exec; + /*taosInitRWLatch(&pExec->lock);*/ + + memcpy(pExec->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN); + pExec->consumerId = req.newConsumerId; + pExec->epoch = -1; + + pExec->subType = req.subType; + pExec->withTbName = req.withTbName; + pExec->withSchema = req.withSchema; + pExec->withTag = req.withTag; + pExec->withTagSchema = req.withTagSchema; + + pExec->qmsg = req.qmsg; + req.qmsg = NULL; + + pExec->pWalReader = walOpenReadHandle(pTq->pVnode->pWal); + for (int32_t i = 0; i < 4; i++) { + pExec->pStreamReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); + SReadHandle handle = { + .reader = pExec->pStreamReader[i], + .meta = pTq->pVnode->pMeta, + }; + pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle); + ASSERT(pExec->task[i]); } + taosHashPut(pTq->execs, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec)); + return 0; } else { - for (int32_t i = 0; i < numOfTopics; i++) { - STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); - if (strcmp(pTopic->topicName, req.topic) == 0) { - STqConsumer* pNewConsumer = tqHandleGet(pTq->tqMeta, req.newConsumerId); - if (pNewConsumer == NULL) { - pNewConsumer = taosMemoryCalloc(1, sizeof(STqConsumer)); - if (pNewConsumer == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - strcpy(pNewConsumer->cgroup, pConsumer->cgroup); - pNewConsumer->topics = taosArrayInit(0, sizeof(STqTopic)); - pNewConsumer->consumerId = req.newConsumerId; - pNewConsumer->epoch = 0; - - taosArrayPush(pNewConsumer->topics, pTopic); - tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); - tqHandleCommit(pTq->tqMeta, req.newConsumerId); - return 0; - } - ASSERT(pNewConsumer->consumerId == req.newConsumerId); - taosArrayPush(pNewConsumer->topics, pTopic); - break; - } - } - // - } - return 0; -} - -int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { - SMqSetCVgReq req = {0}; - tDecodeSMqSetCVgReq(msg, &req); - bool create = false; - - vDebug("vg %d set to consumer %ld", req.vgId, req.consumerId); - STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, req.consumerId); - if (pConsumer == NULL) { - pConsumer = taosMemoryCalloc(1, sizeof(STqConsumer)); - if (pConsumer == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - strcpy(pConsumer->cgroup, req.cgroup); - pConsumer->topics = taosArrayInit(0, sizeof(STqTopic)); - pConsumer->consumerId = req.consumerId; - pConsumer->epoch = 0; - create = true; - } - - STqTopic* pTopic = taosMemoryCalloc(1, sizeof(STqTopic)); - if (pTopic == NULL) { - taosArrayDestroy(pConsumer->topics); - taosMemoryFree(pConsumer); - return -1; - } - strcpy(pTopic->topicName, req.topicName); - pTopic->sql = req.sql; - pTopic->physicalPlan = req.physicalPlan; - pTopic->qmsg = req.qmsg; - /*pTopic->committedOffset = -1;*/ - /*pTopic->currentOffset = -1;*/ - - pTopic->buffer.firstOffset = -1; - pTopic->buffer.lastOffset = -1; - pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); - if (pTopic->pReadhandle == NULL) { - ASSERT(false); - } - for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - pTopic->buffer.output[i].status = 0; - STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); - SReadHandle handle = { - .reader = pReadHandle, - .meta = pTq->pVnodeMeta, - }; - pTopic->buffer.output[i].pReadHandle = pReadHandle; - pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle); - ASSERT(pTopic->buffer.output[i].task); - } - vDebug("set topic %s to consumer %ld on vg %d", pTopic->topicName, req.consumerId, TD_VID(pTq->pVnode)); - taosArrayPush(pConsumer->topics, pTopic); - if (create) { - tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer); - tqHandleCommit(pTq->tqMeta, req.consumerId); + /*if (req.newConsumerId != -1) {*/ + /*taosWLockLatch(&pExec->lock);*/ + ASSERT(pExec->consumerId == req.oldConsumerId); + // TODO handle qmsg and exec modification + atomic_store_32(&pExec->epoch, -1); + atomic_store_64(&pExec->consumerId, req.newConsumerId); + atomic_add_fetch_32(&pExec->epoch, 1); + /*taosWUnLockLatch(&pExec->lock);*/ + return 0; + /*} else {*/ + // TODO + /*taosHashRemove(pTq->tqMetaNew, req.subKey, strlen(req.subKey));*/ + /*return 0;*/ + /*}*/ } - terrno = TSDB_CODE_SUCCESS; - return 0; -} - -int32_t tqProcessCancelConnReq(STQ* pTq, char* msg) { - terrno = TSDB_CODE_SUCCESS; - return 0; } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { @@ -764,12 +673,12 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { return -1; } for (int32_t i = 0; i < parallel; i++) { - STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); + STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); SReadHandle handle = { - .reader = pReadHandle, - .meta = pTq->pVnodeMeta, + .reader = pStreamReader, + .meta = pTq->pVnode->pMeta, }; - pTask->exec.runners[i].inputHandle = pReadHandle; + pTask->exec.runners[i].inputHandle = pStreamReader; pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); ASSERT(pTask->exec.runners[i].executor); } diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index 357917e0ba4afce1127dc041ac00871ef0d2a6af..72469b74ac9fc85f0acd554ebb1e3d80ee32d6d3 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ #include "vnodeInt.h" -// TODO:replace by an abstract file layer // #include // #include // #include @@ -87,7 +86,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F } strcpy(pMeta->dirPath, path); - char name[pathLen + 10]; + char *name = taosMemoryMalloc(pathLen + 10) ; strcpy(name, path); if (!taosDirExist(name) && taosMkDir(name) != 0) { @@ -100,6 +99,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s ", name, terrstr()); // free memory + taosMemoryFree(name); return NULL; } @@ -107,6 +107,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F pMeta->unpersistHead = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + taosMemoryFree(name); return NULL; } pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead; @@ -117,8 +118,10 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s", name, terrstr()); + taosMemoryFree(name); return NULL; } + taosMemoryFree(name); pMeta->pFile = pFile; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index cf62ea87145e58b9d99de2c73574c07503e73dfe..eb45577e0a491ab4ebeb02752f5c07edc3d3bcde 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -82,16 +82,8 @@ bool tqNextDataBlock(STqReadHandle* pHandle) { return false; } -int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) { - // currently only rows are used - - pBlockInfo->numOfCols = taosArrayGetSize(pHandle->pColIdList); - pBlockInfo->rows = pHandle->pBlock->numOfRows; - // pBlockInfo->uid = pHandle->pBlock->uid; // the uid can not be assigned to pBlockData. - return 0; -} - -SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { +int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, int32_t* pNumOfRows, + int16_t* pNumOfCols) { /*int32_t sversion = pHandle->pBlock->sversion;*/ // TODO set to real sversion int32_t sversion = 0; @@ -112,18 +104,18 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { STSchema* pTschema = pHandle->pSchema; SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper; - int32_t numOfRows = pHandle->pBlock->numOfRows; - /*int32_t numOfCols = pHandle->pSchema->numOfCols;*/ + *pNumOfRows = pHandle->pBlock->numOfRows; int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); if (colNumNeed > pSchemaWrapper->nCols) { colNumNeed = pSchemaWrapper->nCols; } - SArray* pArray = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); - if (pArray == NULL) { - return NULL; + *ppCols = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); + if (*ppCols == NULL) { + return -1; } + int32_t colMeta = 0; int32_t colNeed = 0; while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { @@ -136,21 +128,25 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { colNeed++; } else { SColumnInfoData colInfo = {0}; - /*int sz = numOfRows * pColSchema->bytes;*/ colInfo.info.bytes = pColSchema->bytes; colInfo.info.colId = pColSchema->colId; colInfo.info.type = pColSchema->type; - if (colInfoDataEnsureCapacity(&colInfo, 0, numOfRows) < 0) { - taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock); - return NULL; + if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) { + goto FAIL; } - taosArrayPush(pArray, &colInfo); + taosArrayPush(*ppCols, &colInfo); colMeta++; colNeed++; } } + int32_t colActual = taosArrayGetSize(*ppCols); + *pNumOfCols = colActual; + + // TODO in stream shuffle case, fetch groupId + *pGroupId = 0; + STSRowIter iter = {0}; tdSTSRowIterInit(&iter, pTschema); STSRow* row; @@ -159,22 +155,22 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); // get all wanted col of that block - int32_t colTot = taosArrayGetSize(pArray); - for (int32_t i = 0; i < colTot; i++) { - SColumnInfoData* pColData = taosArrayGet(pArray, i); + for (int32_t i = 0; i < colActual; i++) { + SColumnInfoData* pColData = taosArrayGet(*ppCols, i); SCellVal sVal = {0}; if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; } - /*if (colDataAppend(pColData, curRow, sVal.val, false) < 0) {*/ if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) { - taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock); - return NULL; + goto FAIL; } } curRow++; } - return pArray; + return 0; +FAIL: + taosArrayDestroy(*ppCols); + return -1; } void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 09616a8969583abae05e152511ce886983a62823..28ba50c3df6f4451d74fdb9d985760d5ed4168e9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1350,7 +1350,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF if (tBitmaps > 0) { bptr = POINTER_SHIFT(pBlockData, lsize + flen); if (isSuper && !tdDataColsIsBitmapI(pDataCols)) { - tdMergeBitmap((uint8_t *)pDataCol->pBitmap, nBitmaps, (uint8_t *)pDataCol->pBitmap); + tdMergeBitmap((uint8_t *)pDataCol->pBitmap, rowsToWrite, (uint8_t *)pDataCol->pBitmap); } tBitmapsLen = tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, tBitmaps, bptr, tBitmaps + COMP_OVERFLOW_BYTES, diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index dd8723366dd7871363b9f870506961c9620ace17..2753579e9e887919212100d85fc5ef5c3f51e137 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -601,26 +601,26 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { // Check minRowsPerFileBlock and maxRowsPerFileBlock if (pCfg->minRowsPerFileBlock == -1) { - pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; + pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MINROWS_FBLOCK; } else { - if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { + if (pCfg->minRowsPerFileBlock < TSDB_MIN_MINROWS_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MINROWS_FBLOCK) { tsdbError( - "vgId:%d invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MIN_ROW_FBLOCK %d " - "TSDB_MAX_MIN_ROW_FBLOCK %d", - pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); + "vgId:%d invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MINROWS_FBLOCK %d " + "TSDB_MAX_MINROWS_FBLOCK %d", + pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MINROWS_FBLOCK, TSDB_MAX_MINROWS_FBLOCK); terrno = TSDB_CODE_TDB_INVALID_CONFIG; return -1; } } if (pCfg->maxRowsPerFileBlock == -1) { - pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; + pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAXROWS_FBLOCK; } else { - if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { + if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAXROWS_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAXROWS_FBLOCK) { tsdbError( - "vgId:%d invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAX_ROW_FBLOCK %d " - "TSDB_MAX_MAX_ROW_FBLOCK %d", - pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); + "vgId:%d invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAXROWS_FBLOCK %d " + "TSDB_MAX_MAXROWS_FBLOCK %d", + pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MINROWS_FBLOCK, TSDB_MAX_MINROWS_FBLOCK); terrno = TSDB_CODE_TDB_INVALID_CONFIG; return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2ad34d156176c0992175ed1683cc8d9cd65a6552..f70a4478b3a3c740cf8adff91b5a0d225e33e149 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -652,7 +652,7 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe STbData** pMem = NULL; STbData** pIMem = NULL; - TSKEY tLastKey = 0; /// keyToTkey(pCheckInfo->lastKey); + TSKEY tLastKey = keyToTkey(pCheckInfo->lastKey); if (pHandle->pTsdb->mem != NULL) { pMem = taosHashGet(pHandle->pTsdb->mem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId)); if (pMem != NULL) { @@ -1410,30 +1410,33 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t if (!isAllRowsNull(src) && pColInfo->info.colId == src->colId) { if (!IS_VAR_DATA_TYPE(pColInfo->info.type)) { // todo opt performance // memmove(pData, (char*)src->pData + bytes * start, bytes * num); - for (int32_t k = start; k < num + start; ++k) { + int32_t rowIndex = numOfRows; + for (int32_t k = start; k <= end; ++k, ++rowIndex) { SCellVal sVal = {0}; if (tdGetColDataOfRow(&sVal, src, k, pCols->bitmapMode) < 0) { TASSERT(0); } if (sVal.valType == TD_VTYPE_NULL) { - colDataAppendNULL(pColInfo, k); + colDataAppendNULL(pColInfo, rowIndex); } else { - colDataAppend(pColInfo, k, sVal.val, false); + colDataAppend(pColInfo, rowIndex, sVal.val, false); } } } else { // handle the var-string + int32_t rowIndex = numOfRows; + // todo refactor, only copy one-by-one - for (int32_t k = start; k < num + start; ++k) { + for (int32_t k = start; k < num + start; ++k, ++rowIndex) { SCellVal sVal = {0}; if (tdGetColDataOfRow(&sVal, src, k, pCols->bitmapMode) < 0) { TASSERT(0); } if (sVal.valType == TD_VTYPE_NULL) { - colDataAppendNULL(pColInfo, k); + colDataAppendNULL(pColInfo, rowIndex); } else { - colDataAppend(pColInfo, k, sVal.val, false); + colDataAppend(pColInfo, rowIndex, sVal.val, false); } } } @@ -1441,8 +1444,9 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t j++; i++; } else { // pColInfo->info.colId < src->colId, it is a NULL data - for (int32_t k = start; k < num + start; ++k) { // TODO opt performance - colDataAppend(pColInfo, k, NULL, true); + int32_t rowIndex = numOfRows; + for (int32_t k = start; k < num + start; ++k, ++rowIndex) { // TODO opt performance + colDataAppend(pColInfo, rowIndex, NULL, true); } i++; } @@ -1450,8 +1454,10 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t while (i < requiredNumOfCols) { // the remain columns are all null data SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - for (int32_t k = start; k < num + start; ++k) { - colDataAppend(pColInfo, k, NULL, true); // TODO add a fast version to set a number of consecutive NULL value. + int32_t rowIndex = numOfRows; + + for (int32_t k = start; k < num + start; ++k, ++rowIndex) { + colDataAppend(pColInfo, rowIndex, NULL, true); // TODO add a fast version to set a number of consecutive NULL value. } i++; } @@ -1561,12 +1567,19 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit if (isChosenRowDataRow) { colId = pSchema->columns[chosen_itr].colId; offset = pSchema->columns[chosen_itr].offset; - tdSTpRowGetVal(row, colId, pSchema->columns[chosen_itr].type, pSchema->flen, offset, chosen_itr, &sVal); + // TODO: use STSRowIter + tdSTpRowGetVal(row, colId, pSchema->columns[chosen_itr].type, pSchema->flen, offset, chosen_itr - 1, &sVal); } else { - SKvRowIdx* pColIdx = tdKvRowColIdxAt(row, chosen_itr); - colId = pColIdx->colId; - offset = pColIdx->offset; - tdSKvRowGetVal(row, colId, offset, chosen_itr, &sVal); + // TODO: use STSRowIter + if (chosen_itr == 0) { + colId = PRIMARYKEY_TIMESTAMP_COL_ID; + tdSKvRowGetVal(row, PRIMARYKEY_TIMESTAMP_COL_ID, -1, -1, &sVal); + } else { + SKvRowIdx* pColIdx = tdKvRowColIdxAt(row, chosen_itr - 1); + colId = pColIdx->colId; + offset = pColIdx->offset; + tdSKvRowGetVal(row, colId, offset, chosen_itr - 1, &sVal); + } } if (colId == pColInfo->info.colId) { @@ -1742,7 +1755,7 @@ int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* p // be included in the query time window will be discarded static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { SQueryFilePos* cur = &pTsdbReadHandle->cur; - SDataBlockInfo blockInfo = {0}; // GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); + SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; initTableMemIterator(pTsdbReadHandle, pCheckInfo); @@ -1764,9 +1777,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf STable* pTable = NULL; int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo); - tsdbDebug("%p uid:%" PRIu64 " start merge data block, file block range:%" PRIu64 "-%" PRIu64 - " rows:%d, start:%d," - "end:%d, %s", + tsdbDebug("%p uid:%" PRIu64 " start merge data block, file block range:%" PRIu64 "-%" PRIu64 " rows:%d, start:%d, end:%d, %s", pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, cur->pos, endPos, pTsdbReadHandle->idStr); @@ -3249,6 +3260,9 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SColumnDat return TSDB_CODE_SUCCESS; } + tsdbDebug("vgId:%d succeed to load block statis part for uid %" PRIu64, REPO_ID(pHandle->pTsdb), + TSDB_READ_TABLE_UID(&pHandle->rhelper)); + int16_t* colIds = pHandle->defaultLoadColumn->pData; size_t numOfCols = QH_GET_NUM_OF_COLS(pHandle); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index b15271a51a0d2c5b0229b19ac49f06f0f4478bd4..f18f36e07bd0fee50d9928298b431de450508049 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -305,7 +305,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; if (pDataCol->bitmap) { ASSERT(pDataCol->colId != PRIMARYKEY_TIMESTAMP_COL_ID); - tdMergeBitmap(pDataCol->pBitmap, TD_BITMAP_BYTES(pReadh->pDCols[0]->numOfRows), pDataCol->pBitmap); + tdMergeBitmap(pDataCol->pBitmap, pReadh->pDCols[0]->numOfRows, pDataCol->pBitmap); tdDataColsSetBitmapI(pReadh->pDCols[0]); } } @@ -322,15 +322,18 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { ASSERT(pBlock->numOfSubBlocks <= 1); if (!pBlock->aggrStat) { + tsdbDebug("vgId:%d no need to load block statis part for uid %" PRIu64 " since not exist", REPO_ID(pReadh->pRepo), + TSDB_READ_TABLE_UID(pReadh)); return TSDB_STATIS_NONE; } SDFile *pDFileAggr = pBlock->last ? TSDB_READ_SMAL_FILE(pReadh) : TSDB_READ_SMAD_FILE(pReadh); if (tsdbSeekDFile(pDFileAggr, pBlock->aggrOffset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to load block aggr part while seek file %s to offset %" PRIu64 " since %s", - TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, - tstrerror(terrno)); + tsdbError("vgId:%d failed to load block statis part for uid %" PRIu64 " while seek file %s to offset %" PRIu64 + " since %s", + TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), + (uint64_t)pBlock->aggrOffset, tstrerror(terrno)); return -1; } @@ -339,25 +342,28 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { int64_t nreadAggr = tsdbReadDFile(pDFileAggr, (void *)(pReadh->pAggrBlkData), sizeAggr); if (nreadAggr < 0) { - tsdbError("vgId:%d failed to load block aggr part while read file %s since %s, offset:%" PRIu64 " len :%" PRIzu, - TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), tstrerror(terrno), - (uint64_t)pBlock->aggrOffset, sizeAggr); + tsdbError("vgId:%d failed to load block statis part for uid %" PRIu64 + " while read file %s since %s, offset:%" PRIu64 " len :%" PRIzu, + TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), + tstrerror(terrno), (uint64_t)pBlock->aggrOffset, sizeAggr); return -1; } if (nreadAggr < sizeAggr) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block aggr part in file %s is corrupted, offset:%" PRIu64 " expected bytes:%" PRIzu - " read bytes: %" PRId64, - TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr, - nreadAggr); + tsdbError("vgId:%d block statis part for uid %" PRIu64 " in file %s is corrupted, offset:%" PRIu64 + " expected bytes:%" PRIzu " read bytes: %" PRId64, + TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), + (uint64_t)pBlock->aggrOffset, sizeAggr, nreadAggr); return -1; } if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pAggrBlkData), (uint32_t)sizeAggr)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block aggr part in file %s is corrupted since wrong checksum, offset:%" PRIu64 " len :%" PRIzu, - TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr); + tsdbError("vgId:%d block statis part for uid %" PRIu64 + "in file %s is corrupted since wrong checksum, offset:%" PRIu64 " len :%" PRIzu, + TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), + (uint64_t)pBlock->aggrOffset, sizeAggr); return -1; } return 0; @@ -367,7 +373,7 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) { ASSERT(pBlock->numOfSubBlocks <= 1); SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to load block statis part while seek file %s to offset %" PRId64 " since %s", + tsdbError("vgId:%d failed to load block head part while seek file %s to offset %" PRId64 " since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno)); return -1; } @@ -377,14 +383,14 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) { int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size); if (nread < 0) { - tsdbError("vgId:%d failed to load block statis part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu, + tsdbError("vgId:%d failed to load block head part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu, TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, size); return -1; } if (nread < size) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block statis part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu + tsdbError("vgId:%d block head part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu " read bytes: %" PRId64, TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size, nread); return -1; @@ -392,7 +398,7 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) { if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkData), (uint32_t)size)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu, + tsdbError("vgId:%d block head part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu, TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size); return -1; } @@ -546,7 +552,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat int32_t tsize = (int32_t)tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer); if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d", + tsdbError("vgId:%d block head part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tsize); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 4b1c213cdd3d7407ec4436468ac2c7b6db606115..273b7447ffe781075a3956a83237e9f563eb88e4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -643,6 +643,7 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers SSubmitMsgIter msgIter = {0}; SSubmitBlk *pBlock = NULL; SInterval interval = {0}; + TSKEY lastWinSKey = INT64_MIN; if (tInitSubmitMsgIter(pMsg, &msgIter) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; @@ -657,7 +658,7 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers SSubmitBlkIter blkIter = {0}; if (tInitSubmitBlkIter(pBlock, &blkIter) != TSDB_CODE_SUCCESS) { - tdFreeTSmaWrapper(pSW); + pSW = tdFreeTSmaWrapper(pSW); break; } @@ -667,31 +668,37 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers tdFreeTSmaWrapper(pSW); break; } - if (pSW == NULL) { + if (!pSW || (pTSma->tableUid != pBlock->suid)) { + if (pSW) { + pSW = tdFreeTSmaWrapper(pSW); + } if ((pSW = metaGetSmaInfoByTable(REPO_META(pTsdb), pBlock->suid)) == NULL) { break; } if ((pSW->number) <= 0 || (pSW->tSma == NULL)) { - tdFreeTSmaWrapper(pSW); + pSW = tdFreeTSmaWrapper(pSW); break; } + pTSma = pSW->tSma; - } - interval.interval = pTSma->interval; - interval.intervalUnit = pTSma->intervalUnit; - interval.offset = pTSma->offset; - interval.precision = REPO_CFG(pTsdb)->precision; - interval.sliding = pTSma->sliding; - interval.slidingUnit = pTSma->slidingUnit; + interval.interval = pTSma->interval; + interval.intervalUnit = pTSma->intervalUnit; + interval.offset = pTSma->offset; + interval.precision = REPO_CFG(pTsdb)->precision; + interval.sliding = pTSma->sliding; + interval.slidingUnit = pTSma->slidingUnit; + } TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); - tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, winSKey, version); - - // TODO: release only when suid changes. - tdDestroyTSmaWrapper(pSW); - taosMemoryFreeClear(pSW); + if (lastWinSKey != winSKey) { + lastWinSKey = winSKey; + tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, winSKey, version); + } else { + tsdbDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", + REPO_ID(pTsdb), pTSma->indexUid, winSKey); + } } } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index ecae05042715651e76e54b328ec38be7125bbdd8..2b10afa5ed9548b2f6397d6d3bd3487072b82c3a 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -113,7 +113,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // open tq sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); - pVnode->pTq = tqOpen(tdir, pVnode, pVnode->pWal, pVnode->pMeta, vBufPoolGetMAF(pVnode)); + pVnode->pTq = tqOpen(tdir, pVnode, pVnode->pWal); if (pVnode->pTq == NULL) { vError("vgId: %d failed to open vnode tq since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 4202c02a0c0b755fcc9a6082494377a6257c7855..3747e1dbc71b80b4ff79b9d5545a4d72dbcf7f8f 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -152,7 +152,7 @@ _exit: int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { pLoad->vgId = TD_VID(pVnode); - pLoad->role = TAOS_SYNC_STATE_LEADER; + pLoad->syncState = TAOS_SYNC_STATE_LEADER; pLoad->numOfTables = metaGetTbNum(pVnode->pMeta); pLoad->numOfTimeSeries = 400; pLoad->totalStorage = 300; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 56d4ba392c798dc05c5ef893ce19ba73018f0470..e0c37f7eb737b132b6fd03e3f3e276f04aaad1c1 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -84,6 +84,13 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg pRsp->msgType = TDMT_VND_SUBMIT_RSP; vnodeProcessSubmitReq(pVnode, ptr, pRsp); break; + case TDMT_VND_MQ_VG_CHANGE: + if (tqProcessVgChangeReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), + pMsg->contLen - sizeof(SMsgHead)) < 0) { + // TODO: handle error + } + break; +#if 0 case TDMT_VND_MQ_SET_CONN: { if (tqProcessSetConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { // TODO: handle error @@ -97,6 +104,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg if (tqProcessCancelConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { } } break; +#endif case TDMT_VND_TASK_DEPLOY: { if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { @@ -121,6 +129,8 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg // } } break; + case TDMT_VND_ALTER_VNODE: + break; default: ASSERT(0); break; @@ -287,10 +297,21 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq) { return -1; } + // TODO: remove the debug log + SRSmaParam *param = vCreateTbReq.stbCfg.pRSmaParam; + if (param) { + printf("qmsg1 len = %d, body = %s\n", param->qmsg1 ? (int32_t)strlen(param->qmsg1) : 0, + param->qmsg1 ? param->qmsg1 : ""); + printf("qmsg1 len = %d, body = %s\n", param->qmsg2 ? (int32_t)strlen(param->qmsg2) : 0, + param->qmsg2 ? param->qmsg2 : ""); + } + taosMemoryFree(vCreateTbReq.stbCfg.pSchema); taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema); if (vCreateTbReq.stbCfg.pRSmaParam) { taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->pFuncIds); + taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg1); + taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg2); taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam); } taosMemoryFree(vCreateTbReq.name); @@ -389,4 +410,4 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg pRsp->contLen = sizeof(SSubmitRsp); return 0; -} \ No newline at end of file +} diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 8938084724d2f708d269785ee78de7e756a7bdef..918892b78627cdaa0c6658fba9349abb0f511635 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -104,6 +104,10 @@ typedef struct SCatalog { typedef struct SCtgApiStat { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SCtgApiStat; typedef struct SCtgRuntimeStat { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 21e0be40a434e1dd097177a2d21325d8dada9b56..410527c9e6ac48c52e5926abfde59e321557eb7e 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -643,6 +643,49 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt return TSDB_CODE_SUCCESS; } +int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *funcName, SFuncInfo **out) { + char *msg = NULL; + int32_t msgLen = 0; + + ctgDebug("try to get udf info from mnode, funcName:%s", funcName); + + int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)]((void *)funcName, &msg, 0, &msgLen); + if (code) { + ctgError("Build get udf msg failed, code:%x, db:%s", code, funcName); + CTG_ERR_RET(code); + } + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_RETRIEVE_FUNC, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + if (TSDB_CODE_SUCCESS != rpcRsp.code) { + if (TSDB_CODE_MND_FUNC_NOT_EXIST == rpcRsp.code) { + ctgDebug("funcName %s not exist in mnode", funcName); + taosMemoryFreeClear(*out); + CTG_RET(TSDB_CODE_SUCCESS); + } + + ctgError("error rsp for get udf, error:%s, funcName:%s", tstrerror(rpcRsp.code), funcName); + CTG_ERR_RET(rpcRsp.code); + } + + code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)](*out, rpcRsp.pCont, rpcRsp.contLen); + if (code) { + ctgError("Process get udf rsp failed, code:%x, funcName:%s", code, funcName); + CTG_ERR_RET(code); + } + + ctgDebug("Got udf from mnode, funcName:%s", funcName); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { if (NULL == pCtg->dbCache) { @@ -2811,6 +2854,30 @@ int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, CTG_API_LEAVE(ctgGetIndexInfoFromMnode(pCtg, pRpc, pMgmtEps, indexName, pInfo)); } +int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == funcName || NULL == pInfo) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + *pInfo = taosMemoryMalloc(sizeof(SFuncInfo)); + if (NULL == *pInfo) { + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } + + CTG_ERR_JRET(ctgGetUdfInfoFromMnode(pCtg, pRpc, pMgmtEps, funcName, pInfo)); + +_return: + + if (code) { + taosMemoryFreeClear(*pInfo); + } + + CTG_API_LEAVE(code); +} + void catalogDestroy(void) { qInfo("start to destroy catalog"); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 73d0dc20117914fabe9af470d59cfa5ea84e6711..1045acbe939417ed94a1a13f15c561797702146c 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -24,6 +24,9 @@ #pragma GCC diagnostic ignored "-Wformat" #include +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" #include "catalog.h" @@ -108,9 +111,10 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { createReq.precision = 0; createReq.compression = 2; createReq.replications = 1; - createReq.quorum = 1; + createReq.strict = 1; createReq.update = 0; createReq.cacheLastRow = 0; + createReq.ttl = 1; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); @@ -154,11 +158,11 @@ int32_t ctgTestGetVgNumFromVgVersion(int32_t vgVersion) { } void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { - SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName cn = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(cn.dbname, "db1"); strcpy(cn.tname, ctgTestCTablename); - SName sn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName sn = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(sn.dbname, "db1"); strcpy(sn.tname, ctgTestSTablename); @@ -576,9 +580,16 @@ void ctgTestSetRspDbVgroups() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroups); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroups); } @@ -589,9 +600,16 @@ void ctgTestSetRspTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspTableMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspTableMeta); } @@ -602,9 +620,16 @@ void ctgTestSetRspCTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspCTableMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspCTableMeta); } @@ -615,9 +640,16 @@ void ctgTestSetRspSTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspSTableMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspSTableMeta); } @@ -628,9 +660,16 @@ void ctgTestSetRspMultiSTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspMultiSTableMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspMultiSTableMeta); } @@ -641,9 +680,16 @@ void ctgTestSetRspByIdx() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspByIdx); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspByIdx); } @@ -655,9 +701,16 @@ void ctgTestSetRspDbVgroupsAndNormalMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndNormalMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroupsAndNormalMeta); } @@ -668,9 +721,16 @@ void ctgTestSetRspDbVgroupsAndChildMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndChildMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroupsAndChildMeta); } @@ -681,9 +741,16 @@ void ctgTestSetRspDbVgroupsAndSuperMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndSuperMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroupsAndSuperMeta); } @@ -694,9 +761,16 @@ void ctgTestSetRspDbVgroupsAndMultiSuperMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndMultiSuperMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroupsAndMultiSuperMeta); } @@ -788,7 +862,7 @@ void *ctgTestGetCtableMetaThread(void *param) { STableMeta *tbMeta = NULL; bool inCache = false; - SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName cn = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(cn.dbname, "db1"); strcpy(cn.tname, ctgTestCTablename); @@ -870,7 +944,7 @@ TEST(tableMeta, normalTable) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -977,7 +1051,7 @@ TEST(tableMeta, childTableCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestCTablename); @@ -1084,7 +1158,7 @@ TEST(tableMeta, superTableCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); @@ -1209,7 +1283,7 @@ TEST(tableMeta, rmStbMeta) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); @@ -1279,7 +1353,7 @@ TEST(tableMeta, updateStbMeta) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); @@ -1377,7 +1451,7 @@ TEST(refreshGetMeta, normal2normal) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -1456,7 +1530,7 @@ TEST(refreshGetMeta, normal2notexist) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -1528,7 +1602,7 @@ TEST(refreshGetMeta, normal2child) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); ctgTestCurrentCTableName = ctgTestTablename; @@ -1612,7 +1686,7 @@ TEST(refreshGetMeta, stable2child) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); ctgTestCurrentSTableName = ctgTestTablename; @@ -1698,7 +1772,7 @@ TEST(refreshGetMeta, stable2stable) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); ctgTestCurrentSTableName = ctgTestTablename; @@ -1785,7 +1859,7 @@ TEST(refreshGetMeta, child2stable) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); ctgTestCurrentCTableName = ctgTestTablename; @@ -1870,7 +1944,7 @@ TEST(tableDistVgroup, normalTable) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -1912,7 +1986,7 @@ TEST(tableDistVgroup, childTableCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestCTablename); @@ -1955,7 +2029,7 @@ TEST(tableDistVgroup, superTableCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); @@ -2005,7 +2079,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -2090,7 +2164,7 @@ TEST(multiThread, getSetRmSameDbVgroup) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -2142,7 +2216,7 @@ TEST(multiThread, getSetRmDiffDbVgroup) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -2194,7 +2268,7 @@ TEST(multiThread, ctableMeta) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -2246,7 +2320,7 @@ TEST(rentTest, allRent) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); for (int32_t i = 1; i <= 10; ++i) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 230ccf6aceef66b699066da6182edb7355424ffa..5adfb7caca4a3f633ac8b3858581a4e9760fe1bf 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -382,7 +382,7 @@ typedef struct SSysTableScanInfo { void* pCur; // cursor for iterate the local table meta store. SArray* scanCols; // SArray scan column id list - int32_t type; // show type, TODO remove it +// int32_t type; // show type, TODO remove it SName name; SSDataBlock* pRes; int32_t capacity; @@ -427,6 +427,7 @@ typedef struct STableIntervalOperatorInfo { EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] SArray* pUpdatedWindow; // updated time window due to the input data block from the downstream operator. STimeWindowAggSupp twAggSup; + struct SFillInfo* pFillInfo; // fill info } STableIntervalOperatorInfo; typedef struct SAggOperatorInfo { @@ -467,7 +468,6 @@ typedef struct SFillOperatorInfo { SSDataBlock* existNewGroupBlock; bool multigroupResult; SInterval intervalInfo; - int32_t capacity; } SFillOperatorInfo; typedef struct { @@ -479,7 +479,7 @@ typedef struct { typedef struct SGroupbyOperatorInfo { SOptrBasicInfo binfo; - SArray* pGroupCols; + SArray* pGroupCols; // group by columns, SArray SArray* pGroupColVals; // current group column values, SArray SNode* pCondition; bool isInit; // denote if current val is initialized or not @@ -490,6 +490,7 @@ typedef struct SGroupbyOperatorInfo { SExprInfo* pScalarExprInfo; int32_t numOfScalarExpr; // the number of scalar expression in group operator SqlFunctionCtx* pScalarFuncCtx; + int32_t* rowCellInfoOffset; // offset value for each row result cell info } SGroupbyOperatorInfo; typedef struct SDataGroupInfo { @@ -599,7 +600,6 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode *pOnCondition; -// SRspResultInfo resultInfo; } SJoinOperatorInfo; int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); @@ -608,7 +608,7 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, size_t keyBufSize, const char* pkey); void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); -void toSDatablock(SSDataBlock* pBlock, int32_t rowCapacity, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, +void doBuildResultDatablock(SSDataBlock* pBlock, int32_t rowCapacity, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, int32_t* rowCellOffset); void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); @@ -620,7 +620,7 @@ void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput); int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList); -void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow* win); +void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, STimeWindow* win); void doSetOperatorCompleted(SOperatorInfo* pOperator); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); @@ -644,8 +644,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, - SExecTaskInfo* pTaskInfo); + STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, diff --git a/source/libs/function/inc/tfill.h b/source/libs/executor/inc/tfill.h similarity index 71% rename from source/libs/function/inc/tfill.h rename to source/libs/executor/inc/tfill.h index b90dbf7799a904609a293861d9de68775b2ca100..26d066d9a9f9fbe484b51d5802341b55527211bd 100644 --- a/source/libs/function/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -22,15 +22,18 @@ extern "C" { #include "os.h" #include "taosdef.h" +#include "tcommon.h" struct SSDataBlock; typedef struct SFillColInfo { - STColumn col; // column info +// STColumn col; // column info + SResSchema col; int16_t functionId; // sql function id int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN int16_t tagIndex; // index of current tag in SFillTagColInfo array list - union {int64_t i; double d;} fillVal; + int32_t offset; + union {int64_t i; double d;} val; } SFillColInfo; typedef struct { @@ -57,7 +60,6 @@ typedef struct SFillInfo { char * nextValues; // next row of data char** pData; // original result data block involved in filling data int32_t alloc; // data buffer size in rows - int8_t precision; // time resoluation SFillColInfo* pFillCol; // column info for fill operations SFillTagColInfo* pTags; // tags value for filling gap @@ -67,7 +69,19 @@ typedef struct SFillInfo { int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); +void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); +void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); +void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); +struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SValueNode* val); +bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); +SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, + SInterval* pInterval, int32_t fillType, + struct SFillColInfo* pCol, const char* id); + +void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); +int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity); +int64_t getFillInfoStart(struct SFillInfo *pFillInfo); #ifdef __cplusplus diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 87ecdde3ab811108170cbc3f5a423bac3a6b3125..7a7cb68424ff4b74a5596b2dfbdc83cd00bf4493 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -14,11 +14,12 @@ */ #include "filter.h" -#include "functionMgt.h" #include "function.h" +#include "functionMgt.h" +#include "os.h" #include "querynodes.h" #include "tname.h" -#include "os.h" +#include "tfill.h" #include "tdatablock.h" #include "tglobal.h" @@ -514,13 +515,12 @@ static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultR return pResult; } -static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, TSKEY ekey, - bool ascQuery) { +static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, bool ascQuery) { if (ascQuery) { - getAlignQueryTimeWindow(pInterval, precision, ts, ts, ekey, w); + getAlignQueryTimeWindow(pInterval, precision, ts, w); } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - getAlignQueryTimeWindow(pInterval, precision, ts, ekey, ts, w); + getAlignQueryTimeWindow(pInterval, precision, ts, w); int64_t key = w->skey; while (key < ts) { // moving towards end @@ -540,7 +540,7 @@ static STimeWindow getActiveTimeWindow(SDiskbasedBuf * pBuf, SResultRowInfo* pRe STimeWindow w = {0}; if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value - getInitialStartTimeWindow(pInterval, precision, ts, &w, win->ekey, true); + getInitialStartTimeWindow(pInterval, precision, ts, &w, true); w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win; @@ -1087,6 +1087,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pCtx[i].currentStage = MAIN_SCAN; SInputColumnInfoData* pInput = &pCtx[i].input; + pInput->uid = pBlock->info.uid; SExprInfo* pOneExpr = &pOperator->pExpr[i]; for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) { @@ -1101,7 +1102,9 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pInput->pPTS = taosArrayGet(pBlock->pDataBlock, 0); // todo set the correct timestamp column ASSERT(pInput->pData[j] != NULL); } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { - if (createDummyCol) { + // todo avoid case: top(k, 12), 12 is the value parameter. + // sum(11), 11 is also the value parameter. + if (createDummyCol && pOneExpr->base.numOfParams == 1) { code = doCreateConstantValColumnInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1173,6 +1176,9 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* setPseudoOutputColInfo(pResult, pCtx, pPseudoList); pResult->info.groupId = pSrcBlock->info.groupId; + // if the source equals to the destination, it is to create a new column as the result of scalar function or some operators. + bool createNewColModel = (pResult == pSrcBlock); + int32_t numOfRows = 0; for (int32_t k = 0; k < numOfOutput; ++k) { @@ -1181,7 +1187,7 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) { // it is a project query SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId); - if (pResult->info.rows > 0) { + if (pResult->info.rows > 0 && !createNewColModel) { colDataMergeCol(pColInfoData, pResult->info.rows, pfCtx->input.pData[0], pfCtx->input.numOfRows); } else { colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows); @@ -1191,7 +1197,7 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE) { SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId); - int32_t offset = pResult->info.rows; + int32_t offset = createNewColModel? 0: pResult->info.rows; for (int32_t i = 0; i < pSrcBlock->info.rows; ++i) { colDataAppend(pColInfoData, i + offset, taosVariantGet(&pExpr[k].base.pParam[0].param, pExpr[k].base.pParam[0].param.nType), TSDB_DATA_TYPE_NULL == pExpr[k].base.pParam[0].param.nType); } @@ -1207,7 +1213,8 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* SScalarParam dest = {.columnData = &idata}; scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); - colDataMergeCol(pResColData, pResult->info.rows, &idata, dest.numOfRows); + int32_t startOffset = createNewColModel? 0:pResult->info.rows; + colDataMergeCol(pResColData, startOffset, &idata, dest.numOfRows); numOfRows = dest.numOfRows; taosArrayDestroy(pBlockList); @@ -1224,7 +1231,7 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pfCtx->fpSet.init(&pCtx[k], pResInfo); pfCtx->pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId); - pfCtx->offset = pResult->info.rows; // set the start offset + pfCtx->offset = createNewColModel? 0:pResult->info.rows; // set the start offset // set the timestamp(_rowts) output buffer if (taosArrayGetSize(pPseudoList) > 0) { @@ -1242,7 +1249,9 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* SScalarParam dest = {.columnData = &idata}; scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest); - colDataMergeCol(pResColData, pResult->info.rows, &idata, dest.numOfRows); + + int32_t startOffset = createNewColModel? 0:pResult->info.rows; + colDataMergeCol(pResColData, startOffset, &idata, dest.numOfRows); numOfRows = dest.numOfRows; taosArrayDestroy(pBlockList); @@ -1252,7 +1261,9 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* } } - pResult->info.rows += numOfRows; + if (!createNewColModel) { + pResult->info.rows += numOfRows; + } } void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, @@ -1868,67 +1879,58 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, } } pCtx->resDataInfo.interBufSize = env.calcMemSize; - } else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN || pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR || pExpr->pExpr->nodeType == QUERY_NODE_VALUE) { - pCtx->resDataInfo.interBufSize = pFunct->resSchema.bytes; // for simple column, the intermediate buffer needs to hold one element. + } else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN || pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR || + pExpr->pExpr->nodeType == QUERY_NODE_VALUE) { + // for simple column, the intermediate buffer needs to hold one element. + pCtx->resDataInfo.interBufSize = pFunct->resSchema.bytes; } pCtx->input.numOfInputCols = pFunct->numOfParams; pCtx->input.pData = taosMemoryCalloc(pFunct->numOfParams, POINTER_BYTES); pCtx->input.pColumnDataAgg = taosMemoryCalloc(pFunct->numOfParams, POINTER_BYTES); - pCtx->pTsOutput = NULL; + pCtx->pTsOutput = NULL; pCtx->resDataInfo.bytes = pFunct->resSchema.bytes; - pCtx->resDataInfo.type = pFunct->resSchema.type; - pCtx->order = TSDB_ORDER_ASC; - pCtx->start.key = INT64_MIN; - pCtx->end.key = INT64_MIN; -#if 0 - for (int32_t j = 0; j < pCtx->numOfParams; ++j) { -// int16_t type = pFunct->param[j].nType; -// int16_t bytes = pFunct->param[j].nLen; - -// if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { -// taosVariantCreateFromBinary(&pCtx->param[j], pFunct->param[j].pz, bytes, type); -// } else { -// taosVariantCreateFromBinary(&pCtx->param[j], (char *)&pFunct->param[j].i, bytes, type); -// } - } - - // set the order information for top/bottom query - int32_t functionId = pCtx->functionId; - if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { - int32_t f = getExprFunctionId(&pExpr[0]); - assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY); - -// pCtx->param[2].i = pQueryAttr->order.order; - pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[3].i = functionId; - pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; - -// pCtx->param[1].i = pQueryAttr->order.col.info.colId; - } else if (functionId == FUNCTION_INTERP) { -// pCtx->param[2].i = (int8_t)pQueryAttr->fillType; -// if (pQueryAttr->fillVal != NULL) { -// if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) { -// pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; -// } else { // todo refactor, taosVariantCreateFromBinary should handle the NULL value -// if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { -// taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType); -// } -// } + pCtx->resDataInfo.type = pFunct->resSchema.type; + pCtx->order = TSDB_ORDER_ASC; + pCtx->start.key = INT64_MIN; + pCtx->end.key = INT64_MIN; + pCtx->numOfParams = pExpr->base.numOfParams; + + pCtx->param = pFunct->pParam; +// for (int32_t j = 0; j < pCtx->numOfParams; ++j) { +// // set the order information for top/bottom query +// int32_t functionId = pCtx->functionId; +// if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { +// int32_t f = getExprFunctionId(&pExpr[0]); +// assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY); +// +// // pCtx->param[2].i = pQueryAttr->order.order; +// // pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; +// // pCtx->param[3].i = functionId; +// // pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; +// +// // pCtx->param[1].i = pQueryAttr->order.col.info.colId; +// } else if (functionId == FUNCTION_INTERP) { +// // pCtx->param[2].i = (int8_t)pQueryAttr->fillType; +// // if (pQueryAttr->fillVal != NULL) { +// // if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) { +// // pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; +// // } else { // todo refactor, taosVariantCreateFromBinary should handle the NULL value +// // if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { +// // taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType); +// // } +// // } +// // } +// } else if (functionId == FUNCTION_TWA) { +// // pCtx->param[1].i = pQueryAttr->window.skey; +// // pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; +// // pCtx->param[2].i = pQueryAttr->window.ekey; +// // pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; +// } else if (functionId == FUNCTION_ARITHM) { +// // pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); // } - } else if (functionId == FUNCTION_TS_COMP) { -// pCtx->param[0].i = pQueryAttr->vgId; //TODO this should be the parameter from client - pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; - } else if (functionId == FUNCTION_TWA) { -// pCtx->param[1].i = pQueryAttr->window.skey; - pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; -// pCtx->param[2].i = pQueryAttr->window.ekey; - pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - } else if (functionId == FUNCTION_ARITHM) { -// pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); - } -#endif +// } } for (int32_t i = 1; i < numOfOutput; ++i) { @@ -1947,7 +1949,7 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t i = 0; i < numOfOutput; ++i) { for (int32_t j = 0; j < pCtx[i].numOfParams; ++j) { - taosVariantDestroy(&pCtx[i].param[j]); + taosVariantDestroy(&pCtx[i].param[j].param); } taosVariantDestroy(&pCtx[i].tag); @@ -2007,20 +2009,16 @@ static bool isCachedLastQuery(STaskAttr* pQueryAttr) { ///////////////////////////////////////////////////////////////////////////////////////////// // todo refactor : return window -void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, int64_t keyFirst, int64_t keyLast, - STimeWindow* win) { - ASSERT(key >= keyFirst && key <= keyLast); +void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, STimeWindow* win) { win->skey = taosTimeTruncate(key, pInterval, precision); /* * if the realSkey > INT64_MAX - pInterval->interval, the query duration between * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. */ - if (keyFirst > (INT64_MAX - pInterval->interval)) { - assert(keyLast - keyFirst < pInterval->interval); + win->ekey = taosTimeAdd(win->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + if (win->ekey < win->skey) { win->ekey = INT64_MAX; - } else { - win->ekey = taosTimeAdd(win->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } } @@ -3168,10 +3166,12 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, int32_t rowCapacity, SExprInfo* // qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_TASKID(pRuntimeEnv)); pBlock->info.rows = numOfResult; + blockDataUpdateTsWindow(pBlock); + return 0; } -void toSDatablock(SSDataBlock* pBlock, int32_t rowCapacity, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, +void doBuildResultDatablock(SSDataBlock* pBlock, int32_t rowCapacity, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, int32_t* rowCellOffset) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); @@ -4805,7 +4805,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator, bool* newgroup) } blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - toSDatablock(pInfo->pRes, pOperator->resultInfo.capacity, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf, pInfo->rowCellInfoOffset); + doBuildResultDatablock(pInfo->pRes, pOperator->resultInfo.capacity, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf, pInfo->rowCellInfoOffset); if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -5118,6 +5118,8 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgro return NULL; } + SSDataBlock* pBlock = pInfo->binfo.pRes; + if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { return pOperator->getStreamResFn(pOperator, newgroup); } else { @@ -5126,15 +5128,15 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgro return NULL; } - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - toSDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, - pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); + doBuildResultDatablock(pBlock, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, + pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; + return pBlock->info.rows == 0 ? NULL : pBlock; } } @@ -5147,7 +5149,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup } if (pOperator->status == OP_RES_TO_RETURN) { - toSDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + doBuildResultDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -5182,7 +5184,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - toSDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + doBuildResultDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); ASSERT(pInfo->binfo.pRes->info.rows > 0); pOperator->status = OP_RES_TO_RETURN; @@ -5197,7 +5199,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) { STimeSliceOperatorInfo* pSliceInfo = pOperator->info; if (pOperator->status == OP_RES_TO_RETURN) { - // toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -5230,7 +5232,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) { finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput); initGroupResInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); - // toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes); + // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes); if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; @@ -5284,7 +5286,7 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup OPTR_SET_OPENED(pOperator); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - toSDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, + doBuildResultDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { @@ -5375,7 +5377,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - toSDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); + doBuildResultDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -5407,7 +5409,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - toSDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); + doBuildResultDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -5424,7 +5426,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - toSDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); + doBuildResultDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -5456,7 +5458,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - toSDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); + doBuildResultDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -5501,14 +5503,16 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SResultInfo* pResultInfo = &pOperator->resultInfo; - blockDataCleanup(pInfo->pRes); + SSDataBlock* pResBlock = pInfo->pRes; + + blockDataCleanup(pResBlock); if (pOperator->status == OP_EXEC_DONE) { return NULL; } doHandleRemainBlockFromNewGroup(pInfo, pResultInfo, newgroup, pTaskInfo); - if (pInfo->pRes->info.rows > pResultInfo->threshold || (!pInfo->multigroupResult && pInfo->pRes->info.rows > 0)) { - return pInfo->pRes; + if (pResBlock->info.rows > pResultInfo->threshold || (!pInfo->multigroupResult && pResBlock->info.rows > 0)) { + return pResBlock; } SOperatorInfo* pDownstream = pOperator->pDownstream[0]; @@ -5543,25 +5547,25 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { } } - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pInfo->capacity, pInfo->p); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pResBlock, pOperator->resultInfo.capacity, pInfo->p); // current group has no more result to return - if (pInfo->pRes->info.rows > 0) { + if (pResBlock->info.rows > 0) { // 1. The result in current group not reach the threshold of output result, continue // 2. If multiple group results existing in one SSDataBlock is not allowed, return immediately - if (pInfo->pRes->info.rows > pResultInfo->threshold || pBlock == NULL || (!pInfo->multigroupResult)) { - return pInfo->pRes; + if (pResBlock->info.rows > pResultInfo->threshold || pBlock == NULL || (!pInfo->multigroupResult)) { + return pResBlock; } doHandleRemainBlockFromNewGroup(pInfo, pResultInfo, newgroup, pTaskInfo); - if (pInfo->pRes->info.rows > pOperator->resultInfo.threshold || pBlock == NULL) { - return pInfo->pRes; + if (pResBlock->info.rows > pOperator->resultInfo.threshold || pBlock == NULL) { + return pResBlock; } } else if (pInfo->existNewGroupBlock) { // try next group assert(pBlock != NULL); doHandleRemainBlockForNewGroupImpl(pInfo, pResultInfo, newgroup, pTaskInfo); - if (pInfo->pRes->info.rows > pResultInfo->threshold) { - return pInfo->pRes; + if (pResBlock->info.rows > pResultInfo->threshold) { + return pResBlock; } } else { return NULL; @@ -5607,7 +5611,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); - pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize); + pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t)); pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); @@ -5870,8 +5874,7 @@ _error: SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, - SExecTaskInfo* pTaskInfo) { + STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5882,10 +5885,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pInfo->interval = *pInterval; pInfo->execModel = pTaskInfo->execModel; pInfo->win = pTaskInfo->window; - pInfo->win.skey = 0; - pInfo->win.ekey = INT64_MAX; - pInfo->primaryTsIndex = primaryTsSlotId; pInfo->twAggSup = *pTwAggSupp; + pInfo->primaryTsIndex = primaryTsSlotId; int32_t numOfRows = 4096; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; @@ -6055,18 +6056,14 @@ _error: static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, int64_t* fillVal, STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType) { - struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, (int64_t*)fillVal); - - TSKEY sk = TMIN(win.skey, win.ekey); - TSKEY ek = TMAX(win.skey, win.ekey); + SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, NULL); // TODO set correct time precision STimeWindow w = TSWINDOW_INITIALIZER; - getAlignQueryTimeWindow(pInterval, TSDB_TIME_PRECISION_MILLI, win.skey, sk, ek, &w); + getAlignQueryTimeWindow(pInterval, TSDB_TIME_PRECISION_MILLI, win.skey, &w); int32_t order = TSDB_ORDER_ASC; - pInfo->pFillInfo = taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval->sliding, - pInterval->slidingUnit, (int8_t)pInterval->precision, fillType, pColInfo, id); + pInfo->pFillInfo = taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval, fillType, pColInfo, id); pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); @@ -6087,23 +6084,37 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp pInfo->multigroupResult = multigroupResult; pInfo->intervalInfo = *pInterval; + int32_t type = TSDB_FILL_NONE; + switch (fillType) { + case FILL_MODE_PREV: type = TSDB_FILL_PREV;break; + case FILL_MODE_NONE: type = TSDB_FILL_NONE;break; + case FILL_MODE_NULL: type = TSDB_FILL_NULL;break; + case FILL_MODE_NEXT: type = TSDB_FILL_NEXT;break; + case FILL_MODE_VALUE: type = TSDB_FILL_SET_VALUE;break; + case FILL_MODE_LINEAR: type = TSDB_FILL_LINEAR;break; + default: + type = TSDB_FILL_NONE; + } + SResultInfo* pResultInfo = &pOperator->resultInfo; - int32_t code = initFillInfo(pInfo, pExpr, numOfCols, (int64_t*)fillVal, pTaskInfo->window, pResultInfo->capacity, - pTaskInfo->id.str, pInterval, fillType); + initResultSizeInfo(pOperator, 4096); + + int32_t code = initFillInfo(pInfo, pExpr, numOfCols, (int64_t*)fillVal, pTaskInfo->window, pResultInfo->capacity, + pTaskInfo->id.str, pInterval, type); if (code != TSDB_CODE_SUCCESS) { goto _error; } - pOperator->name = "FillOperator"; + pOperator->name = "FillOperator"; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; + pOperator->status = OP_NOT_OPENED; // pOperator->operatorType = OP_Fill; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doFill; - pOperator->pTaskInfo = pTaskInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doFill; + pOperator->pTaskInfo = pTaskInfo; pOperator->closeFn = destroySFillOperatorInfo; @@ -6470,9 +6481,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t numOfCols = 0; tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); - if (pDataReader == NULL) { - return NULL; - } SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); @@ -6574,6 +6582,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t primaryTsSlotId = ((SColumnNode*) pIntervalPhyNode->window.pTspk)->slotId; pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, primaryTsSlotId, &as, pTableGroupInfo, pTaskInfo); + + if (pIntervalPhyNode->pFill != NULL) { + pOptr = createFillOperatorInfo(pOptr, pExprInfo, num, &interval, pResBlock, pIntervalPhyNode->pFill->mode, NULL, false, pTaskInfo); + } + } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { SSortPhysiNode* pSortPhyNode = (SSortPhysiNode*)pPhyNode; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 8018a8dd31022a01cf51f3f68a010ae9ec37c83b..e51284bfef6b4182ebdd5eea3f1f2236c32af6bf 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -265,7 +265,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { - toSDatablock(pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + doBuildResultDatablock(pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -311,7 +311,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); while(1) { - toSDatablock(pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + doBuildResultDatablock(pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); doFilter(pInfo->pCondition, pRes); bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo); @@ -341,7 +341,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx pInfo->pScalarExprInfo = pScalarExprInfo; pInfo->numOfScalarExpr = numOfScalarExpr; - pInfo->pScalarFuncCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pInfo->binfo.rowCellInfoOffset); + pInfo->pScalarFuncCtx = createSqlFunctionCtx(pScalarExprInfo, numOfScalarExpr, &pInfo->rowCellInfoOffset); int32_t code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pGroupColList); if (code != TSDB_CODE_SUCCESS) { @@ -410,8 +410,8 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t contentLen = 0; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - int32_t* offset = pPage + startOffset; - columnLen = pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity; + int32_t* offset = (int32_t*)((char*)pPage + startOffset); + columnLen = (char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity; char* data = (char*)(columnLen + sizeof(int32_t)); if (colDataIsNull_s(pColInfoData, j)) { @@ -424,8 +424,8 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { contentLen = varDataTLen(src); } } else { - char* bitmap = pPage + startOffset; - columnLen = pPage + startOffset + BitmapLen(pInfo->rowCapacity); + char* bitmap = (char*)pPage + startOffset; + columnLen = (char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity); char* data = (char*) columnLen + sizeof(int32_t); bool isNull = colDataIsNull_f(pColInfoData->nullbitmap, j); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6b06f3e89bdb1535129350f423504b1e43230ea5..3a9742d48aecd515c360bfbad7324f16fbdfdc42 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -30,12 +30,11 @@ #include "query.h" #include "tcompare.h" #include "thash.h" -#include "vnode.h" #include "ttypes.h" +#include "vnode.h" #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) -#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) - +#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t i = 0; i < numOfOutput; ++i) { @@ -90,7 +89,7 @@ static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord } int64_t key = tw->skey, interval = pInterval->interval; - //convert key to second + // convert key to second key = convertTimePrecision(key, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; if (pInterval->intervalUnit == 'y') { @@ -98,7 +97,7 @@ static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord } struct tm tm; - time_t t = (time_t)key; + time_t t = (time_t)key; taosLocalTime(&t, &tm); int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); @@ -125,18 +124,18 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn // todo handle the time range case TSKEY sk = INT64_MIN; TSKEY ek = INT64_MAX; -// TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); -// TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); + // TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); + // TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); if (true) { - getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey, sk, ek, &w); + getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey, &w); assert(w.ekey >= pBlockInfo->window.skey); if (w.ekey < pBlockInfo->window.ekey) { return true; } - while(1) { // todo handle the desc order scan case + while (1) { // todo handle the desc order scan case getNextTimeWindow(pInterval, &w, TSDB_ORDER_ASC); if (w.skey > pBlockInfo->window.ekey) { break; @@ -148,31 +147,31 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } } } else { -// getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); -// assert(w.skey <= pBlockInfo->window.ekey); -// -// if (w.skey > pBlockInfo->window.skey) { -// return true; -// } -// -// while(1) { -// getNextTimeWindow(pQueryAttr, &w); -// if (w.ekey < pBlockInfo->window.skey) { -// break; -// } -// -// assert(w.skey < pBlockInfo->window.skey); -// if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) { -// return true; -// } -// } + // getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); + // assert(w.skey <= pBlockInfo->window.ekey); + // + // if (w.skey > pBlockInfo->window.skey) { + // return true; + // } + // + // while(1) { + // getNextTimeWindow(pQueryAttr, &w); + // if (w.ekey < pBlockInfo->window.skey) { + // break; + // } + // + // assert(w.skey < pBlockInfo->window.skey); + // if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) { + // return true; + // } + // } } return false; } int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableScanInfo* pInfo = pOperator->info; STaskCostInfo* pCost = &pTaskInfo->cost; @@ -189,13 +188,13 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, taosMemoryFreeClear(pBlock->pBlockAgg); if (*status == FUNC_DATA_REQUIRED_FILTEROUT) { - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, - pBlockInfo->window.ekey, pBlockInfo->rows); + qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { - qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, - pBlockInfo->window.ekey, pBlockInfo->rows); + qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { @@ -218,12 +217,12 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, } return TSDB_CODE_SUCCESS; - } else { // failed to load the block sma data, data block statistics does not exist, load data block instead + } else { // failed to load the block sma data, data block statistics does not exist, load data block instead *status = FUNC_DATA_REQUIRED_DATA_LOAD; } } - ASSERT (*status == FUNC_DATA_REQUIRED_DATA_LOAD); + ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD); // todo filter data block according to the block sma data firstly #if 0 @@ -249,8 +248,8 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, doFilter(pTableScanInfo->pFilterNode, pBlock); if (pBlock->info.rows == 0) { pCost->filterOutBlocks += 1; - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, - pBlockInfo->window.ekey, pBlockInfo->rows); + qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); } return TSDB_CODE_SUCCESS; @@ -320,7 +319,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // The read handle is not initialized yet, since no qualified tables exists - if (pTableScanInfo->dataReader == NULL) { + if (pTableScanInfo->dataReader == NULL || pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -348,9 +347,9 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; -// if (pResultRowInfo->size > 0) { -// pResultRowInfo->curPos = 0; -// } + // if (pResultRowInfo->size > 0) { + // pResultRowInfo->curPos = 0; + // } qDebug("%s start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); @@ -367,7 +366,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); if (pResultRowInfo->size > 0) { -// pResultRowInfo->curPos = pResultRowInfo->size - 1; + // pResultRowInfo->curPos = pResultRowInfo->size - 1; } p = doTableScanImpl(pOperator, newgroup); @@ -376,7 +375,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { return p; } -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t dataLoadFlag, +SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, int32_t order, int32_t numOfOutput, int32_t dataLoadFlag, int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0); @@ -396,7 +395,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, pInfo->dataBlockLoadFlag= dataLoadFlag; pInfo->pResBlock = pResBlock; pInfo->pFilterNode = pCondition; - pInfo->dataReader = pTsdbReadHandle; + pInfo->dataReader = pDataReader; pInfo->times = repeatTime; pInfo->reverseTimes = reverseTime; pInfo->order = order; @@ -406,11 +405,11 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, pOperator->name = "TableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->getNextFn = doTableScan; - pOperator->pTaskInfo = pTaskInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->getNextFn = doTableScan; + pOperator->pTaskInfo = pTaskInfo; static int32_t cost = 0; pOperator->cost.openCost = ++cost; @@ -451,72 +450,72 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) { STableBlockDistInfo tableBlockDist = {0}; tableBlockDist.numOfTables = 1; // TODO set the correct number of tables - int32_t numRowSteps = TSDB_DEFAULT_MAX_ROW_FBLOCK / TSDB_BLOCK_DIST_STEP_ROWS; - if (TSDB_DEFAULT_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { + int32_t numRowSteps = TSDB_DEFAULT_MAXROWS_FBLOCK / TSDB_BLOCK_DIST_STEP_ROWS; + if (TSDB_DEFAULT_MAXROWS_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { ++numRowSteps; } - tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); + tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps); tableBlockDist.maxRows = INT_MIN; tableBlockDist.minRows = INT_MAX; tsdbGetFileBlocksDistInfo(pTableScanInfo->dataReader, &tableBlockDist); - tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->dataReader); + tableBlockDist.numOfRowsInMemTable = (int32_t)tsdbGetNumOfRowsInMemTable(pTableScanInfo->dataReader); SSDataBlock* pBlock = pTableScanInfo->pResBlock; - pBlock->info.rows = 1; + pBlock->info.rows = 1; pBlock->info.numOfCols = 1; -// SBufferWriter bw = tbufInitWriter(NULL, false); -// blockDistInfoToBinary(&tableBlockDist, &bw); + // SBufferWriter bw = tbufInitWriter(NULL, false); + // blockDistInfoToBinary(&tableBlockDist, &bw); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); -// int32_t len = (int32_t) tbufTell(&bw); -// pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t)); -// *(int32_t*) pColInfo->pData = len; -// memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); -// -// tbufCloseWriter(&bw); + // int32_t len = (int32_t) tbufTell(&bw); + // pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t)); + // *(int32_t*) pColInfo->pData = len; + // memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); + // + // tbufCloseWriter(&bw); -// SArray* g = GET_TABLEGROUP(pOperator->, 0); -// pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); + // SArray* g = GET_TABLEGROUP(pOperator->, 0); + // pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); pOperator->status = OP_EXEC_DONE; return pBlock; } SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo) { - STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } - pInfo->dataReader = dataReader; -// pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + pInfo->dataReader = dataReader; + // pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_BINARY; + infoData.info.type = TSDB_DATA_TYPE_BINARY; infoData.info.bytes = 1024; infoData.info.colId = 0; -// taosArrayPush(pInfo->block.pDataBlock, &infoData); + // taosArrayPush(pInfo->block.pDataBlock, &infoData); - pOperator->name = "DataBlockInfoScanOperator"; + pOperator->name = "DataBlockInfoScanOperator"; // pOperator->operatorType = OP_TableBlockInfoScan; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doBlockInfoScan; + pOperator->blockingOptr = false; + pOperator->status = OP_NOT_OPENED; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doBlockInfoScan; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; return pOperator; - _error: +_error: taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pOperator); return NULL; @@ -558,29 +557,43 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) blockDataCleanup(pInfo->pRes); while (tqNextDataBlock(pInfo->readerHandle)) { - pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - terrno = pTaskInfo->code; - pOperator->status = OP_EXEC_DONE; + SArray* pCols = NULL; + uint64_t groupId; + int32_t numOfRows; + int16_t outputCol; + int32_t code = tqRetrieveDataBlock(&pCols, pInfo->readerHandle, &groupId, &numOfRows, &outputCol); + + if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { + pTaskInfo->code = code; return NULL; } - if (pBlockInfo->rows == 0) { - break; - } - - SArray* pCols = tqRetrieveDataBlock(pInfo->readerHandle); + pInfo->pRes->info.groupId = groupId; + pInfo->pRes->info.rows = numOfRows; int32_t numOfCols = pInfo->pRes->info.numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* p = taosArrayGet(pCols, i); - SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); + SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); if (!pColMatchInfo->output) { continue; } - ASSERT(pColMatchInfo->colId == p->info.colId); - taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, p); + bool colExists = false; + for (int32_t j = 0; j < taosArrayGetSize(pCols); ++j) { + SColumnInfoData* pResCol = taosArrayGet(pCols, j); + if (pResCol->info.colId == pColMatchInfo->colId) { + taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol); + colExists = true; + break; + } + } + + // the required column does not exists in submit block, let's set it to be all null value + if (!colExists) { + SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId); + colInfoDataEnsureCapacity(pDst, 0, pBlockInfo->rows); + colDataAppendNNULL(pDst, 0, pBlockInfo->rows); + } } if (pInfo->pRes->pDataBlock == NULL) { @@ -605,7 +618,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) } } -SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, + SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -618,7 +632,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* int32_t numOfOutput = taosArrayGetSize(pColList); SArray* pColIds = taosArrayInit(4, sizeof(int16_t)); - for(int32_t i = 0; i < numOfOutput; ++i) { + for (int32_t i = 0; i < numOfOutput; ++i) { int16_t* id = taosArrayGet(pColList, i); taosArrayPush(pColIds, id); } @@ -644,16 +658,16 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pInfo->readerHandle = streamReadHandle; pInfo->pRes = pResBlock; - pOperator->name = "StreamBlockScanOperator"; + pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doStreamBlockScan; - pOperator->closeFn = operatorDummyCloseFn; - pOperator->pTaskInfo = pTaskInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = pResBlock->info.numOfCols; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doStreamBlockScan; + pOperator->closeFn = operatorDummyCloseFn; + pOperator->pTaskInfo = pTaskInfo; return pOperator; } @@ -663,7 +677,8 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) { tsem_destroy(&pInfo->ready); blockDataDestroy(pInfo->pRes); - if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + const char* name = tNameGetTableName(&pInfo->name); + if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { metaCloseTbCursor(pInfo->pCur); } } @@ -733,9 +748,9 @@ static int32_t loadSysTableContentCb(void* param, const SDataBuf* pMsg, int32_t SRetrieveMetaTableRsp* pRsp = pScanResInfo->pRsp; pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->handle = htobe64(pRsp->handle); - pRsp->compLen = htonl(pRsp->compLen); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->handle = htobe64(pRsp->handle); + pRsp->compLen = htonl(pRsp->compLen); } else { operator->pTaskInfo->code = code; } @@ -777,7 +792,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { if (rowRes[j] == 0) { continue; } - + colDataAppend(pDest, numOfRow, colDataGetData(pSrc, j), false); numOfRow += 1; } @@ -798,7 +813,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { SSysTableScanInfo* pInfo = pOperator->info; // retrieve local table list info from vnode - if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + const char* name = tNameGetTableName(&pInfo->name); + if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { if (pInfo->pCur == NULL) { pInfo->pCur = metaOpenTbCursor(pInfo->readHandle); } @@ -828,7 +844,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, i); int64_t tmp = 0; char t[10] = {0}; - STR_TO_VARSTR(t, "_"); //TODO + STR_TO_VARSTR(t, "_"); // TODO if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { colDataAppend(pColInfoData, numOfRows, t, false); } else { @@ -850,8 +866,6 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { while (1) { int64_t startTs = taosGetTimestampUs(); - - pInfo->req.type = pInfo->type; strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); if (pInfo->showRewrite) { @@ -926,75 +940,16 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB return NULL; } - pInfo->accountId = accountId; + pInfo->accountId = accountId; pInfo->showRewrite = showRewrite; - pInfo->pRes = pResBlock; - pInfo->capacity = 4096; - pInfo->pCondition = pCondition; - pInfo->scanCols = colList; - - // TODO remove it - int32_t tableType = 0; - const char* name = tNameGetTableName(pName); - if (strncasecmp(name, TSDB_INS_TABLE_DNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_DNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_MNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_MNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_MODULES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_MODULE; - } else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_QNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_BNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_BNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_SNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_CLUSTER; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_DATABASES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_DB; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_FUNC; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, tListLen(pName->tname)) == 0) { - // tableType = TSDB_MGMT_TABLE_INDEX; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_STB; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_STREAMS; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_TABLE; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, tListLen(pName->tname)) == 0) { - // tableType = TSDB_MGMT_TABLE_DIST; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_USER; - } else if (strncasecmp(name, TSDB_INS_TABLE_LICENCES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_GRANTS; - } else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_VGROUP; - } else if (strncasecmp(name, TSDB_INS_TABLE_TOPICS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_TOPICS; - } else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_CONSUMERS; - } else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_SUBSCRIBES; - } else if (strncasecmp(name, TSDB_INS_TABLE_TRANS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_TRANS; - } else if (strncasecmp(name, TSDB_INS_TABLE_SMAS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_SMAS; - } else if (strncasecmp(name, TSDB_INS_TABLE_CONFIGS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_CONFIGS; - } else if (strncasecmp(name, TSDB_INS_TABLE_CONNS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_CONNS; - } else if (strncasecmp(name, TSDB_INS_TABLE_QUERIES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_QUERIES; - } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_VNODES; - }else { - ASSERT(0); - } + pInfo->pRes = pResBlock; + pInfo->capacity = 4096; + pInfo->pCondition = pCondition; + pInfo->scanCols = colList; tNameAssign(&pInfo->name, pName); - pInfo->type = tableType; - if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + const char* name = tNameGetTableName(&pInfo->name); + if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { pInfo->readHandle = pSysTableReadHandle; blockDataEnsureCapacity(pInfo->pRes, pInfo->capacity); } else { @@ -1025,15 +980,15 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB #endif } - pOperator->name = "SysTableScanOperator"; + pOperator->name = "SysTableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->getNextFn = doSysTableScan; - pOperator->closeFn = destroySysScanOperator; - pOperator->pTaskInfo = pTaskInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = pResBlock->info.numOfCols; + pOperator->getNextFn = doSysTableScan; + pOperator->closeFn = destroySysScanOperator; + pOperator->pTaskInfo = pTaskInfo; return pOperator; } diff --git a/source/libs/function/src/tfill.c b/source/libs/executor/src/tfill.c similarity index 88% rename from source/libs/function/src/tfill.c rename to source/libs/executor/src/tfill.c index b6b53621877c7697b87470f7eacbcfb166f00bf2..aeed07c636cdd31cf31a50441f9c85f985b60fc4 100644 --- a/source/libs/function/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -13,17 +13,18 @@ * along with this program. If not, see . */ -#include +#include "function.h" #include "os.h" +#include "querynodes.h" #include "taosdef.h" #include "tmsg.h" #include "ttypes.h" #include "tfill.h" -#include "thash.h" #include "function.h" #include "tcommon.h" +#include "thash.h" #include "ttime.h" #define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC) @@ -41,7 +42,7 @@ static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags); SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - assert (pTag->col.colId == pCol->col.colId); +// assert (pTag->col.colId == pCol->col.colId); assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type); } } @@ -80,7 +81,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData } char* output = elePtrAt(data[i], pCol->col.bytes, index); - assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type); +// assignVal(output, p + pCol->offset, pCol->col.bytes, pCol->col.type); } } else { // no prev value yet, set the value for NULL setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); @@ -96,7 +97,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData } char* output = elePtrAt(data[i], pCol->col.bytes, index); - assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type); +// assignVal(output, p + pCol->offset, pCol->col.bytes, pCol->col.type); } } else { // no prev value yet, set the value for NULL setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); @@ -119,7 +120,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData continue; } - point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->col.offset}; + point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->offset}; point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes}; point = (SPoint){.key = pFillInfo->currentKey, .val = val1}; taosGetLinearInterpolationVal(&point, type, &point1, &point2, type); @@ -135,12 +136,13 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData } char* val1 = elePtrAt(data[i], pCol->col.bytes, index); - assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type); + assignVal(val1, (char*)&pCol->val, pCol->col.bytes, pCol->col.type); } } setTagsValue(pFillInfo, data, index); - pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision); + pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, + pFillInfo->interval.precision); pFillInfo->numOfCurrent++; } @@ -152,7 +154,7 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) { *next = taosMemoryCalloc(1, pFillInfo->rowSize); for (int i = 1; i < pFillInfo->numOfCols; i++) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - setNull(*next + pCol->col.offset, pCol->col.type, pCol->col.bytes); + setNull(*next + pCol->offset, pCol->col.type, pCol->col.bytes); } } @@ -160,7 +162,7 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* bu int32_t rowIndex = pFillInfo->index; for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - memcpy(buf + pCol->col.offset, srcData[i] + rowIndex * pCol->col.bytes, pCol->col.bytes); + memcpy(buf + pCol->offset, srcData[i] + rowIndex * pCol->col.bytes, pCol->col.bytes); } } @@ -227,21 +229,21 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputR if (i == 0 || (pCol->functionId != FUNCTION_COUNT && !isNull(src, pCol->col.type)) || (pCol->functionId == FUNCTION_COUNT && GET_INT64_VAL(src) != 0)) { assignVal(output, src, pCol->col.bytes, pCol->col.type); - memcpy(*prev + pCol->col.offset, src, pCol->col.bytes); + memcpy(*prev + pCol->offset, src, pCol->col.bytes); } else { // i > 0 and data is null , do interpolation if (pFillInfo->type == TSDB_FILL_PREV) { - assignVal(output, *prev + pCol->col.offset, pCol->col.bytes, pCol->col.type); + assignVal(output, *prev + pCol->offset, pCol->col.bytes, pCol->col.type); } else if (pFillInfo->type == TSDB_FILL_LINEAR) { assignVal(output, src, pCol->col.bytes, pCol->col.type); - memcpy(*prev + pCol->col.offset, src, pCol->col.bytes); + memcpy(*prev + pCol->offset, src, pCol->col.bytes); } else if (pFillInfo->type == TSDB_FILL_NEXT) { if (*next) { - assignVal(output, *next + pCol->col.offset, pCol->col.bytes, pCol->col.type); + assignVal(output, *next + pCol->offset, pCol->col.bytes, pCol->col.type); } else { setNull(output, pCol->col.type, pCol->col.bytes); } } else { - assignVal(output, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type); + assignVal(output, (char*)&pCol->val, pCol->col.bytes, pCol->col.type); } } } @@ -250,7 +252,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputR setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent); pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, - pFillInfo->interval.slidingUnit, pFillInfo->precision); + pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); pFillInfo->index += 1; pFillInfo->numOfCurrent += 1; } @@ -301,7 +303,7 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t bool exists = false; int32_t index = -1; for (int32_t j = 0; j < k; ++j) { - if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) { + if (pFillInfo->pTags[j].col.colId == pColInfo->col.slotId) { exists = true; index = j; break; @@ -310,7 +312,7 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t if (!exists) { SSchema* pSchema = &pFillInfo->pTags[k].col; - pSchema->colId = pColInfo->col.colId; + pSchema->colId = pColInfo->col.slotId; pSchema->type = pColInfo->col.type; pSchema->bytes = pColInfo->col.bytes; @@ -341,30 +343,40 @@ static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { } struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, - int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - struct SFillColInfo* pCol, const char* id) { + SInterval* pInterval, int32_t fillType, struct SFillColInfo* pCol, const char* id) { if (fillType == TSDB_FILL_NONE) { return NULL; } SFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SFillInfo)); + if (pFillInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + taosResetFillInfo(pFillInfo, skey); - pFillInfo->order = order; + pFillInfo->order = order; + + switch(fillType) { + case FILL_MODE_NONE: pFillInfo->type = TSDB_FILL_NONE; break; + case FILL_MODE_PREV: pFillInfo->type = TSDB_FILL_PREV; break; + case FILL_MODE_NULL: pFillInfo->type = TSDB_FILL_NULL; break; + case FILL_MODE_LINEAR: pFillInfo->type = TSDB_FILL_LINEAR;break; + case FILL_MODE_NEXT: pFillInfo->type = TSDB_FILL_NEXT; break; + default: + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + pFillInfo->type = fillType; pFillInfo->pFillCol = pCol; pFillInfo->numOfTags = numOfTags; pFillInfo->numOfCols = numOfCols; - pFillInfo->precision = precision; pFillInfo->alloc = capacity; pFillInfo->id = id; - - pFillInfo->interval.interval = slidingTime; - pFillInfo->interval.intervalUnit = slidingUnit; - pFillInfo->interval.sliding = slidingTime; - pFillInfo->interval.slidingUnit = slidingUnit; - - pFillInfo->pData = taosMemoryMalloc(POINTER_BYTES * numOfCols); + pFillInfo->interval = *pInterval; + pFillInfo->pData = taosMemoryMalloc(POINTER_BYTES * numOfCols); // if (numOfTags > 0) { pFillInfo->pTags = taosMemoryCalloc(numOfCols, sizeof(SFillTagColInfo)); @@ -375,7 +387,6 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc); assert(pFillInfo->rowSize > 0); - return pFillInfo; } @@ -417,7 +428,7 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) pFillInfo->end = endKey; if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision); + pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->interval.precision); } pFillInfo->index = 0; @@ -433,7 +444,7 @@ void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - assert (pTag->col.colId == pCol->col.colId); + assert (pTag->col.colId == pCol->col.slotId); memcpy(pTag->tagVal, pColData->pData, pCol->col.bytes); // TODO not memcpy?? } } @@ -460,7 +471,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma TSKEY ekey1 = ekey; if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision); + pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->interval.precision); } int64_t numOfRes = -1; @@ -471,7 +482,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma pFillInfo->currentKey, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, - pFillInfo->precision); + pFillInfo->interval.precision); numOfRes += 1; assert(numOfRes >= numOfRows); } else { // reach the end of data @@ -484,7 +495,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma pFillInfo->currentKey, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, - pFillInfo->precision); + pFillInfo->interval.precision); numOfRes += 1; } @@ -527,7 +538,7 @@ int64_t getFillInfoStart(struct SFillInfo *pFillInfo) { return pFillInfo->start; } -struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const int64_t* fillVal) { +struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SValueNode* val) { int32_t offset = 0; struct SFillColInfo* pFillCol = taosMemoryCalloc(numOfOutput, sizeof(SFillColInfo)); @@ -538,14 +549,15 @@ struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, co for(int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExprInfo = &pExpr[i]; - pFillCol[i].col.bytes = pExprInfo->base.resSchema.bytes; - pFillCol[i].col.type = (int8_t)pExprInfo->base.resSchema.type; - pFillCol[i].col.offset = offset; - pFillCol[i].col.colId = pExprInfo->base.resSchema.slotId; + pFillCol[i].col = pExprInfo->base.resSchema; + pFillCol[i].offset = offset; pFillCol[i].tagIndex = -2; - pFillCol[i].flag = pExprInfo->base.pParam[0].pCol->flag; // always be the normal column for table query + + if (pExprInfo->base.numOfParams > 0) { + pFillCol[i].flag = pExprInfo->base.pParam[0].pCol->flag; // always be the normal column for table query + } // pFillCol[i].functionId = pExprInfo->pExpr->_function.functionId; - pFillCol[i].fillVal.i = fillVal[i]; +// pFillCol[i].val.d = *val; offset += pExprInfo->base.resSchema.bytes; } diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index fe370d93b259b25b9ce797b7fff841d66956fecd..8f7ff04d926257244483c82a6beb22fd8399e9b2 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -48,7 +48,7 @@ struct SSHashObj { }; static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { - int32_t len = MIN(length, HASH_MAX_CAPACITY); + int32_t len = (length < HASH_MAX_CAPACITY ? length : HASH_MAX_CAPACITY); int32_t i = 4; while (i < len) i = (i << 1u); @@ -127,7 +127,7 @@ static void taosHashTableResize(SSHashObj *pHashObj) { } size_t inc = newCapacity - pHashObj->capacity; - memset(pNewEntryList + pHashObj->capacity * sizeof(void*), 0, inc); + memset((char*)pNewEntryList + pHashObj->capacity * sizeof(void*), 0, inc); pHashObj->hashList = pNewEntryList; pHashObj->capacity = newCapacity; @@ -301,7 +301,7 @@ size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) { void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen) { int32_t offset = offsetof(SHNode, data); - SHNode *node = data - offset; + SHNode *node = ((SHNode*)(char*)data - offset); if (keyLen != NULL) { *keyLen = pHashObj->keyLen; } diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 870a0b71fc2fa2ef7cb113752398af276089956c..543df30686caf8f331fa935a72ef00275264407d 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -102,7 +102,7 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } else if (pInfo->type == data_asc) { v = ++pInfo->startVal; } else if (pInfo->type == data_rand) { - v = random(); + v = taosRand(); } colDataAppend(pColInfo, i, reinterpret_cast(&v), false); @@ -172,7 +172,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } else if (pInfo->type == data_asc) { v = ++pInfo->startVal; } else if (pInfo->type == data_rand) { - v = random(); + v = taosRand(); } colDataAppend(pColInfo1, i, reinterpret_cast(&v), false); @@ -938,7 +938,7 @@ TEST(testCase, build_executor_tree_Test) { SExecTaskInfo* pTaskInfo = nullptr; DataSinkHandle sinkHandle = nullptr; - SReadHandle handle = {.reader = reinterpret_cast(0x1), .meta = reinterpret_cast(0x1)}; + SReadHandle handle = { reinterpret_cast(0x1), reinterpret_cast(0x1), NULL }; struct SSubplan *plan = NULL; int32_t code = qStringToSubplan(msg, &plan); diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index aa909361ea091da6c576ba2b46309ea1bb249286..98f1209ad921ce9e86887a6972b4246ad7483d3a 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -5,13 +5,16 @@ target_include_directories( function PUBLIC "${TD_SOURCE_DIR}/include/libs/function" + "${TD_SOURCE_DIR}/include/util" + "${TD_SOURCE_DIR}/include/common" + "${TD_SOURCE_DIR}/include/client" "${TD_SOURCE_DIR}/contrib/libuv/include" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( function - PRIVATE os util common nodes scalar + PRIVATE os util common nodes scalar catalog qcom transport PUBLIC uv_a ) @@ -21,6 +24,9 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/include/libs/function" "${TD_SOURCE_DIR}/contrib/libuv/include" + "${TD_SOURCE_DIR}/include/util" + "${TD_SOURCE_DIR}/include/common" + "${TD_SOURCE_DIR}/include/client" "${TD_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) @@ -35,9 +41,15 @@ target_include_directories( udf1 PUBLIC "${TD_SOURCE_DIR}/include/libs/function" + "${TD_SOURCE_DIR}/include/util" + "${TD_SOURCE_DIR}/include/common" + "${TD_SOURCE_DIR}/include/client" "${TD_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) +target_link_libraries( + udf1 PUBLIC os +) #SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin) add_executable(udfd src/udfd.c) @@ -46,6 +58,10 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/include/libs/function" "${TD_SOURCE_DIR}/contrib/libuv/include" + "${TD_SOURCE_DIR}/include/util" + "${TD_SOURCE_DIR}/include/common" + "${TD_SOURCE_DIR}/include/libs/transport" + "${TD_SOURCE_DIR}/include/client" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) diff --git a/source/libs/function/inc/builtins.h b/source/libs/function/inc/builtins.h index fb36c9d978a1c653dbe48fdd80001cb6c1d08f1d..54a68715a877836b6b0104bfa6cda4e68b240f04 100644 --- a/source/libs/function/inc/builtins.h +++ b/source/libs/function/inc/builtins.h @@ -20,26 +20,7 @@ extern "C" { #endif -#include "functionMgt.h" - -#define FUNCTION_NAME_MAX_LENGTH 32 - -#define FUNC_MGT_FUNC_CLASSIFICATION_MASK(n) (1 << n) - -#define FUNC_MGT_AGG_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(0) -#define FUNC_MGT_SCALAR_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(1) -#define FUNC_MGT_NONSTANDARD_SQL_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(2) -#define FUNC_MGT_STRING_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(3) -#define FUNC_MGT_DATETIME_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(4) -#define FUNC_MGT_TIMELINE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(5) -#define FUNC_MGT_TIMEORDER_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(6) -#define FUNC_MGT_PSEUDO_COLUMN_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(7) -#define FUNC_MGT_WINDOW_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(8) -#define FUNC_MGT_SPECIAL_DATA_REQUIRED FUNC_MGT_FUNC_CLASSIFICATION_MASK(9) -#define FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED FUNC_MGT_FUNC_CLASSIFICATION_MASK(10) -#define FUNC_MGT_MULTI_RES_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(11) - -#define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) +#include "functionMgtInt.h" typedef int32_t (*FTranslateFunc)(SFunctionNode* pFunc, char* pErrBuf, int32_t len); typedef EFuncDataRequired (*FFuncDataRequired)(SFunctionNode* pFunc, STimeWindow* pTimeWindow); diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 11c89f15685b2e361b35171e2881dbe292357534..8473138e4af4fd5f075a2bb7a7ff75087b75d90b 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -58,6 +58,9 @@ bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t firstFunction(SqlFunctionCtx *pCtx); int32_t lastFunction(SqlFunctionCtx *pCtx); +bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); +int32_t topFunction(SqlFunctionCtx *pCtx); + #ifdef __cplusplus } #endif diff --git a/source/libs/function/inc/fnLog.h b/source/libs/function/inc/fnLog.h new file mode 100644 index 0000000000000000000000000000000000000000..f57294890794927e840a1a749f7e5adc5bbf9164 --- /dev/null +++ b/source/libs/function/inc/fnLog.h @@ -0,0 +1,24 @@ +// +// Created by slzhou on 22-4-20. +// + +#ifndef TDENGINE_FNLOG_H +#define TDENGINE_FNLOG_H +#include "tlog.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define fnFatal(...) { if (fnDebugFlag & DEBUG_FATAL) { taosPrintLog("FN FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define fnError(...) { if (fnDebugFlag & DEBUG_ERROR) { taosPrintLog("FN ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define fnWarn(...) { if (fnDebugFlag & DEBUG_WARN) { taosPrintLog("FN WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define fnInfo(...) { if (fnDebugFlag & DEBUG_INFO) { taosPrintLog("FN ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define fnDebug(...) { if (fnDebugFlag & DEBUG_DEBUG) { taosPrintLog("FN ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} +#define fnTrace(...) { if (fnDebugFlag & DEBUG_TRACE) { taosPrintLog("FN ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_FNLOG_H diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index 4baf338e9b750ea7ad35f6b63052dc2c05a57fc6..39287f08ee62a535265147855c1f611b7ab90d10 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -21,9 +21,29 @@ extern "C" { #endif #include "functionMgt.h" -// #include "builtins.h" +#define FUNCTION_NAME_MAX_LENGTH 32 +#define FUNC_MGT_FUNC_CLASSIFICATION_MASK(n) (1 << n) + +#define FUNC_MGT_AGG_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(0) +#define FUNC_MGT_SCALAR_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(1) +#define FUNC_MGT_NONSTANDARD_SQL_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(2) +#define FUNC_MGT_STRING_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(3) +#define FUNC_MGT_DATETIME_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(4) +#define FUNC_MGT_TIMELINE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(5) +#define FUNC_MGT_TIMEORDER_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(6) +#define FUNC_MGT_PSEUDO_COLUMN_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(7) +#define FUNC_MGT_WINDOW_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(8) +#define FUNC_MGT_SPECIAL_DATA_REQUIRED FUNC_MGT_FUNC_CLASSIFICATION_MASK(9) +#define FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED FUNC_MGT_FUNC_CLASSIFICATION_MASK(10) +#define FUNC_MGT_MULTI_RES_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(11) + +#define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) + +#define FUNC_UDF_ID_START_OFFSET_VAL 5000 + +extern const int funcMgtUdfNum; #ifdef __cplusplus } diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index 72875239d2c7f335d08ccf1c3af73108e4b04ef9..5f4f96c4ccc93b84036bb6ae14c1853231b92e10 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -16,47 +16,43 @@ #ifndef TDENGINE_TUDF_H #define TDENGINE_TUDF_H + +#include +#include +#include "tmsg.h" +#include "tcommon.h" +#include "function.h" + #ifdef __cplusplus extern "C" { #endif +#define UDF_LISTEN_PIPE_NAME_LEN 32 +#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock." + //====================================================================================== //begin API to taosd and qworker + +enum { + UDFC_CODE_STOPPING = -1, + UDFC_CODE_PIPE_READ_ERR = -3, +}; + +typedef void *UdfcHandle; +typedef void *UdfcFuncHandle; + /** - * start udf dameon service + * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf * @return error code */ -int32_t startUdfService(); +int32_t udfcOpen(int32_t dnodeId, UdfcHandle* proxyHandle); /** - * stop udf dameon service + * destroy udfd proxy * @return error code */ -int32_t stopUdfService(); - -enum { - TSDB_UDF_TYPE_SCALAR = 0, - TSDB_UDF_TYPE_AGGREGATE = 1 -}; +int32_t udfcClose(UdfcHandle proxyhandle); -enum { - TSDB_UDF_SCRIPT_BIN_LIB = 0, - TSDB_UDF_SCRIPT_LUA = 1, -}; - -typedef struct SUdfInfo { - char *udfName; // function name - int32_t udfType; // scalar function or aggregate function - int8_t scriptType; - char *path; - - int8_t resType; // result type - int16_t resBytes; // result byte - int32_t bufSize; //interbuf size - -} SUdfInfo; - -typedef void *UdfHandle; /** * setup udf @@ -64,69 +60,94 @@ typedef void *UdfHandle; * @param handle, out * @return error code */ -int32_t setupUdf(SUdfInfo* udf, UdfHandle *handle); - - -enum { - TSDB_UDF_STEP_NORMAL = 0, - TSDB_UDF_STEP_MERGE, - TSDb_UDF_STEP_FINALIZE, - TSDB_UDF_STEP_MAX_NUM -}; -/** - * call udf - * @param handle udf handle - * @param step - * @param state - * @param stateSize - * @param input - * @param newstate - * @param newStateSize - * @param output - * @return error code - */ +int32_t setupUdf(UdfcHandle proxyHandle, char udfName[], SEpSet *epSet, UdfcFuncHandle *handle); + +typedef struct SUdfColumnMeta { + int16_t type; + int32_t bytes; // <0 var length, others fixed length bytes + uint8_t precision; + uint8_t scale; +} SUdfColumnMeta; + +typedef struct SUdfColumnData { + int32_t numOfRows; + bool varLengthColumn; + union { + struct { + int32_t nullBitmapLen; + char *nullBitmap; + int32_t dataLen; + char *data; + } fixLenCol; + + struct { + int32_t varOffsetsLen; + char *varOffsets; + int32_t payloadLen; + char *payload; + } varLenCol; + }; +} SUdfColumnData; + + +typedef struct SUdfColumn { + SUdfColumnMeta colMeta; + SUdfColumnData colData; +} SUdfColumn; -//TODO: must change the following after metadata flow and data flow between qworker and udfd is well defined typedef struct SUdfDataBlock { - char* data; - int32_t size; + int32_t numOfRows; + int32_t numOfCols; + SUdfColumn **udfCols; } SUdfDataBlock; -int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, char **newstate, - int32_t *newStateSize, SUdfDataBlock *output); - +typedef struct SUdfInterBuf { + int32_t bufLen; + char* buf; +} SUdfInterBuf; + +// output: interBuf +int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf); +// input: block, state +// output: newState +int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState); +// input: interBuf +// output: resultData +int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData); +// input: interbuf1, interbuf2 +// output: resultBuf +int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf); +// input: block +// output: resultData +int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam *output); /** * tearn down udf * @param handle * @return */ -int32_t teardownUdf(UdfHandle handle); +int32_t teardownUdf(UdfcFuncHandle handle); // end API to taosd and qworker //============================================================================================================================= -// TODO: Must change // begin API to UDF writer. -// script +// dynamic lib init and destroy +typedef int32_t (*TUdfSetupFunc)(); +typedef int32_t (*TUdfTeardownFunc)(); -//typedef int32_t (*scriptInitFunc)(void* pCtx); -//typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows, -// int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput, -// int16_t oType, int16_t oBytes); -//typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput); -//typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput); -//typedef void (*scriptDestroyFunc)(void* pCtx); +//TODO: add API to check function arguments type, number etc. +//TODO: another way to manage memory is provide api for UDF to add data to SUdfColumnData and UDF framework will allocate memory. +// then UDF framework will free the memory +//typedef int32_t addFixedLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t colBytes, char* data); +//typedef int32_t addVariableLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t dataLen, char * data); -// dynamic lib -typedef int32_t (*TUdfInitFunc)(); -typedef void (*TUdfDestroyFunc)(); +typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column); -typedef void (*TUdfFunc)(int8_t step, - char *state, int32_t stateSize, SUdfDataBlock input, - char **newstate, int32_t *newStateSize, SUdfDataBlock *output); +typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock block, SUdfColumn *resultCol); +typedef int32_t (*TUdfAggInitFunc)(SUdfInterBuf *buf); +typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock block, SUdfInterBuf *interBuf); +typedef int32_t (*TUdfAggFinalizeFunc)(SUdfInterBuf buf, SUdfInterBuf *resultData); -//typedef void (*udfMergeFunc)(char *data, int32_t numOfRows, char *dataOutput, int32_t* numOfOutput); -//typedef void (*udfFinalizeFunc)(char* state, int32_t stateSize, SUdfDataBlock *output); // end API to UDF writer //======================================================================================================================= diff --git a/source/libs/function/inc/tudfInt.h b/source/libs/function/inc/tudfInt.h index 5f757c1ef0a36d47d52a1b74670d3f1b888b676f..2c16afbd0dd940e59fd9962961240e4cec4bab94 100644 --- a/source/libs/function/inc/tudfInt.h +++ b/source/libs/function/inc/tudfInt.h @@ -15,7 +15,6 @@ #ifndef TDENGINE_TUDF_INT_H #define TDENGINE_TUDF_INT_H - #ifdef __cplusplus extern "C" { #endif @@ -30,36 +29,37 @@ enum { }; +enum { + TSDB_UDF_CALL_AGG_INIT = 0, + TSDB_UDF_CALL_AGG_PROC, + TSDB_UDF_CALL_AGG_MERGE, + TSDB_UDF_CALL_AGG_FIN, + TSDB_UDF_CALL_SCALA_PROC, +}; + typedef struct SUdfSetupRequest { - char udfName[16]; // - int8_t scriptType; // 0:c, 1: lua, 2:js - int8_t udfType; //udaf, udf - int16_t pathSize; - char *path; + char udfName[TSDB_FUNC_NAME_LEN]; + SEpSet epSet; } SUdfSetupRequest; typedef struct SUdfSetupResponse { int64_t udfHandle; } SUdfSetupResponse; - typedef struct SUdfCallRequest { int64_t udfHandle; - int8_t step; - - int32_t inputBytes; - char *input; + int8_t callType; - int32_t stateBytes; - char *state; + SSDataBlock block; + SUdfInterBuf interBuf; + SUdfInterBuf interBuf2; + int8_t initFirst; } SUdfCallRequest; - typedef struct SUdfCallResponse { - int32_t outputBytes; - char *output; - int32_t newStateBytes; - char *newState; + int8_t callType; + SSDataBlock resultData; + SUdfInterBuf resultBuf; } SUdfCallResponse; @@ -69,6 +69,9 @@ typedef struct SUdfTeardownRequest { typedef struct SUdfTeardownResponse { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } SUdfTeardownResponse; typedef struct SUdfRequest { @@ -76,7 +79,11 @@ typedef struct SUdfRequest { int64_t seqNum; int8_t type; - void *subReq; + union { + SUdfSetupRequest setup; + SUdfCallRequest call; + SUdfTeardownRequest teardown; + }; } SUdfRequest; typedef struct SUdfResponse { @@ -85,13 +92,25 @@ typedef struct SUdfResponse { int8_t type; int32_t code; - void *subRsp; + union { + SUdfSetupResponse setupRsp; + SUdfCallResponse callRsp; + SUdfTeardownResponse teardownRsp; + }; } SUdfResponse; -int32_t decodeRequest(char *buf, int32_t bufLen, SUdfRequest **pRequest); -int32_t encodeResponse(char **buf, int32_t *bufLen, SUdfResponse *response); -int32_t encodeRequest(char **buf, int32_t *bufLen, SUdfRequest *request); -int32_t decodeResponse(char *buf, int32_t bufLen, SUdfResponse **pResponse); +int32_t encodeUdfRequest(void **buf, const SUdfRequest* request); +void* decodeUdfRequest(const void *buf, SUdfRequest* request); + +int32_t encodeUdfResponse(void **buf, const SUdfResponse *response); +void* decodeUdfResponse(const void* buf, SUdfResponse *response); + +void freeUdfColumnData(SUdfColumnData *data); +void freeUdfColumn(SUdfColumn* col); +void freeUdfDataDataBlock(SUdfDataBlock *block); + +int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock); +int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block); #ifdef __cplusplus } diff --git a/source/libs/function/inc/udfc.h b/source/libs/function/inc/udfc.h index 4d2aeb7049c21a07d1875c2ad9169ee3b5c9212b..a693e476e83b05659509dc4718d5f27581ce2b52 100644 --- a/source/libs/function/inc/udfc.h +++ b/source/libs/function/inc/udfc.h @@ -30,20 +30,20 @@ typedef struct SUdfInfo { char *path; } SUdfInfo; -typedef void *UdfHandle; +typedef void *UdfcFuncHandle; -int32_t startUdfService(); +int32_t createUdfdProxy(); -int32_t stopUdfService(); +int32_t destroyUdfdProxy(); -//int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfHandle *handles); +//int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfcFuncHandle *handles); -int32_t setupUdf(SUdfInfo* udf, UdfHandle* handle); +int32_t setupUdf(SUdfInfo* udf, UdfcFuncHandle* handle); -int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate, +int32_t callUdf(UdfcFuncHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate, int32_t *newStateSize, SSDataBlock *output); -int32_t teardownUdf(UdfHandle handle); +int32_t teardownUdf(UdfcFuncHandle handle); typedef struct SUdfSetupRequest { char udfName[16]; // @@ -84,6 +84,9 @@ typedef struct SUdfTeardownRequest { typedef struct SUdfTeardownResponse { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } SUdfTeardownResponse; typedef struct SUdfRequest { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 78dec26be59b067ef7fddd66b98b12b01e1a33e4..efd8de84c40b233f8a3a18f144b5112ec5993a2b 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -28,11 +28,15 @@ static int32_t buildFuncErrMsg(char* pErrBuf, int32_t len, int32_t errCode, cons } static int32_t invaildFuncParaNumErrMsg(char* pErrBuf, int32_t len, const char* pFuncName) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_NUM, "Invalid number of arguments : %s", pFuncName); + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_NUM, "Invalid number of parameters : %s", pFuncName); } static int32_t invaildFuncParaTypeErrMsg(char* pErrBuf, int32_t len, const char* pFuncName) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_TYPE, "Inconsistent datatypes : %s", pFuncName); + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_TYPE, "Invalid parameter data type : %s", pFuncName); +} + +static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char* pFuncName) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName); } // There is only one parameter of numeric type, and the return type is parameter type @@ -139,8 +143,9 @@ static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, in } static int32_t translateTimezone(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - // pseudo column do not need to check parameters - pFunc->node.resType = (SDataType){.bytes = TD_TIMEZONE_LEN, .type = TSDB_DATA_TYPE_BINARY}; + SExprNode* pPara1 = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); + pFunc->node.resType = (SDataType){.bytes = pPara1->resType.bytes, .type = pPara1->resType.type}; + //pFunc->node.resType = (SDataType){.bytes = TD_TIMEZONE_LEN, .type = TSDB_DATA_TYPE_BINARY}; return TSDB_CODE_SUCCESS; } @@ -188,8 +193,15 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t return TSDB_CODE_SUCCESS; } +static int32_t translateTbnameColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + // pseudo column do not need to check parameters + pFunc->node.resType = (SDataType){.bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}; + return TSDB_CODE_SUCCESS; +} + static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - // todo + SDataType* pType = &((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; + pFunc->node.resType = (SDataType){.bytes = pType->bytes, .type = pType->type}; return TSDB_CODE_SUCCESS; } @@ -317,10 +329,19 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; // The function return type has been set during syntax parsing uint8_t para2Type = pFunc->node.resType.type; - if ((TSDB_DATA_TYPE_JSON == para1Type || TSDB_DATA_TYPE_BLOB == para1Type || TSDB_DATA_TYPE_MEDIUMBLOB == para1Type) || - (TSDB_DATA_TYPE_JSON == para2Type || TSDB_DATA_TYPE_BLOB == para2Type || TSDB_DATA_TYPE_MEDIUMBLOB == para2Type)) { + if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT && + para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR && + para2Type != TSDB_DATA_TYPE_TIMESTAMP) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) || + (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + int32_t para2Bytes = pFunc->node.resType.bytes; + if (para2Bytes <= 0) { //non-positive value or overflow + return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); + } return TSDB_CODE_SUCCESS; } @@ -483,9 +504,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_TOP, .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateTop, - .getEnvFunc = getMinmaxFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, + .getEnvFunc = getTopBotFuncEnv, + .initFunc = functionSetup, + .processFunc = topFunction, .finalizeFunc = functionFinalize }, { @@ -862,7 +883,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "tbname", .type = FUNCTION_TYPE_TBNAME, .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, - .translateFunc = NULL, + .translateFunc = translateTbnameColumn, .getEnvFunc = NULL, .initFunc = NULL, .sprocessFunc = NULL, @@ -900,7 +921,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { }, { .name = "_wendts", - .type = FUNCTION_TYPE_QENDTS, + .type = FUNCTION_TYPE_WENDTS, .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC, .translateFunc = translateTimePseudoColumn, .getEnvFunc = getTimePseudoFuncEnv, @@ -925,7 +946,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateToJson, .getEnvFunc = NULL, .initFunc = NULL, - .sprocessFunc = NULL, + .sprocessFunc = toJsonFunction, .finalizeFunc = NULL } }; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 2b1e4b9406d477c34e79ad4ccb0e6da4cd81f1ff..09427265889f921b7c023738dde788b9669495a4 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -14,10 +14,11 @@ */ #include "builtinsimpl.h" -#include "tpercentile.h" +#include #include "querynodes.h" #include "taggfunction.h" #include "tdatablock.h" +#include "tpercentile.h" #define SET_VAL(_info, numOfElem, res) \ do { \ @@ -472,17 +473,6 @@ int32_t maxFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } -typedef struct STopBotRes { - int32_t num; -} STopBotRes; - -bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { - SColumnNode* pColNode = (SColumnNode*) nodesListGetNode(pFunc->pParameterList, 0); - int32_t bytes = pColNode->node.resType.bytes; - SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1); - return true; -} - typedef struct SStddevRes { double result; int64_t count; @@ -523,7 +513,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { switch (type) { case TSDB_DATA_TYPE_TINYINT: { int8_t* plist = (int8_t*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + for (int32_t i = start; i < numOfRows + start; ++i) { if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { continue; } @@ -749,9 +739,9 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } -// TODO set the correct parameter. void percentileFinalize(SqlFunctionCtx* pCtx) { - double v = 50;//pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64 : pCtx->param[0].dKey; + SVariant* pVal = &pCtx->param[1].param; + double v = pVal->nType == TSDB_DATA_TYPE_INT ? pVal->i : pVal->d; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); @@ -819,6 +809,8 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) { continue; } + numOfElems++; + char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); @@ -828,9 +820,8 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) { // DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; + break; } - - numOfElems++; } } else { // in case of descending order time stamp serial, which usually happens as the results of the nest query, @@ -847,6 +838,8 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) { continue; } + numOfElems++; + char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); @@ -855,9 +848,8 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) { *(TSKEY*)(buf + bytes) = cts; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; + break; } - - numOfElems++; } } @@ -874,43 +866,55 @@ int32_t lastFunction(SqlFunctionCtx *pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; + int32_t bytes = pInputCol->info.bytes; + // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { ASSERT(pInputCol->hasNull == true); return 0; } - if (pCtx->order == TSDB_ORDER_DESC) { + SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet)? pInput->pColumnDataAgg[0]:NULL; + + TSKEY startKey = getRowPTs(pInput->pPTS, 0); + TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1); + + int32_t blockDataOrder = (startKey <= endKey)? TSDB_ORDER_ASC:TSDB_ORDER_DESC; + + if (blockDataOrder == TSDB_ORDER_ASC) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, NULL)) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { continue; } - char* data = colDataGetData(pInputCol, i); - memcpy(buf, data, pInputCol->info.bytes); - -// TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; -// DO_UPDATE_TAG_COLUMNS(pCtx, ts); - pResInfo->complete = true; // set query completed on this column numOfElems++; + + char* data = colDataGetData(pInputCol, i); + TSKEY cts = getRowPTs(pInput->pPTS, i); + if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) < cts) { + memcpy(buf, data, bytes); + *(TSKEY*)(buf + bytes) = cts; + // DO_UPDATE_TAG_COLUMNS(pCtx, ts); + pResInfo->numOfRes = 1; + } break; } - } else { // ascending order + } else { // descending order for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, NULL)) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { continue; } - char* data = colDataGetData(pInputCol, i); - TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; + numOfElems++; - if (pResInfo->numOfRes == 0 || (*(TSKEY*)buf) < ts) { - memcpy(buf, data, pCtx->inputBytes); - *(TSKEY*)buf = ts; + char* data = colDataGetData(pInputCol, i); + TSKEY cts = getRowPTs(pInput->pPTS, i); + if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) < cts) { + memcpy(buf, data, bytes); + *(TSKEY*)(buf + bytes) = cts; + pResInfo->numOfRes = 1; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); } - - numOfElems++; break; } } @@ -1159,3 +1163,130 @@ int32_t diffFunction(SqlFunctionCtx *pCtx) { } } +typedef struct STopBotResItem { + SVariant v; + uint64_t uid; // it is a table uid, used to extract tag data during building of the final result for the tag data + struct { + int32_t pageId; + int32_t offset; + } tuplePos; // tuple data of this chosen row +} STopBotResItem; + +typedef struct STopBotRes { + int32_t num; + STopBotResItem *pItems; +} STopBotRes; + +bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + SColumnNode* pColNode = (SColumnNode*) nodesListGetNode(pFunc->pParameterList, 0); + int32_t bytes = pColNode->node.resType.bytes; + SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1); + + pEnv->calcMemSize = sizeof(STopBotRes) + pkNode->datum.i * bytes; + return true; +} + +static STopBotRes *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + STopBotRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); + pRes->pItems = (STopBotResItem*)((char*) pRes + sizeof(STopBotRes)); + + return pRes; +} + +static void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, uint16_t type, uint64_t uid); + +int32_t topFunction(SqlFunctionCtx *pCtx) { + int32_t numOfElems = 0; + + STopBotRes *pRes = getTopBotOutputInfo(pCtx); + assert(pRes->num >= 0); + +// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotRes) + POINTER_BYTES * pCtx->param[0].i)) { +// buildTopBotStruct(pRes, pCtx); +// } + + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pCol = pInput->pData[0]; + + int32_t type = pInput->pData[0]->info.type; + + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; + + for (int32_t i = start; i < numOfRows + start; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + numOfElems++; + + char* data = colDataGetData(pCol, i); + doAddIntoResult(pRes, pCtx->param[1].param.i, data, type, pInput->uid); + } + + // treat the result as only one result + SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); + return TSDB_CODE_SUCCESS; +} + +static int32_t topBotResComparFn(const void *p1, const void *p2, const void *param) { + uint16_t type = *(uint16_t *) param; + + STopBotResItem *val1 = (STopBotResItem *) p1; + STopBotResItem *val2 = (STopBotResItem *) p2; + + if (IS_SIGNED_NUMERIC_TYPE(type)) { + if (val1->v.i == val2->v.i) { + return 0; + } + + return (val1->v.i > val2->v.i) ? 1 : -1; + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + if (val1->v.u == val2->v.u) { + return 0; + } + + return (val1->v.u > val2->v.u) ? 1 : -1; + } + + if (val1->v.d == val2->v.d) { + return 0; + } + + return (val1->v.d > val2->v.d) ? 1 : -1; +} + +void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, uint16_t type, uint64_t uid) { + SVariant val = {0}; + taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); + + STopBotResItem *pItems = pRes->pItems; + assert(pItems != NULL); + + // not full yet + if (pRes->num < maxSize) { + STopBotResItem* pItem = &pItems[pRes->num]; + pItem->v = val; + pItem->uid = uid; + pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer + + pRes->num++; + taosheapsort((void *) pItem, sizeof(STopBotResItem), pRes->num, (const void *) &type, topBotResComparFn, false); + } else { // replace the minimum value in the result + if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pItems[0].v.i) || + (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) || + (IS_FLOAT_TYPE(type) && val.d > pItems[0].v.d)) { + STopBotResItem* pItem = &pItems[pRes->num]; + pItem->v = val; + pItem->uid = uid; + pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer + + taosheapadjust((void *) pItem, sizeof(STopBotResItem), 0, pRes->num - 1, (const void *) &type, topBotResComparFn, NULL, false); + } + } +} + +void topBotFinalize(SqlFunctionCtx* pCtx) { + functionFinalize(pCtx); + +} \ No newline at end of file diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 950006494932fdd889745ae1304d09915bec9fb4..d44e3e251bd934e22e80a172ca4586e1b58970bf 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -20,16 +20,23 @@ #include "taoserror.h" #include "thash.h" #include "builtins.h" +#include "catalog.h" typedef struct SFuncMgtService { SHashObj* pFuncNameHashTable; + SArray* pUdfTable; // SUdfInfo } SFuncMgtService; +typedef struct SUdfInfo { + SDataType outputDt; + int8_t funcType; +} SUdfInfo; + static SFuncMgtService gFunMgtService; static TdThreadOnce functionHashTableInit = PTHREAD_ONCE_INIT; static int32_t initFunctionCode = 0; -static void doInitFunctionHashTable() { +static void doInitFunctionTable() { gFunMgtService.pFuncNameHashTable = taosHashInit(funcMgtBuiltinsNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (NULL == gFunMgtService.pFuncNameHashTable) { initFunctionCode = TSDB_CODE_FAILED; @@ -42,34 +49,78 @@ static void doInitFunctionHashTable() { return; } } + + gFunMgtService.pUdfTable = NULL; +} + +static int8_t getUdfType(int32_t funcId) { + SUdfInfo* pUdf = taosArrayGet(gFunMgtService.pUdfTable, funcId - FUNC_UDF_ID_START_OFFSET_VAL - 1); + return pUdf->funcType; } static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) { + if (fmIsUserDefinedFunc(funcId)) { + return getUdfType(funcId); + } if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; } return FUNC_MGT_TEST_MASK(funcMgtBuiltins[funcId].classification, classification); } -int32_t fmFuncMgtInit() { - taosThreadOnce(&functionHashTableInit, doInitFunctionHashTable); - return initFunctionCode; +static int32_t getUdfId(SFmGetFuncInfoParam* pParam, const char* pFuncName) { + SFuncInfo* pInfo = NULL; + int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFuncName, &pInfo); + if (TSDB_CODE_SUCCESS != code || NULL == pInfo) { + return -1; + } + if (NULL == gFunMgtService.pUdfTable) { + gFunMgtService.pUdfTable = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SUdfInfo)); + } + SUdfInfo info = { .outputDt.type = pInfo->outputType, .outputDt.bytes = pInfo->outputLen, .funcType = pInfo->funcType }; + taosArrayPush(gFunMgtService.pUdfTable, &info); + tFreeSFuncInfo(pInfo); + taosMemoryFree(pInfo); + return taosArrayGetSize(gFunMgtService.pUdfTable) + FUNC_UDF_ID_START_OFFSET_VAL; } -int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) { +static int32_t getFuncId(SFmGetFuncInfoParam* pParam, const char* pFuncName) { void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, pFuncName, strlen(pFuncName)); if (NULL == pVal) { - return TSDB_CODE_FAILED; + return getUdfId(pParam, pFuncName); } - *pFuncId = *(int32_t*)pVal; - if (*pFuncId < 0 || *pFuncId >= funcMgtBuiltinsNum) { + return *(int32_t*)pVal; +} + +static int32_t getUdfResultType(SFunctionNode* pFunc) { + SUdfInfo* pUdf = taosArrayGet(gFunMgtService.pUdfTable, pFunc->funcId - FUNC_UDF_ID_START_OFFSET_VAL - 1); + pFunc->node.resType = pUdf->outputDt; + return TSDB_CODE_SUCCESS; +} + +int32_t fmFuncMgtInit() { + taosThreadOnce(&functionHashTableInit, doInitFunctionTable); + return initFunctionCode; +} + +int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) { + *pFuncId = getFuncId(pParam, pFuncName); + if (*pFuncId < 0) { return TSDB_CODE_FAILED; } - *pFuncType = funcMgtBuiltins[*pFuncId].type; + if (fmIsUserDefinedFunc(*pFuncId)) { + *pFuncType = FUNCTION_TYPE_UDF; + } else { + *pFuncType = funcMgtBuiltins[*pFuncId].type; + } return TSDB_CODE_SUCCESS; } int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (fmIsUserDefinedFunc(pFunc->funcId)) { + return getUdfResultType(pFunc); + } + if (pFunc->funcId < 0 || pFunc->funcId >= funcMgtBuiltinsNum) { return TSDB_CODE_FAILED; } @@ -77,7 +128,7 @@ int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { - if (pFunc->funcId < 0 || pFunc->funcId >= funcMgtBuiltinsNum) { + if (fmIsUserDefinedFunc(pFunc->funcId) || pFunc->funcId < 0 || pFunc->funcId >= funcMgtBuiltinsNum) { return FUNC_DATA_REQUIRED_DATA_LOAD; } if (NULL == funcMgtBuiltins[pFunc->funcId].dataRequiredFunc) { @@ -87,7 +138,7 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin } int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) { - if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { + if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) { return TSDB_CODE_FAILED; } pFpSet->getEnv = funcMgtBuiltins[funcId].getEnvFunc; @@ -98,7 +149,7 @@ int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) { } int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) { - if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { + if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) { return TSDB_CODE_FAILED; } pFpSet->process = funcMgtBuiltins[funcId].sprocessFunc; @@ -142,6 +193,10 @@ bool fmIsMultiResFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_MULTI_RES_FUNC); } +bool fmIsUserDefinedFunc(int32_t funcId) { + return funcId > FUNC_UDF_ID_START_OFFSET_VAL; +} + void fmFuncMgtDestroy() { void* m = gFunMgtService.pFuncNameHashTable; if (m != NULL && atomic_val_compare_exchange_ptr((void**)&gFunMgtService.pFuncNameHashTable, m, 0) == m) { diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index e001ef407146ccdf1709be63e11e0a259f9f6775..c26342bfa878fb7fbb092c5e913964a4c96b8615 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -19,14 +19,14 @@ #include "thash.h" #include "ttypes.h" +//#include "tfill.h" #include "function.h" #include "taggfunction.h" -#include "tfill.h" -#include "thistogram.h" -#include "ttszip.h" -#include "tpercentile.h" #include "tbuffer.h" #include "tcompression.h" +#include "thistogram.h" +#include "tpercentile.h" +#include "ttszip.h" //#include "queryLog.h" #include "tdatablock.h" #include "tudf.h" @@ -765,9 +765,9 @@ static int32_t firstFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t c } static int32_t lastFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order != pCtx->param[0].i) { - return BLK_DATA_NOT_LOAD; - } +// if (pCtx->order != pCtx->param[0].param.i) { +// return BLK_DATA_NOT_LOAD; +// } if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) { return BLK_DATA_DATA_LOAD; @@ -797,9 +797,9 @@ static int32_t firstDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32 } static int32_t lastDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order != pCtx->param[0].i) { - return BLK_DATA_NOT_LOAD; - } +// if (pCtx->order != pCtx->param[0].param.i) { +// return BLK_DATA_NOT_LOAD; +// } // not initialized yet, it is the first block, load it. if (pCtx->pOutput == NULL) { @@ -1261,128 +1261,6 @@ int32_t tsCompare(const void* p1, const void* p2) { } } -static void stddev_dst_function(SqlFunctionCtx *pCtx) { - SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - - // the second stage to calculate standard deviation - double *retVal = &pStd->res; - - // all data are null, no need to proceed - SArray* resList = (SArray*) pCtx->param[0].pz; - if (resList == NULL) { - return; - } - - // find the correct group average results according to the tag value - int32_t len = (int32_t) taosArrayGetSize(resList); - assert(len > 0); - - double avg = 0; - if (len == 1) { - SResPair* p = taosArrayGet(resList, 0); - avg = p->avg; - } else { // todo opt performance by using iterator since the timestamp lsit is matched with the output result - SResPair* p = bsearch(&pCtx->startTs, resList->pData, len, sizeof(SResPair), tsCompare); - if (p == NULL) { - return; - } - - avg = p->avg; - } - - void *pData = GET_INPUT_DATA_LIST(pCtx); - int32_t num = 0; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { - continue; - } - num += 1; - *retVal += TPOW2(((int32_t *)pData)[i] - avg); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UINT: { - LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - default: - assert(0); -// qError("stddev function not support data type:%d", pCtx->inputType); - } - - pStd->num += num; - SET_VAL(pCtx, num, 1); - - // copy to the final output buffer for super table - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)), sizeof(SAvgInfo)); -} - -static void stddev_dst_merge(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevdstInfo* pRes = GET_ROWCELL_INTERBUF(pResInfo); - - char *input = GET_INPUT_DATA_LIST(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SStddevdstInfo *pInput = (SStddevdstInfo *)input; - if (pInput->num == 0) { // current input is null - continue; - } - - pRes->num += pInput->num; - pRes->res += pInput->res; - } -} - -static void stddev_dst_finalizer(SqlFunctionCtx *pCtx) { - SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - - if (pStd->num <= 0) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } else { - double *retValue = (double *)pCtx->pOutput; - SET_DOUBLE_VAL(retValue, sqrt(pStd->res / pStd->num)); - SET_VAL(pCtx, 1, 1); - } - - doFinalizer(pCtx); -} - ////////////////////////////////////////////////////////////////////////////////////// static bool first_last_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { @@ -1390,8 +1268,8 @@ static bool first_last_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* } // used to keep the timestamp for comparison - pCtx->param[1].nType = 0; - pCtx->param[1].i = 0; +// pCtx->param[1].param.nType = 0; +// pCtx->param[1].param.i = 0; return true; } @@ -1487,13 +1365,13 @@ static void first_dist_func_merge(SqlFunctionCtx *pCtx) { } // The param[1] is used to keep the initial value of max ts value - if (pCtx->param[1].nType != pCtx->resDataInfo.type || pCtx->param[1].i > pInput->ts) { - memcpy(pCtx->pOutput, pData, pCtx->resDataInfo.bytes); - pCtx->param[1].i = pInput->ts; - pCtx->param[1].nType = pCtx->resDataInfo.type; - -// DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } +// if (pCtx->param[1].param.nType != pCtx->resDataInfo.type || pCtx->param[1].param.i > pInput->ts) { +// memcpy(pCtx->pOutput, pData, pCtx->resDataInfo.bytes); +// pCtx->param[1].param.i = pInput->ts; +// pCtx->param[1].param.nType = pCtx->resDataInfo.type; +// +//// DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); +// } SET_VAL(pCtx, 1, 1); // GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; @@ -1508,9 +1386,9 @@ static void first_dist_func_merge(SqlFunctionCtx *pCtx) { * least one data in this block that is not null.(TODO opt for this case) */ static void last_function(SqlFunctionCtx *pCtx) { - if (pCtx->order != pCtx->param[0].i) { - return; - } +// if (pCtx->order != pCtx->param[0].param.i) { +// return; +// } SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -1582,9 +1460,9 @@ static void last_dist_function(SqlFunctionCtx *pCtx) { * 1. for scan data is not the required order * 2. for data blocks that are not loaded, no need to check data */ - if (pCtx->order != pCtx->param[0].i) { - return; - } +// if (pCtx->order != pCtx->param[0].param.i) { +// return; +// } int32_t notNullElems = 0; for (int32_t i = pCtx->size - 1; i >= 0; --i) { @@ -1624,10 +1502,10 @@ static void last_dist_func_merge(SqlFunctionCtx *pCtx) { * param[1] used to keep the corresponding timestamp to decide if current result is * the true last result */ - if (pCtx->param[1].nType != pCtx->resDataInfo.type || pCtx->param[1].i < pInput->ts) { + if (pCtx->param[1].param.nType != pCtx->resDataInfo.type || pCtx->param[1].param.i < pInput->ts) { memcpy(pCtx->pOutput, pData, pCtx->resDataInfo.bytes); - pCtx->param[1].i = pInput->ts; - pCtx->param[1].nType = pCtx->resDataInfo.type; + pCtx->param[1].param.i = pInput->ts; + pCtx->param[1].param.nType = pCtx->resDataInfo.type; // DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } @@ -1955,11 +1833,11 @@ static STopBotInfo *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SqlFunctionCtx *pCtx) { char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); pTopBotInfo->res = (tValuePair**) tmp; - tmp += POINTER_BYTES * pCtx->param[0].i; +// tmp += POINTER_BYTES * pCtx->param[0].param.i; // size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; -// for (int32_t i = 0; i < pCtx->param[0].i; ++i) { +// for (int32_t i = 0; i < pCtx->param[0].param.i; ++i) { // pTopBotInfo->res[i] = (tValuePair*) tmp; // pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); // tmp += size; @@ -1975,13 +1853,13 @@ bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const cha STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx); // required number of results are not reached, continue load data block - if (pTopBotInfo->num < pCtx->param[0].i) { - return true; - } +// if (pTopBotInfo->num < pCtx->param[0].param.i) { +// return true; +// } - if ((void *)pTopBotInfo->res[0] != (void *)((char *)pTopBotInfo + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i)) { - buildTopBotStruct(pTopBotInfo, pCtx); - } +// if ((void *)pTopBotInfo->res[0] != (void *)((char *)pTopBotInfo + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { +// buildTopBotStruct(pTopBotInfo, pCtx); +// } tValuePair **pRes = (tValuePair**) pTopBotInfo->res; @@ -2038,9 +1916,9 @@ static void top_function(SqlFunctionCtx *pCtx) { STopBotInfo *pRes = getTopBotOutputInfo(pCtx); assert(pRes->num >= 0); - if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i)) { - buildTopBotStruct(pRes, pCtx); - } +// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { +// buildTopBotStruct(pRes, pCtx); +// } for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_DATA(pCtx, i); @@ -2052,7 +1930,7 @@ static void top_function(SqlFunctionCtx *pCtx) { // NOTE: Set the default timestamp if it is missing [todo refactor] TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; -// do_top_function_add(pRes, (int32_t)pCtx->param[0].i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); +// do_top_function_add(pRes, (int32_t)pCtx->param[0].param.i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2079,7 +1957,7 @@ static void top_func_merge(SqlFunctionCtx *pCtx) { // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->resDataInfo.type; -// do_top_function_add(pOutput, (int32_t)pCtx->param[0].i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, +// do_top_function_add(pOutput, (int32_t)pCtx->param[0].param.i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, // type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } @@ -2096,9 +1974,9 @@ static void bottom_function(SqlFunctionCtx *pCtx) { STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i)) { - buildTopBotStruct(pRes, pCtx); - } +// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { +// buildTopBotStruct(pRes, pCtx); +// } for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_DATA(pCtx, i); @@ -2109,7 +1987,7 @@ static void bottom_function(SqlFunctionCtx *pCtx) { notNullElems++; // NOTE: Set the default timestamp if it is missing [todo refactor] TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; -// do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); +// do_bottom_function_add(pRes, (int32_t)pCtx->param[0].param.i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2136,7 +2014,7 @@ static void bottom_func_merge(SqlFunctionCtx *pCtx) { // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT) ? TSDB_DATA_TYPE_DOUBLE : pCtx->resDataInfo.type; -// do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, type, +// do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].param.i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, type, // &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } @@ -2162,11 +2040,11 @@ static void top_bottom_func_finalizer(SqlFunctionCtx *pCtx) { tValuePair **tvp = pRes->res; // user specify the order of output by sort the result according to timestamp - if (pCtx->param[1].i == PRIMARYKEY_TIMESTAMP_COL_ID) { - __compar_fn_t comparator = (pCtx->param[2].i == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; + if (pCtx->param[1].param.i == PRIMARYKEY_TIMESTAMP_COL_ID) { + __compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); - } else /*if (pCtx->param[1].i > PRIMARYKEY_TIMESTAMP_COL_ID)*/ { - __compar_fn_t comparator = (pCtx->param[2].i == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; + } else /*if (pCtx->param[1].param.i > PRIMARYKEY_TIMESTAMP_COL_ID)*/ { + __compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } @@ -2277,7 +2155,7 @@ static void percentile_function(SqlFunctionCtx *pCtx) { } static void percentile_finalizer(SqlFunctionCtx *pCtx) { - double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i : pCtx->param[0].d; +// double v = pCtx->param[0].param.nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].param.i : pCtx->param[0].param.d; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); @@ -2287,7 +2165,7 @@ static void percentile_finalizer(SqlFunctionCtx *pCtx) { assert(ppInfo->numOfElems == 0); setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); } else { - SET_DOUBLE_VAL((double *)pCtx->pOutput, getPercentile(pMemBucket, v)); +// SET_DOUBLE_VAL((double *)pCtx->pOutput, getPercentile(pMemBucket, v)); } tMemBucketDestroy(pMemBucket); @@ -2389,7 +2267,7 @@ static void apercentile_func_merge(SqlFunctionCtx *pCtx) { } static void apercentile_finalizer(SqlFunctionCtx *pCtx) { - double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i : pCtx->param[0].d; + double v = (pCtx->param[0].param.nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].param.i : pCtx->param[0].param.d; SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); @@ -2432,7 +2310,7 @@ static bool leastsquares_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInf SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); // 2*3 matrix - pInfo->startVal = pCtx->param[0].d; +// pInfo->startVal = pCtx->param[0].param.d; return true; } @@ -2478,54 +2356,54 @@ static void leastsquares_function(SqlFunctionCtx *pCtx) { param[0][2] += x * p[i]; param[1][2] += p[i]; - x += pCtx->param[1].d; + x += pCtx->param[1].param.d; numOfElem++; } break; } case TSDB_DATA_TYPE_BIGINT: { int64_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } case TSDB_DATA_TYPE_DOUBLE: { double *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } case TSDB_DATA_TYPE_FLOAT: { float *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; }; case TSDB_DATA_TYPE_SMALLINT: { int16_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } case TSDB_DATA_TYPE_TINYINT: { int8_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } case TSDB_DATA_TYPE_UTINYINT: { uint8_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } case TSDB_DATA_TYPE_USMALLINT: { uint16_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } case TSDB_DATA_TYPE_UINT: { uint32_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } case TSDB_DATA_TYPE_UBIGINT: { uint64_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } } @@ -2584,16 +2462,16 @@ static void col_project_function(SqlFunctionCtx *pCtx) { } // only one row is required. - if (pCtx->param[0].i == 1) { - SET_VAL(pCtx, pCtx->size, 1); - } else { - INC_INIT_VAL(pCtx, pCtx->size); - } +// if (pCtx->param[0].param.i == 1) { +// SET_VAL(pCtx, pCtx->size, 1); +// } else { +// INC_INIT_VAL(pCtx, pCtx->size); +// } char *pData = GET_INPUT_DATA_LIST(pCtx); if (pCtx->order == TSDB_ORDER_ASC) { - int32_t numOfRows = (pCtx->param[0].i == 1)? 1:pCtx->size; - memcpy(pCtx->pOutput, pData, (size_t) numOfRows * pCtx->inputBytes); +// int32_t numOfRows = (pCtx->param[0].param.i == 1)? 1:pCtx->size; +// memcpy(pCtx->pOutput, pData, (size_t) numOfRows * pCtx->inputBytes); } else { for(int32_t i = 0; i < pCtx->size; ++i) { memcpy(pCtx->pOutput + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, @@ -2658,7 +2536,7 @@ static bool diff_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResI } // diff function require the value is set to -1 - pCtx->param[1].nType = INITIAL_VALUE_NOT_ASSIGNED; + pCtx->param[1].param.nType = INITIAL_VALUE_NOT_ASSIGNED; return false; } @@ -2670,9 +2548,9 @@ static bool deriv_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pRes // diff function require the value is set to -1 SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResultInfo); - pDerivInfo->ignoreNegative = pCtx->param[1].i; +// pDerivInfo->ignoreNegative = pCtx->param[1].param.i; pDerivInfo->prevTs = -1; - pDerivInfo->tsWindow = pCtx->param[0].i; +// pDerivInfo->tsWindow = pCtx->param[0].param.i; pDerivInfo->valueSet = false; return false; } @@ -2861,12 +2739,12 @@ static void deriv_function(SqlFunctionCtx *pCtx) { #define DIFF_IMPL(ctx, d, type) \ do { \ - if ((ctx)->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { \ - (ctx)->param[1].nType = (ctx)->inputType; \ - *(type *)&(ctx)->param[1].i = *(type *)(d); \ + if ((ctx)->param[1].param.nType == INITIAL_VALUE_NOT_ASSIGNED) { \ + (ctx)->param[1].param.nType = (ctx)->inputType; \ + *(type *)&(ctx)->param[1].param.i = *(type *)(d); \ } else { \ - *(type *)(ctx)->pOutput = *(type *)(d) - (*(type *)(&(ctx)->param[1].i)); \ - *(type *)(&(ctx)->param[1].i) = *(type *)(d); \ + *(type *)(ctx)->pOutput = *(type *)(d) - (*(type *)(&(ctx)->param[1].param.i)); \ + *(type *)(&(ctx)->param[1].param.i) = *(type *)(d); \ *(int64_t *)(ctx)->pTsOutput = GET_TS_DATA(ctx, index); \ } \ } while (0); @@ -2874,7 +2752,7 @@ static void deriv_function(SqlFunctionCtx *pCtx) { // TODO difference in date column static void diff_function(SqlFunctionCtx *pCtx) { void *data = GET_INPUT_DATA_LIST(pCtx); - bool isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED); + bool isFirstBlock = (pCtx->param[1].param.nType == INITIAL_VALUE_NOT_ASSIGNED); int32_t notNullElems = 0; @@ -2894,15 +2772,15 @@ static void diff_function(SqlFunctionCtx *pCtx) { continue; } - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int32_t)(pData[i] - pCtx->param[1].i); // direct previous may be null + if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + *pOutput = (int32_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].i = pData[i]; - pCtx->param[1].nType = pCtx->inputType; + pCtx->param[1].param.i = pData[i]; + pCtx->param[1].param.nType = pCtx->inputType; notNullElems++; } break; @@ -2916,15 +2794,15 @@ static void diff_function(SqlFunctionCtx *pCtx) { continue; } - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = pData[i] - pCtx->param[1].i; // direct previous may be null + if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + *pOutput = pData[i] - pCtx->param[1].param.i; // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].i = pData[i]; - pCtx->param[1].nType = pCtx->inputType; + pCtx->param[1].param.i = pData[i]; + pCtx->param[1].param.nType = pCtx->inputType; notNullElems++; } break; @@ -2938,15 +2816,15 @@ static void diff_function(SqlFunctionCtx *pCtx) { continue; } - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - SET_DOUBLE_VAL(pOutput, pData[i] - pCtx->param[1].d); // direct previous may be null + if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + SET_DOUBLE_VAL(pOutput, pData[i] - pCtx->param[1].param.d); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].d = pData[i]; - pCtx->param[1].nType = pCtx->inputType; + pCtx->param[1].param.d = pData[i]; + pCtx->param[1].param.nType = pCtx->inputType; notNullElems++; } break; @@ -2960,15 +2838,15 @@ static void diff_function(SqlFunctionCtx *pCtx) { continue; } - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (float)(pData[i] - pCtx->param[1].d); // direct previous may be null + if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + *pOutput = (float)(pData[i] - pCtx->param[1].param.d); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].d = pData[i]; - pCtx->param[1].nType = pCtx->inputType; + pCtx->param[1].param.d = pData[i]; + pCtx->param[1].param.nType = pCtx->inputType; notNullElems++; } break; @@ -2982,15 +2860,15 @@ static void diff_function(SqlFunctionCtx *pCtx) { continue; } - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int16_t)(pData[i] - pCtx->param[1].i); // direct previous may be null + if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + *pOutput = (int16_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].i = pData[i]; - pCtx->param[1].nType = pCtx->inputType; + pCtx->param[1].param.i = pData[i]; + pCtx->param[1].param.nType = pCtx->inputType; notNullElems++; } break; @@ -3005,15 +2883,15 @@ static void diff_function(SqlFunctionCtx *pCtx) { continue; } - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int8_t)(pData[i] - pCtx->param[1].i); // direct previous may be null + if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + *pOutput = (int8_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].i = pData[i]; - pCtx->param[1].nType = pCtx->inputType; + pCtx->param[1].param.i = pData[i]; + pCtx->param[1].param.nType = pCtx->inputType; notNullElems++; } break; @@ -3024,7 +2902,7 @@ static void diff_function(SqlFunctionCtx *pCtx) { } // initial value is not set yet - if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED || notNullElems <= 0) { + if (pCtx->param[1].param.nType == INITIAL_VALUE_NOT_ASSIGNED || notNullElems <= 0) { /* * 1. current block and blocks before are full of null * 2. current block may be null value @@ -3091,8 +2969,8 @@ static bool spread_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pRe // this is the server-side setup function in client-side, the secondary merge do not need this procedure if (pCtx->currentStage == MERGE_STAGE) { - pCtx->param[0].d = DBL_MAX; - pCtx->param[3].d = -DBL_MAX; +// pCtx->param[0].param.d = DBL_MAX; +// pCtx->param[3].param.d = -DBL_MAX; } else { pInfo->min = DBL_MAX; pInfo->max = -DBL_MAX; @@ -3192,13 +3070,13 @@ void spread_func_merge(SqlFunctionCtx *pCtx) { return; } - if (pCtx->param[0].d > pData->min) { - pCtx->param[0].d = pData->min; - } +// if (pCtx->param[0].param.d > pData->min) { +// pCtx->param[0].param.d = pData->min; +// } - if (pCtx->param[3].d < pData->max) { - pCtx->param[3].d = pData->max; - } +// if (pCtx->param[3].param.d < pData->max) { +// pCtx->param[3].param.d = pData->max; +// } // GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } @@ -3218,7 +3096,7 @@ void spread_function_finalizer(SqlFunctionCtx *pCtx) { // return; // } - SET_DOUBLE_VAL((double *)pCtx->pOutput, pCtx->param[3].d - pCtx->param[0].d); +// SET_DOUBLE_VAL((double *)pCtx->pOutput, pCtx->param[3].param.d - pCtx->param[0].param.d); } else { assert(IS_NUMERIC_TYPE(pCtx->inputType) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); @@ -3571,7 +3449,7 @@ void twa_function_finalizer(SqlFunctionCtx *pCtx) { */ static void interp_function_impl(SqlFunctionCtx *pCtx) { - int32_t type = (int32_t) pCtx->param[2].i; + int32_t type = (int32_t) pCtx->param[2].param.i; if (type == TSDB_FILL_NONE) { return; } @@ -3583,7 +3461,7 @@ static void interp_function_impl(SqlFunctionCtx *pCtx) { } else if (type == TSDB_FILL_NULL) { setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); } else if (type == TSDB_FILL_SET_VALUE) { - taosVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true); +// taosVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true); } else { if (pCtx->start.key != INT64_MIN && ((ascQuery && pCtx->start.key <= pCtx->startTs && pCtx->end.key >= pCtx->startTs) || ((!ascQuery) && pCtx->start.key >= pCtx->startTs && pCtx->end.key <= pCtx->startTs))) { if (type == TSDB_FILL_PREV) { @@ -3608,7 +3486,7 @@ static void interp_function_impl(SqlFunctionCtx *pCtx) { if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) { setNull(pCtx->pOutput, srcType, pCtx->inputBytes); } else { - taosGetLinearInterpolationVal(&point, pCtx->resDataInfo.type, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); +// taosGetLinearInterpolationVal(&point, pCtx->resDataInfo.type, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); } } else { setNull(pCtx->pOutput, srcType, pCtx->inputBytes); @@ -3681,7 +3559,7 @@ static void interp_function_impl(SqlFunctionCtx *pCtx) { if (isNull(start, srcType) || isNull(end, srcType)) { setNull(pCtx->pOutput, srcType, pCtx->inputBytes); } else { - taosGetLinearInterpolationVal(&point, pCtx->resDataInfo.type, &point1, &point2, srcType); +// taosGetLinearInterpolationVal(&point, pCtx->resDataInfo.type, &point1, &point2, srcType); } } else { setNull(pCtx->pOutput, srcType, pCtx->inputBytes); @@ -3755,11 +3633,11 @@ static void ts_comp_function(SqlFunctionCtx *pCtx) { // primary ts must be existed, so no need to check its existance if (pCtx->order == TSDB_ORDER_ASC) { - tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); +// tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].param.i, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *d = GET_INPUT_DATA(pCtx, i); - tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); +// tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].param.i, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); } } @@ -3911,7 +3789,7 @@ static void rate_finalizer(SqlFunctionCtx *pCtx) { return; } - SET_DOUBLE_VAL((double*) pCtx->pOutput, do_calc_rate(pRateInfo, (double) TSDB_TICK_PER_SECOND(pCtx->param[0].i))); +// SET_DOUBLE_VAL((double*) pCtx->pOutput, do_calc_rate(pRateInfo, (double) TSDB_TICK_PER_SECOND(pCtx->param[0].param.i))); // cannot set the numOfIteratedElems again since it is set during previous iteration pResInfo->numOfRes = 1; @@ -4008,7 +3886,7 @@ static void blockInfo_func(SqlFunctionCtx* pCtx) { int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); - pDist->rowSize = (uint16_t)pCtx->param[0].i; +// pDist->rowSize = (uint16_t)pCtx->param[0].param.i; memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len); @@ -4034,8 +3912,8 @@ static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlock pDist->maxRows = pSrc->maxRows; pDist->minRows = pSrc->minRows; - int32_t maxSteps = TSDB_MAX_MAX_ROW_FBLOCK/TSDB_BLOCK_DIST_STEP_ROWS; - if (TSDB_MAX_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { + int32_t maxSteps = TSDB_MAX_MAXROWS_FBLOCK/TSDB_BLOCK_DIST_STEP_ROWS; + if (TSDB_MAX_MAXROWS_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { ++maxSteps; } pDist->dataBlockInfos = taosArrayInit(maxSteps, sizeof(SFileBlockInfo)); @@ -4160,7 +4038,7 @@ void blockinfo_func_finalizer(SqlFunctionCtx* pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); - pDist->rowSize = (uint16_t)pCtx->param[0].i; +// pDist->rowSize = (uint16_t)pCtx->param[0].param.i; generateBlockDistResult(pDist, pCtx->pOutput); if (pDist->dataBlockInfos != NULL) { @@ -4557,9 +4435,9 @@ SAggFunctionInfo aggFunc[35] = {{ FUNCTION_AVG, FUNCSTATE_SO | FUNCSTATE_STABLE, function_setup, - stddev_dst_function, - stddev_dst_finalizer, - stddev_dst_merge, + NULL, + NULL, + NULL, dataBlockRequired, }, { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index a1030f6c21d5e42552515fd0e6af372e95d1a3f1..317339af04e6f93e16d29b43c1d18bfbaf0db1de 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -14,16 +14,107 @@ */ #include "uv.h" #include "os.h" -#include "tlog.h" +#include "fnLog.h" #include "tudf.h" #include "tudfInt.h" +#include "tarray.h" +#include "tdatablock.h" -//TODO: when startup, set thread poll size. add it to cfg -//TODO: udfd restart when exist or aborts //TODO: network error processing. //TODO: add unit test -//TODO: add lua support -void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf); +//TODO: include all global variable under context struct + +/* Copyright (c) 2013, Ben Noordhuis + * The QUEUE is copied from queue.h under libuv + * */ + +typedef void *QUEUE[2]; + +/* Private macros. */ +#define QUEUE_NEXT(q) (*(QUEUE **) &((*(q))[0])) +#define QUEUE_PREV(q) (*(QUEUE **) &((*(q))[1])) +#define QUEUE_PREV_NEXT(q) (QUEUE_NEXT(QUEUE_PREV(q))) +#define QUEUE_NEXT_PREV(q) (QUEUE_PREV(QUEUE_NEXT(q))) + +/* Public macros. */ +#define QUEUE_DATA(ptr, type, field) \ + ((type *) ((char *) (ptr) - offsetof(type, field))) + +/* Important note: mutating the list while QUEUE_FOREACH is + * iterating over its elements results in undefined behavior. + */ +#define QUEUE_FOREACH(q, h) \ + for ((q) = QUEUE_NEXT(h); (q) != (h); (q) = QUEUE_NEXT(q)) + +#define QUEUE_EMPTY(q) \ + ((const QUEUE *) (q) == (const QUEUE *) QUEUE_NEXT(q)) + +#define QUEUE_HEAD(q) \ + (QUEUE_NEXT(q)) + +#define QUEUE_INIT(q) \ + do { \ + QUEUE_NEXT(q) = (q); \ + QUEUE_PREV(q) = (q); \ + } \ + while (0) + +#define QUEUE_ADD(h, n) \ + do { \ + QUEUE_PREV_NEXT(h) = QUEUE_NEXT(n); \ + QUEUE_NEXT_PREV(n) = QUEUE_PREV(h); \ + QUEUE_PREV(h) = QUEUE_PREV(n); \ + QUEUE_PREV_NEXT(h) = (h); \ + } \ + while (0) + +#define QUEUE_SPLIT(h, q, n) \ + do { \ + QUEUE_PREV(n) = QUEUE_PREV(h); \ + QUEUE_PREV_NEXT(n) = (n); \ + QUEUE_NEXT(n) = (q); \ + QUEUE_PREV(h) = QUEUE_PREV(q); \ + QUEUE_PREV_NEXT(h) = (h); \ + QUEUE_PREV(q) = (n); \ + } \ + while (0) + +#define QUEUE_MOVE(h, n) \ + do { \ + if (QUEUE_EMPTY(h)) \ + QUEUE_INIT(n); \ + else { \ + QUEUE* q = QUEUE_HEAD(h); \ + QUEUE_SPLIT(h, q, n); \ + } \ + } \ + while (0) + +#define QUEUE_INSERT_HEAD(h, q) \ + do { \ + QUEUE_NEXT(q) = QUEUE_NEXT(h); \ + QUEUE_PREV(q) = (h); \ + QUEUE_NEXT_PREV(q) = (q); \ + QUEUE_NEXT(h) = (q); \ + } \ + while (0) + +#define QUEUE_INSERT_TAIL(h, q) \ + do { \ + QUEUE_NEXT(q) = (h); \ + QUEUE_PREV(q) = QUEUE_PREV(h); \ + QUEUE_PREV_NEXT(q) = (q); \ + QUEUE_PREV(h) = (q); \ + } \ + while (0) + +#define QUEUE_REMOVE(q) \ + do { \ + QUEUE_PREV_NEXT(q) = QUEUE_NEXT(q); \ + QUEUE_NEXT_PREV(q) = QUEUE_PREV(q); \ + } \ + while (0) + enum { UV_TASK_CONNECT = 0, @@ -31,12 +122,35 @@ enum { UV_TASK_DISCONNECT = 2 }; +int64_t gUdfTaskSeqNum = 0; +typedef struct SUdfdProxy { + int32_t dnodeId; + uv_barrier_t gUdfInitBarrier; + + uv_loop_t gUdfdLoop; + uv_thread_t gUdfLoopThread; + uv_async_t gUdfLoopTaskAync; + + uv_async_t gUdfLoopStopAsync; + + uv_mutex_t gUdfTaskQueueMutex; + int8_t gUdfcState; + QUEUE gUdfTaskQueue; + QUEUE gUvProcTaskQueue; + // int8_t gUdfcState = UDFC_STATE_INITAL; + // QUEUE gUdfTaskQueue = {0}; + // QUEUE gUvProcTaskQueue = {0}; +} SUdfdProxy; + + typedef struct SUdfUvSession { + SUdfdProxy *udfc; int64_t severHandle; uv_pipe_t *udfSvcPipe; } SUdfUvSession; typedef struct SClientUvTaskNode { + SUdfdProxy *udfc; int8_t type; int errCode; @@ -48,8 +162,9 @@ typedef struct SClientUvTaskNode { uv_sem_t taskSem; uv_buf_t rspBuf; - struct SClientUvTaskNode *prev; - struct SClientUvTaskNode *next; + QUEUE recvTaskQueue; + QUEUE procTaskQueue; + QUEUE connTaskQueue; } SClientUvTaskNode; typedef struct SClientUdfTask { @@ -74,7 +189,6 @@ typedef struct SClientUdfTask { } _teardown; }; - } SClientUdfTask; typedef struct SClientConnBuf { @@ -86,393 +200,401 @@ typedef struct SClientConnBuf { typedef struct SClientUvConn { uv_pipe_t *pipe; - SClientUvTaskNode taskQueue; + QUEUE taskQueue; SClientConnBuf readBuf; } SClientUvConn; -uv_process_t gUdfdProcess; - -uv_barrier_t gUdfInitBarrier; - -uv_loop_t gUdfdLoop; -uv_thread_t gUdfLoopThread; -uv_async_t gUdfLoopTaskAync; - -uv_async_t gUdfLoopStopAsync; - -uv_mutex_t gUdfTaskQueueMutex; -int64_t gUdfTaskSeqNum = 0; - -//double circular linked list -typedef SClientUvTaskNode *SClientUvTaskQueue; -SClientUvTaskNode gUdfQueueNode; -SClientUvTaskQueue gUdfTaskQueue = &gUdfQueueNode; - -//add SClientUvTaskNode task that close conn - - +enum { + UDFC_STATE_INITAL = 0, // initial state + UDFC_STATE_STARTNG, // starting after udfcOpen + UDFC_STATE_READY, // started and begin to receive quests + UDFC_STATE_STOPPING, // stopping after udfcClose + UDFC_STATUS_FINAL, // stopped +}; -void udfTaskQueueInit(SClientUvTaskQueue q) { - q->next = q; - q->prev = q; +int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup) { + int32_t len = 0; + len += taosEncodeBinary(buf, setup->udfName, TSDB_FUNC_NAME_LEN); + len += taosEncodeSEpSet(buf, &setup->epSet); + return len; } -bool udfTaskQueueIsEmpty(SClientUvTaskQueue q) { - return q == q->next; +void* decodeUdfSetupRequest(const void* buf, SUdfSetupRequest *request) { + buf = taosDecodeBinaryTo(buf, request->udfName, TSDB_FUNC_NAME_LEN); + buf = taosDecodeSEpSet((void*)buf, &request->epSet); + return (void*)buf; } -void udfTaskQueueInsertTail(SClientUvTaskQueue q, SClientUvTaskNode *e) { - e->next = q; - e->prev = q->prev; - e->prev->next = e; - q->prev = e; +int32_t encodeUdfInterBuf(void **buf, const SUdfInterBuf* state) { + int32_t len = 0; + len += taosEncodeFixedI32(buf, state->bufLen); + len += taosEncodeBinary(buf, state->buf, state->bufLen); + return len; } -void udfTaskQueueInsertTaskAtHead(SClientUvTaskQueue q, SClientUvTaskNode *e) { - e->next = q->next; - e->prev = q; - q->next->prev = e; - q->next = e; +void* decodeUdfInterBuf(const void* buf, SUdfInterBuf* state) { + buf = taosDecodeFixedI32(buf, &state->bufLen); + buf = taosDecodeBinary(buf, (void**)&state->buf, state->bufLen); + return (void*)buf; } -void udfTaskQueueRemoveTask(SClientUvTaskNode *e) { - e->prev->next = e->next; - e->next->prev = e->prev; +int32_t encodeUdfCallRequest(void **buf, const SUdfCallRequest *call) { + int32_t len = 0; + len += taosEncodeFixedI64(buf, call->udfHandle); + len += taosEncodeFixedI8(buf, call->callType); + if (call->callType == TSDB_UDF_CALL_SCALA_PROC) { + len += tEncodeDataBlock(buf, &call->block); + } else if (call->callType == TSDB_UDF_CALL_AGG_INIT) { + len += taosEncodeFixedI8(buf, call->initFirst); + } else if (call->callType == TSDB_UDF_CALL_AGG_PROC) { + len += tEncodeDataBlock(buf, &call->block); + len += encodeUdfInterBuf(buf, &call->interBuf); + } else if (call->callType == TSDB_UDF_CALL_AGG_MERGE) { + len += encodeUdfInterBuf(buf, &call->interBuf); + len += encodeUdfInterBuf(buf, &call->interBuf2); + } else if (call->callType == TSDB_UDF_CALL_AGG_FIN) { + len += encodeUdfInterBuf(buf, &call->interBuf); + } + return len; } -void udfTaskQueueSplit(SClientUvTaskQueue q, SClientUvTaskNode *from, SClientUvTaskQueue n) { - n->prev = q->prev; - n->prev->next = n; - n->next = from; - q->prev = from->prev; - q->prev->next = q; - from->prev = n; +void* decodeUdfCallRequest(const void* buf, SUdfCallRequest* call) { + buf = taosDecodeFixedI64(buf, &call->udfHandle); + buf = taosDecodeFixedI8(buf, &call->callType); + switch (call->callType) { + case TSDB_UDF_CALL_SCALA_PROC: + buf = tDecodeDataBlock(buf, &call->block); + break; + case TSDB_UDF_CALL_AGG_INIT: + buf = taosDecodeFixedI8(buf, &call->initFirst); + break; + case TSDB_UDF_CALL_AGG_PROC: + buf = tDecodeDataBlock(buf, &call->block); + buf = decodeUdfInterBuf(buf, &call->interBuf); + break; + case TSDB_UDF_CALL_AGG_MERGE: + buf = decodeUdfInterBuf(buf, &call->interBuf); + buf = decodeUdfInterBuf(buf, &call->interBuf2); + break; + case TSDB_UDF_CALL_AGG_FIN: + buf = decodeUdfInterBuf(buf, &call->interBuf); + break; + } + return (void*)buf; } -SClientUvTaskNode *udfTaskQueueHeadTask(SClientUvTaskQueue q) { - return q->next; +int32_t encodeUdfTeardownRequest(void **buf, const SUdfTeardownRequest *teardown) { + int32_t len = 0; + len += taosEncodeFixedI64(buf, teardown->udfHandle); + return len; } -SClientUvTaskNode *udfTaskQueueTailTask(SClientUvTaskQueue q) { - return q->prev; +void* decodeUdfTeardownRequest(const void* buf, SUdfTeardownRequest *teardown) { + buf = taosDecodeFixedI64(buf, &teardown->udfHandle); + return (void*)buf; } -SClientUvTaskNode *udfTaskQueueNext(SClientUvTaskNode *e) { - return e->next; +int32_t encodeUdfRequest(void** buf, const SUdfRequest* request) { + int32_t len = 0; + if (buf == NULL) { + len += sizeof(request->msgLen); + } else { + *(int32_t*)(*buf) = request->msgLen; + *buf = POINTER_SHIFT(*buf, sizeof(request->msgLen)); + } + len += taosEncodeFixedI64(buf, request->seqNum); + len += taosEncodeFixedI8(buf, request->type); + if (request->type == UDF_TASK_SETUP) { + len += encodeUdfSetupRequest(buf, &request->setup); + } else if (request->type == UDF_TASK_CALL) { + len += encodeUdfCallRequest(buf, &request->call); + } else if (request->type == UDF_TASK_TEARDOWN) { + len += encodeUdfTeardownRequest(buf, &request->teardown); + } + return len; } -void udfTaskQueueMove(SClientUvTaskQueue q, SClientUvTaskQueue n) { - if (udfTaskQueueIsEmpty(q)) { - udfTaskQueueInit(n); - } else { - SClientUvTaskNode *h = udfTaskQueueHeadTask(q); - udfTaskQueueSplit(q, h, n); +void* decodeUdfRequest(const void* buf, SUdfRequest* request) { + request->msgLen = *(int32_t*)(buf); + buf = POINTER_SHIFT(buf, sizeof(request->msgLen)); + + buf = taosDecodeFixedI64(buf, &request->seqNum); + buf = taosDecodeFixedI8(buf, &request->type); + + if (request->type == UDF_TASK_SETUP) { + buf = decodeUdfSetupRequest(buf, &request->setup); + } else if (request->type == UDF_TASK_CALL) { + buf = decodeUdfCallRequest(buf, &request->call); + } else if (request->type == UDF_TASK_TEARDOWN) { + buf = decodeUdfTeardownRequest(buf, &request->teardown); } + return (void*)buf; } +int32_t encodeUdfSetupResponse(void **buf, const SUdfSetupResponse *setupRsp) { + int32_t len = 0; + len += taosEncodeFixedI64(buf, setupRsp->udfHandle); + return len; +} -int32_t encodeRequest(char **pBuf, int32_t *pBufLen, SUdfRequest *request) { - debugPrint("%s", "encoding request"); +void* decodeUdfSetupResponse(const void* buf, SUdfSetupResponse* setupRsp) { + buf = taosDecodeFixedI64(buf, &setupRsp->udfHandle); + return (void*)buf; +} - int len = sizeof(SUdfRequest) - sizeof(void *); - switch (request->type) { - case UDF_TASK_SETUP: { - SUdfSetupRequest *setup = (SUdfSetupRequest *) (request->subReq); - len += sizeof(SUdfSetupRequest) - 1 * sizeof(char *) + setup->pathSize; +int32_t encodeUdfCallResponse(void **buf, const SUdfCallResponse *callRsp) { + int32_t len = 0; + len += taosEncodeFixedI8(buf, callRsp->callType); + switch (callRsp->callType) { + case TSDB_UDF_CALL_SCALA_PROC: + len += tEncodeDataBlock(buf, &callRsp->resultData); break; - } - case UDF_TASK_CALL: { - SUdfCallRequest *call = (SUdfCallRequest *) (request->subReq); - len += sizeof(SUdfCallRequest) - 2 * sizeof(char *) + call->inputBytes + call->stateBytes; + case TSDB_UDF_CALL_AGG_INIT: + len += encodeUdfInterBuf(buf, &callRsp->resultBuf); break; - } - case UDF_TASK_TEARDOWN: { - SUdfTeardownRequest *teardown = (SUdfTeardownRequest *) (request->subReq); - len += sizeof(SUdfTeardownRequest); + case TSDB_UDF_CALL_AGG_PROC: + len += encodeUdfInterBuf(buf, &callRsp->resultBuf); break; - } - default: + case TSDB_UDF_CALL_AGG_MERGE: + len += encodeUdfInterBuf(buf, &callRsp->resultBuf); + break; + case TSDB_UDF_CALL_AGG_FIN: + len += encodeUdfInterBuf(buf, &callRsp->resultBuf); break; } + return len; +} - char *bufBegin = taosMemoryMalloc(len); - char *buf = bufBegin; - - //skip msgLen first - buf += sizeof(int32_t); - - *(int64_t *) buf = request->seqNum; - buf += sizeof(int64_t); - *(int8_t *) buf = request->type; - buf += sizeof(int8_t); - - switch (request->type) { - case UDF_TASK_SETUP: { - SUdfSetupRequest *setup = (SUdfSetupRequest *) (request->subReq); - memcpy(buf, setup->udfName, 16); - buf += 16; - *(int8_t *) buf = setup->scriptType; - buf += sizeof(int8_t); - *(int8_t *) buf = setup->udfType; - buf += sizeof(int8_t); - *(int16_t *) buf = setup->pathSize; - buf += sizeof(int16_t); - memcpy(buf, setup->path, setup->pathSize); - buf += setup->pathSize; +void* decodeUdfCallResponse(const void* buf, SUdfCallResponse* callRsp) { + buf = taosDecodeFixedI8(buf, &callRsp->callType); + switch (callRsp->callType) { + case TSDB_UDF_CALL_SCALA_PROC: + buf = tDecodeDataBlock(buf, &callRsp->resultData); break; - } - - case UDF_TASK_CALL: { - SUdfCallRequest *call = (SUdfCallRequest *) (request->subReq); - *(int64_t *) buf = call->udfHandle; - buf += sizeof(int64_t); - *(int8_t *) buf = call->step; - buf += sizeof(int8_t); - *(int32_t *) buf = call->inputBytes; - buf += sizeof(int32_t); - memcpy(buf, call->input, call->inputBytes); - buf += call->inputBytes; - *(int32_t *) buf = call->stateBytes; - buf += sizeof(int32_t); - memcpy(buf, call->state, call->stateBytes); - buf += call->stateBytes; + case TSDB_UDF_CALL_AGG_INIT: + buf = decodeUdfInterBuf(buf, &callRsp->resultBuf); break; - } - - case UDF_TASK_TEARDOWN: { - SUdfTeardownRequest *teardown = (SUdfTeardownRequest *) (request->subReq); - *(int64_t *) buf = teardown->udfHandle; - buf += sizeof(int64_t); + case TSDB_UDF_CALL_AGG_PROC: + buf = decodeUdfInterBuf(buf, &callRsp->resultBuf); break; - } - default: + case TSDB_UDF_CALL_AGG_MERGE: + buf = decodeUdfInterBuf(buf, &callRsp->resultBuf); + break; + case TSDB_UDF_CALL_AGG_FIN: + buf = decodeUdfInterBuf(buf, &callRsp->resultBuf); break; } + return (void*)buf; +} - request->msgLen = buf - bufBegin; - *(int32_t *) bufBegin = request->msgLen; - *pBuf = bufBegin; - *pBufLen = request->msgLen; +int32_t encodeUdfTeardownResponse(void** buf, const SUdfTeardownResponse* teardownRsp) { return 0; } -int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { - debugPrint("%s", "decoding request"); - if (*(int32_t *) bufMsg != bufLen) { - debugPrint("%s", "decoding request error"); - return -1; - } - char *buf = bufMsg; - SUdfRequest *request = taosMemoryMalloc(sizeof(SUdfRequest)); - request->subReq = NULL; - request->msgLen = *(int32_t *) (buf); - buf += sizeof(int32_t); - request->seqNum = *(int64_t *) (buf); - buf += sizeof(int64_t); - request->type = *(int8_t *) (buf); - buf += sizeof(int8_t); - - switch (request->type) { - case UDF_TASK_SETUP: { - SUdfSetupRequest *setup = taosMemoryMalloc(sizeof(SUdfSetupRequest)); - - memcpy(setup->udfName, buf, 16); - buf += 16; - setup->scriptType = *(int8_t *) buf; - buf += sizeof(int8_t); - setup->udfType = *(int8_t *) buf; - buf += sizeof(int8_t); - setup->pathSize = *(int16_t *) buf; - buf += sizeof(int16_t); - setup->path = buf; - buf += setup->pathSize; - - request->subReq = setup; - break; - } - case UDF_TASK_CALL: { - SUdfCallRequest *call = taosMemoryMalloc(sizeof(SUdfCallRequest)); - - call->udfHandle = *(int64_t *) buf; - buf += sizeof(int64_t); - call->step = *(int8_t *) buf; - buf += sizeof(int8_t); - call->inputBytes = *(int32_t *) buf; - buf += sizeof(int32_t); - call->input = buf; - buf += call->inputBytes; - call->stateBytes = *(int32_t *) buf; - buf += sizeof(int32_t); - call->state = buf; - buf += call->stateBytes; - - request->subReq = call; - break; - } - - case UDF_TASK_TEARDOWN: { - SUdfTeardownRequest *teardown = taosMemoryMalloc(sizeof(SUdfTeardownRequest)); - - teardown->udfHandle = *(int64_t *) buf; - buf += sizeof(int64_t); - - request->subReq = teardown; - } +void* decodeUdfTeardownResponse(const void* buf, SUdfTeardownResponse* teardownResponse) { + return (void*)buf; +} +int32_t encodeUdfResponse(void** buf, const SUdfResponse* rsp) { + int32_t len = 0; + if (buf == NULL) { + len += sizeof(rsp->msgLen); + } else { + *(int32_t*)(*buf) = rsp->msgLen; + *buf = POINTER_SHIFT(*buf, sizeof(rsp->msgLen)); } - if (buf - bufMsg != bufLen) { - debugPrint("%s", "decode request error"); - taosMemoryFree(request->subReq); - taosMemoryFree(request); - return -1; - } - *pRequest = request; - return 0; -} -int32_t encodeResponse(char **pBuf, int32_t *pBufLen, SUdfResponse *response) { - debugPrint("%s", "encoding response"); + if (buf == NULL) { + len += sizeof(rsp->seqNum); + } else { + *(int64_t*)(*buf) = rsp->seqNum; + *buf = POINTER_SHIFT(*buf, sizeof(rsp->seqNum)); + } - int32_t len = sizeof(SUdfResponse) - sizeof(void *); + len += taosEncodeFixedI64(buf, rsp->seqNum); + len += taosEncodeFixedI8(buf, rsp->type); + len += taosEncodeFixedI32(buf, rsp->code); - switch (response->type) { - case UDF_TASK_SETUP: { - len += sizeof(SUdfSetupResponse); + switch (rsp->type) { + case UDF_TASK_SETUP: + len += encodeUdfSetupResponse(buf, &rsp->setupRsp); break; - } - case UDF_TASK_CALL: { - SUdfCallResponse *callResp = (SUdfCallResponse *) (response->subRsp); - len += sizeof(SUdfCallResponse) - 2 * sizeof(char *) + - callResp->outputBytes + callResp->newStateBytes; + case UDF_TASK_CALL: + len += encodeUdfCallResponse(buf, &rsp->callRsp); break; - } - case UDF_TASK_TEARDOWN: { - len += sizeof(SUdfTeardownResponse); + case UDF_TASK_TEARDOWN: + len += encodeUdfTeardownResponse(buf, &rsp->teardownRsp); + break; + default: + //TODO: log error break; - } } + return len; +} - char *bufBegin = taosMemoryMalloc(len); - char *buf = bufBegin; - - //skip msgLen - buf += sizeof(int32_t); - - *(int64_t *) buf = response->seqNum; - buf += sizeof(int64_t); - *(int8_t *) buf = response->type; - buf += sizeof(int8_t); - *(int32_t *) buf = response->code; - buf += sizeof(int32_t); - +void* decodeUdfResponse(const void* buf, SUdfResponse* rsp) { + rsp->msgLen = *(int32_t*)(buf); + buf = POINTER_SHIFT(buf, sizeof(rsp->msgLen)); + rsp->seqNum = *(int64_t*)(buf); + buf = POINTER_SHIFT(buf, sizeof(rsp->seqNum)); + buf = taosDecodeFixedI64(buf, &rsp->seqNum); + buf = taosDecodeFixedI8(buf, &rsp->type); + buf = taosDecodeFixedI32(buf, &rsp->code); - switch (response->type) { - case UDF_TASK_SETUP: { - SUdfSetupResponse *setupResp = (SUdfSetupResponse *) (response->subRsp); - *(int64_t *) buf = setupResp->udfHandle; - buf += sizeof(int64_t); + switch (rsp->type) { + case UDF_TASK_SETUP: + buf = decodeUdfSetupResponse(buf, &rsp->setupRsp); break; - } - case UDF_TASK_CALL: { - SUdfCallResponse *callResp = (SUdfCallResponse *) (response->subRsp); - *(int32_t *) buf = callResp->outputBytes; - buf += sizeof(int32_t); - memcpy(buf, callResp->output, callResp->outputBytes); - buf += callResp->outputBytes; - - *(int32_t *) buf = callResp->newStateBytes; - buf += sizeof(int32_t); - memcpy(buf, callResp->newState, callResp->newStateBytes); - buf += callResp->newStateBytes; + case UDF_TASK_CALL: + buf = decodeUdfCallResponse(buf, &rsp->callRsp); break; - } - case UDF_TASK_TEARDOWN: { - SUdfTeardownResponse *teardownResp = (SUdfTeardownResponse *) (response->subRsp); + case UDF_TASK_TEARDOWN: + buf = decodeUdfTeardownResponse(buf, &rsp->teardownRsp); break; - } default: + //TODO: log error break; } - response->msgLen = buf - bufBegin; - *(int32_t *) bufBegin = response->msgLen; - *pBuf = bufBegin; - *pBufLen = response->msgLen; - return 0; + return (void*)buf; } -int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { - debugPrint("%s", "decoding response"); - - if (*(int32_t *) bufMsg != bufLen) { - debugPrint("%s", "can not decode response"); - return -1; +void freeUdfColumnData(SUdfColumnData *data) { + if (data->varLengthColumn) { + taosMemoryFree(data->varLenCol.varOffsets); + data->varLenCol.varOffsets = NULL; + taosMemoryFree(data->varLenCol.payload); + data->varLenCol.payload = NULL; + } else { + taosMemoryFree(data->fixLenCol.nullBitmap); + data->fixLenCol.nullBitmap = NULL; + taosMemoryFree(data->fixLenCol.data); + data->fixLenCol.data = NULL; } - char *buf = bufMsg; - SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); - rsp->msgLen = *(int32_t *) buf; - buf += sizeof(int32_t); - rsp->seqNum = *(int64_t *) buf; - buf += sizeof(int64_t); - rsp->type = *(int8_t *) buf; - buf += sizeof(int8_t); - rsp->code = *(int32_t *) buf; - buf += sizeof(int32_t); +} - switch (rsp->type) { - case UDF_TASK_SETUP: { - SUdfSetupResponse *setupRsp = (SUdfSetupResponse *) taosMemoryMalloc(sizeof(SUdfSetupResponse)); - setupRsp->udfHandle = *(int64_t *) buf; - buf += sizeof(int64_t); - rsp->subRsp = (char *) setupRsp; - break; - } - case UDF_TASK_CALL: { - SUdfCallResponse *callRsp = (SUdfCallResponse *) taosMemoryMalloc(sizeof(SUdfCallResponse)); - callRsp->outputBytes = *(int32_t *) buf; - buf += sizeof(int32_t); +void freeUdfColumn(SUdfColumn* col) { + freeUdfColumnData(&col->colData); +} - callRsp->output = buf; - buf += callRsp->outputBytes; +void freeUdfDataDataBlock(SUdfDataBlock *block) { + for (int32_t i = 0; i < block->numOfCols; ++i) { + freeUdfColumn(block->udfCols[i]); + taosMemoryFree(block->udfCols[i]); + block->udfCols[i] = NULL; + } + taosMemoryFree(block->udfCols); + block->udfCols = NULL; +} - callRsp->newStateBytes = *(int32_t *) buf; - buf += sizeof(int32_t); +void freeUdfInterBuf(SUdfInterBuf *buf) { + taosMemoryFree(buf->buf); + buf->buf = NULL; +} - callRsp->newState = buf; - buf += callRsp->newStateBytes; - rsp->subRsp = callRsp; - break; +int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock) { + udfBlock->numOfRows = block->info.rows; + udfBlock->numOfCols = block->info.numOfCols; + udfBlock->udfCols = taosMemoryCalloc(udfBlock->numOfCols, sizeof(SUdfColumn*)); + for (int32_t i = 0; i < udfBlock->numOfCols; ++i) { + udfBlock->udfCols[i] = taosMemoryCalloc(1, sizeof(SUdfColumn)); + SColumnInfoData *col= (SColumnInfoData*)taosArrayGet(block->pDataBlock, i); + SUdfColumn *udfCol = udfBlock->udfCols[i]; + udfCol->colMeta.type = col->info.type; + udfCol->colMeta.bytes = col->info.bytes; + udfCol->colMeta.scale = col->info.scale; + udfCol->colMeta.precision = col->info.precision; + udfCol->colData.numOfRows = udfBlock->numOfRows; + udfCol->colData.varLengthColumn = IS_VAR_DATA_TYPE(udfCol->colMeta.type); + if (udfCol->colData.varLengthColumn) { + udfCol->colData.varLenCol.varOffsetsLen = sizeof(int32_t) * udfBlock->numOfRows; + udfCol->colData.varLenCol.varOffsets = taosMemoryMalloc(udfCol->colData.varLenCol.varOffsetsLen); + memcpy(udfCol->colData.varLenCol.varOffsets, col->varmeta.offset, udfCol->colData.varLenCol.varOffsetsLen); + udfCol->colData.varLenCol.payloadLen = colDataGetLength(col, udfBlock->numOfRows); + udfCol->colData.varLenCol.payload = taosMemoryMalloc(udfCol->colData.varLenCol.payloadLen); + memcpy(udfCol->colData.varLenCol.payload, col->pData, udfCol->colData.varLenCol.payloadLen); + } else { + udfCol->colData.fixLenCol.nullBitmapLen = BitmapLen(udfCol->colData.numOfRows); + int32_t bitmapLen = udfCol->colData.fixLenCol.nullBitmapLen; + udfCol->colData.fixLenCol.nullBitmap = taosMemoryMalloc(udfCol->colData.fixLenCol.nullBitmapLen); + char* bitmap = udfCol->colData.fixLenCol.nullBitmap; + memcpy(bitmap, col->nullbitmap, bitmapLen); + udfCol->colData.fixLenCol.dataLen = colDataGetLength(col, udfBlock->numOfRows); + int32_t dataLen = udfCol->colData.fixLenCol.dataLen; + udfCol->colData.fixLenCol.data = taosMemoryMalloc(udfCol->colData.fixLenCol.dataLen); + char* data = udfCol->colData.fixLenCol.data; + memcpy(data, col->pData, dataLen); } - case UDF_TASK_TEARDOWN: { - SUdfTeardownResponse *teardownRsp = (SUdfTeardownResponse *) taosMemoryMalloc(sizeof(SUdfTeardownResponse)); - rsp->subRsp = teardownRsp; + } + return 0; +} + +int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { + block->info.numOfCols = 1; + block->info.rows = udfCol->colData.numOfRows; + block->info.hasVarCol = udfCol->colData.varLengthColumn; + + block->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + taosArraySetSize(block->pDataBlock, 1); + SColumnInfoData *col = taosArrayGet(block->pDataBlock, 0); + SUdfColumnMeta *meta = &udfCol->colMeta; + col->info.precision = meta->precision; + col->info.bytes = meta->bytes; + col->info.scale = meta->scale; + col->info.type = meta->type; + SUdfColumnData *data = &udfCol->colData; + + if (!IS_VAR_DATA_TYPE(meta->type)) { + col->nullbitmap = taosMemoryMalloc(data->fixLenCol.nullBitmapLen); + memcpy(col->nullbitmap, data->fixLenCol.nullBitmap, data->fixLenCol.nullBitmapLen); + col->pData = taosMemoryMalloc(data->fixLenCol.dataLen); + memcpy(col->pData, data->fixLenCol.data, data->fixLenCol.dataLen); + } else { + col->varmeta.offset = taosMemoryMalloc(data->varLenCol.varOffsetsLen); + memcpy(col->varmeta.offset, data->varLenCol.varOffsets, data->varLenCol.varOffsetsLen); + col->pData = taosMemoryMalloc(data->varLenCol.payloadLen); + memcpy(col->pData, data->varLenCol.payload, data->varLenCol.payloadLen); + } + return 0; +} + +int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) { + output->info.rows = input->numOfRows; + output->info.numOfCols = numOfCols; + bool hasVarCol = false; + for (int32_t i = 0; i < numOfCols; ++i) { + if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) { + hasVarCol = true; break; } - default: - break; - } - if (buf - bufMsg != bufLen) { - debugPrint("%s", "can not decode response"); - taosMemoryFree(rsp->subRsp); - taosMemoryFree(rsp); - return -1; } - *pResponse = rsp; + output->info.hasVarCol = hasVarCol; + + //TODO: free the array output->pDataBlock + output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + taosArrayPush(output->pDataBlock, input->columnData); return 0; } -void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) { - debugPrint("Process exited with status %" PRId64 ", signal %d", exit_status, term_signal); - uv_close((uv_handle_t *) req, NULL); - //TODO: restart the udfd process +int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) { + if (input->info.numOfCols != 1) { + fnError("scalar function only support one column"); + return -1; + } + output->numOfRows = input->info.rows; + //TODO: memory + output->columnData = taosArrayGet(input->pDataBlock, 0); + return 0; } void onUdfcPipeClose(uv_handle_t *handle) { SClientUvConn *conn = handle->data; - if (!udfTaskQueueIsEmpty(&conn->taskQueue)) { - SClientUvTaskNode *task = udfTaskQueueHeadTask(&conn->taskQueue); + if (!QUEUE_EMPTY(&conn->taskQueue)) { + QUEUE* h = QUEUE_HEAD(&conn->taskQueue); + SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); task->errCode = 0; uv_sem_post(&task->taskSem); + QUEUE_REMOVE(&task->procTaskQueue); } taosMemoryFree(conn->readBuf.buf); @@ -485,23 +607,24 @@ int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvT debugPrint("%s", "get uv task result"); if (uvTask->type == UV_TASK_REQ_RSP) { if (uvTask->rspBuf.base != NULL) { - SUdfResponse *rsp; - decodeResponse(uvTask->rspBuf.base, uvTask->rspBuf.len, &rsp); - task->errCode = rsp->code; + SUdfResponse rsp; + void* buf = decodeUdfResponse(uvTask->rspBuf.base, &rsp); + assert(uvTask->rspBuf.len == POINTER_DISTANCE(buf, uvTask->rspBuf.base)); + task->errCode = rsp.code; switch (task->type) { case UDF_TASK_SETUP: { //TODO: copy or not - task->_setup.rsp = *(SUdfSetupResponse *) (rsp->subRsp); + task->_setup.rsp = rsp.setupRsp; break; } case UDF_TASK_CALL: { - task->_call.rsp = *(SUdfCallResponse *) (rsp->subRsp); + task->_call.rsp = rsp.callRsp; //TODO: copy or not break; } case UDF_TASK_TEARDOWN: { - task->_teardown.rsp = *(SUdfTeardownResponse *) (rsp->subRsp); + task->_teardown.rsp = rsp.teardownRsp; //TODO: copy or not? break; } @@ -512,8 +635,6 @@ int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvT // TODO: the call buffer is setup and freed by udf invocation taosMemoryFree(uvTask->rspBuf.base); - taosMemoryFree(rsp->subRsp); - taosMemoryFree(rsp); } else { task->errCode = uvTask->errCode; } @@ -577,14 +698,16 @@ void udfcUvHandleRsp(SClientUvConn *conn) { SClientConnBuf *connBuf = &conn->readBuf; int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen int32_t then seqnum - if (udfTaskQueueIsEmpty(&conn->taskQueue)) { + if (QUEUE_EMPTY(&conn->taskQueue)) { //LOG error return; } bool found = false; SClientUvTaskNode *taskFound = NULL; - SClientUvTaskNode *task = udfTaskQueueNext(&conn->taskQueue); - while (task != &conn->taskQueue) { + QUEUE* h = QUEUE_NEXT(&conn->taskQueue); + SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); + + while (h != &conn->taskQueue) { if (task->seqNum == seqNum) { if (found == false) { found = true; @@ -594,15 +717,17 @@ void udfcUvHandleRsp(SClientUvConn *conn) { continue; } } - task = udfTaskQueueNext(task); + h = QUEUE_NEXT(h); + task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); } if (taskFound) { taskFound->rspBuf = uv_buf_init(connBuf->buf, connBuf->len); - udfTaskQueueRemoveTask(taskFound); + QUEUE_REMOVE(&taskFound->connTaskQueue); uv_sem_post(&taskFound->taskSem); + QUEUE_REMOVE(&taskFound->procTaskQueue); } else { - //LOG error + //TODO: LOG error } connBuf->buf = NULL; connBuf->total = -1; @@ -611,7 +736,18 @@ void udfcUvHandleRsp(SClientUvConn *conn) { } void udfcUvHandleError(SClientUvConn *conn) { - uv_close((uv_handle_t *) conn->pipe, onUdfcPipeClose); + while (!QUEUE_EMPTY(&conn->taskQueue)) { + QUEUE* h = QUEUE_HEAD(&conn->taskQueue); + SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); + task->errCode = UDFC_CODE_PIPE_READ_ERR; + uv_sem_post(&task->taskSem); + QUEUE_REMOVE(&task->procTaskQueue); + } + + uv_close((uv_handle_t *) conn->pipe, NULL); + taosMemoryFree(conn->pipe); + taosMemoryFree(conn->readBuf.buf); + taosMemoryFree(conn); } void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { @@ -643,7 +779,7 @@ void onUdfClientWrite(uv_write_t *write, int status) { if (status == 0) { uv_pipe_t *pipe = uvTask->pipe; SClientUvConn *conn = pipe->data; - udfTaskQueueInsertTail(&conn->taskQueue, uvTask); + QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); } else { //TODO Log error; } @@ -661,35 +797,39 @@ void onUdfClientConnect(uv_connect_t *connect, int status) { uv_read_start((uv_stream_t *) uvTask->pipe, udfcAllocateBuffer, onUdfcRead); taosMemoryFree(connect); uv_sem_post(&uvTask->taskSem); + QUEUE_REMOVE(&uvTask->procTaskQueue); } int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask) { SClientUvTaskNode *uvTask = taosMemoryCalloc(1, sizeof(SClientUvTaskNode)); uvTask->type = uvTaskType; + uvTask->udfc = task->session->udfc; if (uvTaskType == UV_TASK_CONNECT) { } else if (uvTaskType == UV_TASK_REQ_RSP) { uvTask->pipe = task->session->udfSvcPipe; SUdfRequest request; request.type = task->type; - request.seqNum = gUdfTaskSeqNum++; + request.seqNum = atomic_fetch_add_64(&gUdfTaskSeqNum, 1); if (task->type == UDF_TASK_SETUP) { - request.subReq = &task->_setup.req; + request.setup = task->_setup.req; request.type = UDF_TASK_SETUP; } else if (task->type == UDF_TASK_CALL) { - request.subReq = &task->_call.req; + request.call = task->_call.req; request.type = UDF_TASK_CALL; } else if (task->type == UDF_TASK_TEARDOWN) { - request.subReq = &task->_teardown.req; + request.teardown = task->_teardown.req; request.type = UDF_TASK_TEARDOWN; } else { //TODO log and return error } - char *buf = NULL; - int32_t bufLen = 0; - encodeRequest(&buf, &bufLen, &request); - uvTask->reqBuf = uv_buf_init(buf, bufLen); + int32_t bufLen = encodeUdfRequest(NULL, &request); + request.msgLen = bufLen; + void *bufBegin = taosMemoryMalloc(bufLen); + void *buf = bufBegin; + encodeUdfRequest(&buf, &request); + uvTask->reqBuf = uv_buf_init(bufBegin, bufLen); uvTask->seqNum = request.seqNum; } else if (uvTaskType == UV_TASK_DISCONNECT) { uvTask->pipe = task->session->udfSvcPipe; @@ -702,11 +842,11 @@ int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { debugPrint("%s, %d", "queue uv task", uvTask->type); - - uv_mutex_lock(&gUdfTaskQueueMutex); - udfTaskQueueInsertTail(gUdfTaskQueue, uvTask); - uv_mutex_unlock(&gUdfTaskQueueMutex); - uv_async_send(&gUdfLoopTaskAync); + SUdfdProxy *udfc = uvTask->udfc; + uv_mutex_lock(&udfc->gUdfTaskQueueMutex); + QUEUE_INSERT_TAIL(&udfc->gUdfTaskQueue, &uvTask->recvTaskQueue); + uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); + uv_async_send(&udfc->gUdfLoopTaskAync); uv_sem_wait(&uvTask->taskSem); uv_sem_destroy(&uvTask->taskSem); @@ -719,7 +859,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { switch (uvTask->type) { case UV_TASK_CONNECT: { uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); - uv_pipe_init(&gUdfdLoop, pipe, 0); + uv_pipe_init(&uvTask->udfc->gUdfdLoop, pipe, 0); uvTask->pipe = pipe; SClientUvConn *conn = taosMemoryMalloc(sizeof(SClientUvConn)); @@ -728,7 +868,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { conn->readBuf.cap = 0; conn->readBuf.buf = 0; conn->readBuf.total = -1; - udfTaskQueueInit(&conn->taskQueue); + QUEUE_INIT(&conn->taskQueue); pipe->data = conn; @@ -747,7 +887,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { } case UV_TASK_DISCONNECT: { SClientUvConn *conn = uvTask->pipe->data; - udfTaskQueueInsertTail(&conn->taskQueue, uvTask); + QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); uv_close((uv_handle_t *) uvTask->pipe, onUdfcPipeClose); break; } @@ -760,67 +900,98 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { } void udfClientAsyncCb(uv_async_t *async) { - SClientUvTaskNode node; - SClientUvTaskQueue q = &node; - udfTaskQueueInit(q); + SUdfdProxy *udfc = async->data; + QUEUE wq; - uv_mutex_lock(&gUdfTaskQueueMutex); - udfTaskQueueMove(gUdfTaskQueue, q); - uv_mutex_unlock(&gUdfTaskQueueMutex); + uv_mutex_lock(&udfc->gUdfTaskQueueMutex); + QUEUE_MOVE(&udfc->gUdfTaskQueue, &wq); + uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); - while (!udfTaskQueueIsEmpty(q)) { - SClientUvTaskNode *task = udfTaskQueueHeadTask(q); - udfTaskQueueRemoveTask(task); + while (!QUEUE_EMPTY(&wq)) { + QUEUE* h = QUEUE_HEAD(&wq); + QUEUE_REMOVE(h); + SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue); startUvUdfTask(task); + QUEUE_INSERT_TAIL(&udfc->gUvProcTaskQueue, &task->procTaskQueue); } } -void udfStopAsyncCb(uv_async_t *async) { - uv_stop(&gUdfdLoop); - uv_loop_close(&gUdfdLoop); -} - -void startUdfd(void *argsThread) { - uv_loop_init(&gUdfdLoop); - - //TODO: path - uv_process_options_t options; - static char path[256] = {0}; - size_t cwdSize; - uv_cwd(path, &cwdSize); - strcat(path, "./udfd"); - char* args[2] = {path, NULL}; - options.args = args; - options.file = path; - options.exit_cb = onUdfdExit; - - int err = uv_spawn(&gUdfdLoop, &gUdfdProcess, &options); - if (err != 0) { - debugPrint("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err)); +void cleanUpUvTasks(SUdfdProxy *udfc) { + QUEUE wq; + + uv_mutex_lock(&udfc->gUdfTaskQueueMutex); + QUEUE_MOVE(&udfc->gUdfTaskQueue, &wq); + uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); + + while (!QUEUE_EMPTY(&wq)) { + QUEUE* h = QUEUE_HEAD(&wq); + QUEUE_REMOVE(h); + SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue); + if (udfc->gUdfcState == UDFC_STATE_STOPPING) { + task->errCode = UDFC_CODE_STOPPING; } + uv_sem_post(&task->taskSem); + } + + // TODO: deal with tasks that are waiting result. + while (!QUEUE_EMPTY(&udfc->gUvProcTaskQueue)) { + QUEUE* h = QUEUE_HEAD(&udfc->gUvProcTaskQueue); + QUEUE_REMOVE(h); + SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, procTaskQueue); + if (udfc->gUdfcState == UDFC_STATE_STOPPING) { + task->errCode = UDFC_CODE_STOPPING; + } + uv_sem_post(&task->taskSem); + } +} - uv_async_init(&gUdfdLoop, &gUdfLoopTaskAync, udfClientAsyncCb); - uv_async_init(&gUdfdLoop, &gUdfLoopStopAsync, udfStopAsyncCb); - uv_mutex_init(&gUdfTaskQueueMutex); - udfTaskQueueInit(gUdfTaskQueue); - uv_barrier_wait(&gUdfInitBarrier); - uv_run(&gUdfdLoop, UV_RUN_DEFAULT); +void udfStopAsyncCb(uv_async_t *async) { + SUdfdProxy *udfc = async->data; + cleanUpUvTasks(udfc); + if (udfc->gUdfcState == UDFC_STATE_STOPPING) { + uv_stop(&udfc->gUdfdLoop); + } +} + +void constructUdfService(void *argsThread) { + SUdfdProxy *udfc = (SUdfdProxy*)argsThread; + uv_loop_init(&udfc->gUdfdLoop); + + uv_async_init(&udfc->gUdfdLoop, &udfc->gUdfLoopTaskAync, udfClientAsyncCb); + udfc->gUdfLoopTaskAync.data = udfc; + uv_async_init(&udfc->gUdfdLoop, &udfc->gUdfLoopStopAsync, udfStopAsyncCb); + udfc->gUdfLoopStopAsync.data = udfc; + uv_mutex_init(&udfc->gUdfTaskQueueMutex); + QUEUE_INIT(&udfc->gUdfTaskQueue); + QUEUE_INIT(&udfc->gUvProcTaskQueue); + uv_barrier_wait(&udfc->gUdfInitBarrier); + //TODO return value of uv_run + uv_run(&udfc->gUdfdLoop, UV_RUN_DEFAULT); + uv_loop_close(&udfc->gUdfdLoop); } -int32_t startUdfService() { - uv_barrier_init(&gUdfInitBarrier, 2); - uv_thread_create(&gUdfLoopThread, startUdfd, 0); - uv_barrier_wait(&gUdfInitBarrier); +int32_t udfcOpen(int32_t dnodeId, UdfcHandle *udfc) { + SUdfdProxy *proxy = taosMemoryCalloc(1, sizeof(SUdfdProxy)); + proxy->dnodeId = dnodeId; + proxy->gUdfcState = UDFC_STATE_STARTNG; + uv_barrier_init(&proxy->gUdfInitBarrier, 2); + uv_thread_create(&proxy->gUdfLoopThread, constructUdfService, proxy); + uv_barrier_wait(&proxy->gUdfInitBarrier); + proxy->gUdfcState = UDFC_STATE_READY; + *udfc = proxy; return 0; } -int32_t stopUdfService() { - uv_barrier_destroy(&gUdfInitBarrier); - uv_process_kill(&gUdfdProcess, SIGINT); - uv_async_send(&gUdfLoopStopAsync); - uv_mutex_destroy(&gUdfTaskQueueMutex); - uv_thread_join(&gUdfLoopThread); +int32_t udfcClose(UdfcHandle udfcHandle) { + SUdfdProxy *udfc = udfcHandle; + udfc->gUdfcState = UDFC_STATE_STOPPING; + uv_async_send(&udfc->gUdfLoopStopAsync); + uv_thread_join(&udfc->gUdfLoopThread); + uv_mutex_destroy(&udfc->gUdfTaskQueueMutex); + uv_barrier_destroy(&udfc->gUdfInitBarrier); + udfc->gUdfcState = UDFC_STATUS_FINAL; + taosMemoryFree(udfc); return 0; } @@ -838,19 +1009,16 @@ int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) { return task->errCode; } -int32_t setupUdf(SUdfInfo *udfInfo, UdfHandle *handle) { +int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle *funcHandle) { debugPrint("%s", "client setup udf"); SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; task->session = taosMemoryMalloc(sizeof(SUdfUvSession)); + task->session->udfc = udfc; task->type = UDF_TASK_SETUP; SUdfSetupRequest *req = &task->_setup.req; - memcpy(req->udfName, udfInfo->udfName, 16); - req->path = udfInfo->path; - req->pathSize = strlen(req->path) + 1; - req->udfType = udfInfo->udfType; - req->scriptType = udfInfo->scriptType; + memcpy(req->udfName, udfName, TSDB_FUNC_NAME_LEN); int32_t errCode = udfcRunUvTask(task, UV_TASK_CONNECT); if (errCode != 0) { @@ -862,14 +1030,14 @@ int32_t setupUdf(SUdfInfo *udfInfo, UdfHandle *handle) { SUdfSetupResponse *rsp = &task->_setup.rsp; task->session->severHandle = rsp->udfHandle; - *handle = task->session; + *funcHandle = task->session; int32_t err = task->errCode; taosMemoryFree(task); return err; } -int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, char **newState, - int32_t *newStateSize, SUdfDataBlock *output) { +int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, + SSDataBlock* output, SUdfInterBuf *newState) { debugPrint("%s", "client call udf"); SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); @@ -878,27 +1046,108 @@ int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, S task->type = UDF_TASK_CALL; SUdfCallRequest *req = &task->_call.req; - - req->state = state; - req->stateBytes = stateSize; - req->inputBytes = input.size; - req->input = input.data; req->udfHandle = task->session->severHandle; - req->step = step; + req->callType = callType; + + switch (callType) { + case TSDB_UDF_CALL_AGG_INIT: { + req->initFirst = 1; + break; + } + case TSDB_UDF_CALL_AGG_PROC: { + req->block = *input; + req->interBuf = *state; + break; + } + case TSDB_UDF_CALL_AGG_MERGE: { + req->interBuf = *state; + req->interBuf2 = *state2; + break; + } + case TSDB_UDF_CALL_AGG_FIN: { + req->interBuf = *state; + break; + } + case TSDB_UDF_CALL_SCALA_PROC: { + req->block = *input; + break; + } + } udfcRunUvTask(task, UV_TASK_REQ_RSP); SUdfCallResponse *rsp = &task->_call.rsp; - *newState = rsp->newState; - *newStateSize = rsp->newStateBytes; - output->size = rsp->outputBytes; - output->data = rsp->output; - int32_t err = task->errCode; + switch (callType) { + case TSDB_UDF_CALL_AGG_INIT: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_AGG_PROC: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_AGG_MERGE: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_AGG_FIN: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_SCALA_PROC: { + *output = rsp->resultData; + break; + } + } + taosMemoryFree(task); + return task->errCode; +} + +//TODO: translate these calls to callUdf +int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) { + int8_t callType = TSDB_UDF_CALL_AGG_INIT; + + int32_t err = callUdf(handle, callType, NULL, NULL, NULL, NULL, interBuf); + + return err; +} + +// input: block, state +// output: interbuf, +int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState) { + int8_t callType = TSDB_UDF_CALL_AGG_PROC; + int32_t err = callUdf(handle, callType, block, state, NULL, NULL, newState); + return err; +} + +// input: interbuf1, interbuf2 +// output: resultBuf +int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf) { + int8_t callType = TSDB_UDF_CALL_AGG_MERGE; + int32_t err = callUdf(handle, callType, NULL, interBuf1, interBuf2, NULL, resultBuf); + return err; +} + +// input: interBuf +// output: resultData +int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) { + int8_t callType = TSDB_UDF_CALL_AGG_PROC; + int32_t err = callUdf(handle, callType, NULL, interBuf, NULL, NULL, resultData); + return err; +} + +int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam* output) { + int8_t callType = TSDB_UDF_CALL_SCALA_PROC; + SSDataBlock inputBlock = {0}; + convertScalarParamToDataBlock(input, numOfCols, &inputBlock); + SSDataBlock resultBlock = {0}; + int32_t err = callUdf(handle, callType, &inputBlock, NULL, NULL, &resultBlock, NULL); + convertDataBlockToScalarParm(&resultBlock, output); return err; } -int32_t teardownUdf(UdfHandle handle) { +int32_t teardownUdf(UdfcFuncHandle handle) { debugPrint("%s", "client teardown udf"); SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index b473f060c099c1277e6c92752980e8c2e6a7bed6..5f7532da87c1ceeb313a6eefe104219e0b3690d9 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -12,345 +12,608 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - #include "uv.h" #include "os.h" -#include "tlog.h" +#include "fnLog.h" +#include "thash.h" #include "tudf.h" #include "tudfInt.h" +#include "tdataformat.h" +#include "tglobal.h" +#include "tmsg.h" +#include "trpc.h" + +typedef struct SUdfdContext { + uv_loop_t *loop; + uv_pipe_t ctrlPipe; + uv_signal_t intrSignal; + char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN]; + uv_pipe_t listeningPipe; + void *clientRpc; + + uv_mutex_t udfsMutex; + SHashObj *udfsHash; + + bool printVersion; +} SUdfdContext; -static uv_loop_t *loop; +SUdfdContext global; typedef struct SUdfdUvConn { - uv_stream_t *client; - char *inputBuf; - int32_t inputLen; - int32_t inputCap; - int32_t inputTotal; + uv_stream_t *client; + char *inputBuf; + int32_t inputLen; + int32_t inputCap; + int32_t inputTotal; } SUdfdUvConn; typedef struct SUvUdfWork { - uv_stream_t *client; - uv_buf_t input; - uv_buf_t output; + uv_stream_t *client; + uv_buf_t input; + uv_buf_t output; } SUvUdfWork; -typedef struct SUdf { - int32_t refCount; - - char name[16]; - int8_t type; +typedef enum { UDF_STATE_INIT = 0, UDF_STATE_LOADING, UDF_STATE_READY, UDF_STATE_UNLOADING } EUdfState; - uv_lib_t lib; - TUdfFunc normalFunc; +typedef struct SUdf { + int32_t refCount; + EUdfState state; + uv_mutex_t lock; + uv_cond_t condReady; + + char name[16]; + int8_t type; + char path[PATH_MAX]; + + uv_lib_t lib; + TUdfScalarProcFunc scalarProcFunc; + TUdfFreeUdfColumnFunc freeUdfColumn; } SUdf; -//TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix -//TODO: add private udf structure. -typedef struct SUdfHandle { - SUdf *udf; -} SUdfHandle; - +// TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix +// TODO: add private udf structure. +typedef struct SUdfcFuncHandle { + SUdf *udf; +} SUdfcFuncHandle; + +int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf); + +int32_t udfdLoadUdf(char *udfName, SEpSet *pEpSet, SUdf *udf) { + strcpy(udf->name, udfName); + + udfdFillUdfInfoFromMNode(global.clientRpc, pEpSet, udf->name, udf); + + int err = uv_dlopen(udf->path, &udf->lib); + if (err != 0) { + fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); + // TODO set error + } + // TODO: find all the functions + char normalFuncName[TSDB_FUNC_NAME_LEN] = {0}; + strcpy(normalFuncName, udfName); + uv_dlsym(&udf->lib, normalFuncName, (void **)(&udf->scalarProcFunc)); + char freeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; + char *freeSuffix = "_free"; + strncpy(freeFuncName, normalFuncName, strlen(normalFuncName)); + strncat(freeFuncName, freeSuffix, strlen(freeSuffix)); + uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn)); + return 0; +} void udfdProcessRequest(uv_work_t *req) { - SUvUdfWork *uvUdf = (SUvUdfWork *) (req->data); - SUdfRequest *request = NULL; - decodeRequest(uvUdf->input.base, uvUdf->input.len, &request); - - switch (request->type) { - case UDF_TASK_SETUP: { - debugPrint("%s", "process setup request"); - SUdf *udf = taosMemoryMalloc(sizeof(SUdf)); - udf->refCount = 0; - SUdfSetupRequest *setup = request->subReq; - strcpy(udf->name, setup->udfName); - int err = uv_dlopen(setup->path, &udf->lib); - if (err != 0) { - debugPrint("can not load library %s. error: %s", setup->path, uv_strerror(err)); - //TODO set error - } - - char normalFuncName[32] = {0}; - strcpy(normalFuncName, setup->udfName); - //TODO error, - //TODO find all functions normal, init, destroy, normal, merge, finalize - uv_dlsym(&udf->lib, normalFuncName, (void **) (&udf->normalFunc)); - - SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle)); - handle->udf = udf; - udf->refCount++; - //TODO: allocate private structure and call init function and set it to handle - SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); - rsp->seqNum = request->seqNum; - rsp->type = request->type; - rsp->code = 0; - SUdfSetupResponse *subRsp = taosMemoryMalloc(sizeof(SUdfSetupResponse)); - subRsp->udfHandle = (int64_t) (handle); - rsp->subRsp = subRsp; - char *buf; - int32_t len; - encodeResponse(&buf, &len, rsp); - - uvUdf->output = uv_buf_init(buf, len); - - taosMemoryFree(rsp->subRsp); - taosMemoryFree(rsp); - taosMemoryFree(request->subReq); - taosMemoryFree(request); - taosMemoryFree(uvUdf->input.base); - break; - } - - case UDF_TASK_CALL: { - debugPrint("%s", "process call request"); - SUdfCallRequest *call = request->subReq; - SUdfHandle *handle = (SUdfHandle *) (call->udfHandle); - SUdf *udf = handle->udf; - char *newState; - int32_t newStateSize; - SUdfDataBlock input = {.data = call->input, .size= call->inputBytes}; - SUdfDataBlock output; - //TODO: call different functions according to the step - udf->normalFunc(call->step, call->state, call->stateBytes, input, &newState, &newStateSize, &output); - - SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); - rsp->seqNum = request->seqNum; - rsp->type = request->type; - rsp->code = 0; - SUdfCallResponse *subRsp = taosMemoryMalloc(sizeof(SUdfCallResponse)); - subRsp->outputBytes = output.size; - subRsp->output = output.data; - subRsp->newStateBytes = newStateSize; - subRsp->newState = newState; - rsp->subRsp = subRsp; - - char *buf; - int32_t len; - encodeResponse(&buf, &len, rsp); - uvUdf->output = uv_buf_init(buf, len); - - taosMemoryFree(rsp->subRsp); - taosMemoryFree(rsp); - taosMemoryFree(newState); - taosMemoryFree(output.data); - taosMemoryFree(request->subReq); - taosMemoryFree(request); - taosMemoryFree(uvUdf->input.base); - break; + SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data); + SUdfRequest request = {0}; + decodeUdfRequest(uvUdf->input.base, &request); + + switch (request.type) { + case UDF_TASK_SETUP: { + // TODO: tracable id from client. connect, setup, call, teardown + fnInfo("%" PRId64 " setup request. udf name: %s", request.seqNum, request.setup.udfName); + SUdfSetupRequest *setup = &request.setup; + + SUdf *udf = NULL; + uv_mutex_lock(&global.udfsMutex); + SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN); + if (*udfInHash) { + ++(*udfInHash)->refCount; + udf = *udfInHash; + uv_mutex_unlock(&global.udfsMutex); + } else { + SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); + udfNew->refCount = 1; + udfNew->state = UDF_STATE_INIT; + + uv_mutex_init(&udfNew->lock); + uv_cond_init(&udfNew->condReady); + udf = udfNew; + taosHashPut(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN, &udfNew, sizeof(&udfNew)); + uv_mutex_unlock(&global.udfsMutex); + } + + uv_mutex_lock(&udf->lock); + if (udf->state == UDF_STATE_INIT) { + udf->state = UDF_STATE_LOADING; + udfdLoadUdf(setup->udfName, &setup->epSet, udf); + udf->state = UDF_STATE_READY; + uv_cond_broadcast(&udf->condReady); + uv_mutex_unlock(&udf->lock); + } else { + while (udf->state != UDF_STATE_READY) { + uv_cond_wait(&udf->condReady, &udf->lock); } - case UDF_TASK_TEARDOWN: { - debugPrint("%s", "process teardown request"); - - SUdfTeardownRequest *teardown = request->subReq; - SUdfHandle *handle = (SUdfHandle *) (teardown->udfHandle); - SUdf *udf = handle->udf; - udf->refCount--; - if (udf->refCount == 0) { - uv_dlclose(&udf->lib); - } - taosMemoryFree(udf); - //TODO: call destroy and free udf private - taosMemoryFree(handle); - - SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); - rsp->seqNum = request->seqNum; - rsp->type = request->type; - rsp->code = 0; - SUdfTeardownResponse *subRsp = taosMemoryMalloc(sizeof(SUdfTeardownResponse)); - rsp->subRsp = subRsp; - char *buf; - int32_t len; - encodeResponse(&buf, &len, rsp); - uvUdf->output = uv_buf_init(buf, len); - - taosMemoryFree(rsp->subRsp); - taosMemoryFree(rsp); - taosMemoryFree(request->subReq); - taosMemoryFree(request); - taosMemoryFree(uvUdf->input.base); - break; - } - default: { - break; - } - + uv_mutex_unlock(&udf->lock); + } + SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle)); + handle->udf = udf; + // TODO: allocate private structure and call init function and set it to handle + SUdfResponse rsp; + rsp.seqNum = request.seqNum; + rsp.type = request.type; + rsp.code = 0; + rsp.setupRsp.udfHandle = (int64_t)(handle); + int32_t len = encodeUdfResponse(NULL, &rsp); + rsp.msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, &rsp); + + uvUdf->output = uv_buf_init(bufBegin, len); + + taosMemoryFree(uvUdf->input.base); + break; } + case UDF_TASK_CALL: { + SUdfCallRequest *call = &request.call; + fnDebug("%" PRId64 "call request. call type %d, handle: %" PRIx64, request.seqNum, call->callType, + call->udfHandle); + SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle); + SUdf *udf = handle->udf; + + SUdfDataBlock input = {0}; + convertDataBlockToUdfDataBlock(&call->block, &input); + SUdfColumn output = {0}; + // TODO: call different functions according to call type, for now just calar + if (call->callType == TSDB_UDF_CALL_SCALA_PROC) { + udf->scalarProcFunc(input, &output); + } + + SUdfResponse response = {0}; + SUdfResponse *rsp = &response; + if (call->callType == TSDB_UDF_CALL_SCALA_PROC) { + rsp->seqNum = request.seqNum; + rsp->type = request.type; + rsp->code = 0; + SUdfCallResponse *subRsp = &rsp->callRsp; + subRsp->callType = call->callType; + convertUdfColumnToDataBlock(&output, &subRsp->resultData); + } + + int32_t len = encodeUdfResponse(NULL, rsp); + rsp->msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, rsp); + uvUdf->output = uv_buf_init(bufBegin, len); + + // TODO: free udf column + udf->freeUdfColumn(&output); + + taosMemoryFree(uvUdf->input.base); + break; + } + case UDF_TASK_TEARDOWN: { + SUdfTeardownRequest *teardown = &request.teardown; + fnInfo("teardown. %" PRId64 "handle:%" PRIx64, request.seqNum, teardown->udfHandle) SUdfcFuncHandle *handle = + (SUdfcFuncHandle *)(teardown->udfHandle); + SUdf *udf = handle->udf; + bool unloadUdf = false; + uv_mutex_lock(&global.udfsMutex); + udf->refCount--; + if (udf->refCount == 0) { + unloadUdf = true; + taosHashRemove(global.udfsHash, udf->name, TSDB_FUNC_NAME_LEN); + } + uv_mutex_unlock(&global.udfsMutex); + if (unloadUdf) { + uv_cond_destroy(&udf->condReady); + uv_mutex_destroy(&udf->lock); + uv_dlclose(&udf->lib); + taosMemoryFree(udf); + } + // TODO: call destroy and free udf private + taosMemoryFree(handle); + + SUdfResponse response; + SUdfResponse *rsp = &response; + rsp->seqNum = request.seqNum; + rsp->type = request.type; + rsp->code = 0; + int32_t len = encodeUdfResponse(NULL, rsp); + rsp->msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, rsp); + uvUdf->output = uv_buf_init(bufBegin, len); + + taosMemoryFree(uvUdf->input.base); + break; + } + default: { + break; + } + } } void udfdOnWrite(uv_write_t *req, int status) { - debugPrint("%s", "after writing to pipe"); - if (status < 0) { - debugPrint("Write error %s", uv_err_name(status)); - } - SUvUdfWork *work = (SUvUdfWork *) req->data; - debugPrint("\tlength: %zu", work->output.len); - taosMemoryFree(work->output.base); - taosMemoryFree(work); - taosMemoryFree(req); + SUvUdfWork *work = (SUvUdfWork *)req->data; + if (status < 0) { + // TODO:log error and process it. + } + fnDebug("send response. length:%zu, status: %s", work->output.len, uv_err_name(status)); + taosMemoryFree(work->output.base); + taosMemoryFree(work); + taosMemoryFree(req); } - void udfdSendResponse(uv_work_t *work, int status) { - debugPrint("%s", "send response"); - SUvUdfWork *udfWork = (SUvUdfWork *) (work->data); + SUvUdfWork *udfWork = (SUvUdfWork *)(work->data); - uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t)); - write_req->data = udfWork; - uv_write(write_req, udfWork->client, &udfWork->output, 1, udfdOnWrite); + uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t)); + write_req->data = udfWork; + uv_write(write_req, udfWork->client, &udfWork->output, 1, udfdOnWrite); - taosMemoryFree(work); + taosMemoryFree(work); } void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { - debugPrint("%s", "allocate buffer for read"); - SUdfdUvConn *ctx = handle->data; - int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); - if (ctx->inputCap == 0) { - ctx->inputBuf = taosMemoryMalloc(msgHeadSize); - if (ctx->inputBuf) { - ctx->inputLen = 0; - ctx->inputCap = msgHeadSize; - ctx->inputTotal = -1; - - buf->base = ctx->inputBuf; - buf->len = ctx->inputCap; - } else { - //TODO: log error - buf->base = NULL; - buf->len = 0; - } + SUdfdUvConn *ctx = handle->data; + int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); + if (ctx->inputCap == 0) { + ctx->inputBuf = taosMemoryMalloc(msgHeadSize); + if (ctx->inputBuf) { + ctx->inputLen = 0; + ctx->inputCap = msgHeadSize; + ctx->inputTotal = -1; + + buf->base = ctx->inputBuf; + buf->len = ctx->inputCap; } else { - ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap; - void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap); - if (inputBuf) { - ctx->inputBuf = inputBuf; - buf->base = ctx->inputBuf + ctx->inputLen; - buf->len = ctx->inputCap - ctx->inputLen; - } else { - //TODO: log error - buf->base = NULL; - buf->len = 0; - } + // TODO: log error + buf->base = NULL; + buf->len = 0; } - debugPrint("\tinput buf cap - len - total : %d - %d - %d", ctx->inputCap, ctx->inputLen, ctx->inputTotal); - + } else { + ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap; + void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap); + if (inputBuf) { + ctx->inputBuf = inputBuf; + buf->base = ctx->inputBuf + ctx->inputLen; + buf->len = ctx->inputCap - ctx->inputLen; + } else { + // TODO: log error + buf->base = NULL; + buf->len = 0; + } + } + fnDebug("allocate buf. input buf cap - len - total : %d - %d - %d", ctx->inputCap, ctx->inputLen, ctx->inputTotal); } bool isUdfdUvMsgComplete(SUdfdUvConn *pipe) { - if (pipe->inputTotal == -1 && pipe->inputLen >= sizeof(int32_t)) { - pipe->inputTotal = *(int32_t *) (pipe->inputBuf); - } - if (pipe->inputLen == pipe->inputCap && pipe->inputTotal == pipe->inputCap) { - return true; - } - return false; + if (pipe->inputTotal == -1 && pipe->inputLen >= sizeof(int32_t)) { + pipe->inputTotal = *(int32_t *)(pipe->inputBuf); + } + if (pipe->inputLen == pipe->inputCap && pipe->inputTotal == pipe->inputCap) { + fnDebug("receive request complete. length %d", pipe->inputLen); + return true; + } + return false; } void udfdHandleRequest(SUdfdUvConn *conn) { - uv_work_t *work = taosMemoryMalloc(sizeof(uv_work_t)); - SUvUdfWork *udfWork = taosMemoryMalloc(sizeof(SUvUdfWork)); - udfWork->client = conn->client; - udfWork->input = uv_buf_init(conn->inputBuf, conn->inputLen); - conn->inputBuf = NULL; - conn->inputLen = 0; - conn->inputCap = 0; - conn->inputTotal = -1; - work->data = udfWork; - uv_queue_work(loop, work, udfdProcessRequest, udfdSendResponse); + uv_work_t *work = taosMemoryMalloc(sizeof(uv_work_t)); + SUvUdfWork *udfWork = taosMemoryMalloc(sizeof(SUvUdfWork)); + udfWork->client = conn->client; + udfWork->input = uv_buf_init(conn->inputBuf, conn->inputLen); + conn->inputBuf = NULL; + conn->inputLen = 0; + conn->inputCap = 0; + conn->inputTotal = -1; + work->data = udfWork; + uv_queue_work(global.loop, work, udfdProcessRequest, udfdSendResponse); } void udfdPipeCloseCb(uv_handle_t *pipe) { - SUdfdUvConn *conn = pipe->data; - taosMemoryFree(conn->client); - taosMemoryFree(conn->inputBuf); - taosMemoryFree(conn); + SUdfdUvConn *conn = pipe->data; + taosMemoryFree(conn->client); + taosMemoryFree(conn->inputBuf); + taosMemoryFree(conn); } -void udfdUvHandleError(SUdfdUvConn *conn) { - uv_close((uv_handle_t *) conn->client, udfdPipeCloseCb); -} +void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn->client, udfdPipeCloseCb); } void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { - debugPrint("%s, nread: %zd", "read from pipe", nread); + fnDebug("udf read %zu bytes from client", nread); + if (nread == 0) return; - if (nread == 0) return; + SUdfdUvConn *conn = client->data; - SUdfdUvConn *conn = client->data; - - if (nread > 0) { - conn->inputLen += nread; - if (isUdfdUvMsgComplete(conn)) { - udfdHandleRequest(conn); - } else { - //log error or continue; - } - return; + if (nread > 0) { + conn->inputLen += nread; + if (isUdfdUvMsgComplete(conn)) { + udfdHandleRequest(conn); + } else { + // log error or continue; } + return; + } - if (nread < 0) { - debugPrint("Read error %s", uv_err_name(nread)); - if (nread == UV_EOF) { - //TODO check more when close - } else { - } - udfdUvHandleError(conn); + if (nread < 0) { + fnDebug("Receive error %s", uv_err_name(nread)); + if (nread == UV_EOF) { + // TODO check more when close + } else { } + udfdUvHandleError(conn); + } } void udfdOnNewConnection(uv_stream_t *server, int status) { - debugPrint("%s", "on new connection"); - if (status < 0) { - // TODO - return; - } + fnDebug("new connection"); + if (status < 0) { + // TODO + return; + } + + uv_pipe_t *client = (uv_pipe_t *)taosMemoryMalloc(sizeof(uv_pipe_t)); + uv_pipe_init(global.loop, client, 0); + if (uv_accept(server, (uv_stream_t *)client) == 0) { + SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn)); + ctx->client = (uv_stream_t *)client; + ctx->inputBuf = 0; + ctx->inputLen = 0; + ctx->inputCap = 0; + client->data = ctx; + ctx->client = (uv_stream_t *)client; + uv_read_start((uv_stream_t *)client, udfdAllocBuffer, udfdPipeRead); + } else { + uv_close((uv_handle_t *)client, NULL); + } +} - uv_pipe_t *client = (uv_pipe_t *) taosMemoryMalloc(sizeof(uv_pipe_t)); - uv_pipe_init(loop, client, 0); - if (uv_accept(server, (uv_stream_t *) client) == 0) { - SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn)); - ctx->client = (uv_stream_t *) client; - ctx->inputBuf = 0; - ctx->inputLen = 0; - ctx->inputCap = 0; - client->data = ctx; - ctx->client = (uv_stream_t *) client; - uv_read_start((uv_stream_t *) client, udfdAllocBuffer, udfdPipeRead); +void udfdIntrSignalHandler(uv_signal_t *handle, int signum) { + fnInfo("udfd signal received: %d\n", signum); + uv_fs_t req; + uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL); + uv_signal_stop(handle); + uv_stop(global.loop); +} + +void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; } + +int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf) { + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 1; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, udfName); + + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void *pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = contLen; + rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(clientRpc, pEpSet, &rpcMsg, &rpcRsp); + SRetrieveFuncRsp retrieveRsp = {0}; + tDeserializeSRetrieveFuncRsp(rpcRsp.pCont, rpcRsp.contLen, &retrieveRsp); + + SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0); + + char path[PATH_MAX] = {0}; + taosGetTmpfilePath("/tmp", "libudf", path); + TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + // TODO check for failure of flush to disk + taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); + taosCloseFile(&file); + strncpy(udf->path, path, strlen(path)); + taosArrayDestroy(retrieveRsp.pFuncInfos); + + rpcFreeCont(rpcRsp.pCont); + return 0; +} + +int32_t udfdOpenClientRpc() { + char *pass = "taosdata"; + char *user = "root"; + char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt); + SRpcInit rpcInit = {0}; + rpcInit.label = (char *)"UDFD"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = udfdProcessRpcRsp; + rpcInit.sessions = 1024; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = 30 * 1000; + rpcInit.parent = &global; + + rpcInit.user = (char *)user; + rpcInit.ckey = (char *)"key"; + rpcInit.secret = (char *)secretEncrypt; + rpcInit.spi = 1; + + global.clientRpc = rpcOpen(&rpcInit); + + return 0; +} + +int32_t udfdCloseClientRpc() { + rpcClose(global.clientRpc); + return 0; +} + +static void udfdPrintVersion() { +#ifdef TD_ENTERPRISE + char *releaseName = "enterprise"; +#else + char *releaseName = "community"; +#endif + printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); + printf("gitinfo: %s\n", gitinfo); + printf("buildInfo: %s\n", buildinfo); +} + +static int32_t udfdParseArgs(int32_t argc, char *argv[]) { + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default is %s\n", configDir); + return -1; + } + } else if (strcmp(argv[i], "-V") == 0) { + global.printVersion = true; } else { - uv_close((uv_handle_t *) client, NULL); } + } + + return 0; } -void removeListeningPipe(int sig) { - uv_fs_t req; - uv_fs_unlink(loop, &req, "udf.sock", NULL); - exit(0); +static int32_t udfdInitLog() { + char logName[12] = {0}; + snprintf(logName, sizeof(logName), "%slog", "udfd"); + return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0); } -int main() { - debugPrint("libuv version: %x", UV_VERSION_HEX); +void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) { + buf->base = taosMemoryMalloc(suggested_size); + buf->len = suggested_size; +} - loop = uv_default_loop(); - uv_fs_t req; - uv_fs_unlink(loop, &req, "udf.sock", NULL); +void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf) { + if (nread < 0) { + fnError("udfd ctrl pipe read error. %s", uv_err_name(nread)); + uv_close((uv_handle_t *)q, NULL); + uv_stop(global.loop); + return; + } + fnError("udfd ctrl pipe read %zu bytes", nread); + taosMemoryFree(buf->base); +} - uv_pipe_t server; - uv_pipe_init(loop, &server, 0); +static int32_t removeListeningPipe() { + uv_fs_t req; + int err = uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL); + uv_fs_req_cleanup(&req); + return err; +} - signal(SIGINT, removeListeningPipe); +static int32_t udfdUvInit() { + uv_loop_t *loop = taosMemoryMalloc(sizeof(uv_loop_t)); + if (loop) { + uv_loop_init(loop); + } + global.loop = loop; + + uv_pipe_init(global.loop, &global.ctrlPipe, 1); + uv_pipe_open(&global.ctrlPipe, 0); + uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb); + + char dnodeId[8] = {0}; + size_t dnodeIdSize; + int32_t err = uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize); + if (err != 0) { + dnodeId[0] = '1'; + } + char listenPipeName[32] = {0}; + snprintf(listenPipeName, sizeof(listenPipeName), "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId); + strcpy(global.listenPipeName, listenPipeName); + + removeListeningPipe(); + + uv_pipe_init(global.loop, &global.listeningPipe, 0); + + uv_signal_init(global.loop, &global.intrSignal); + uv_signal_start(&global.intrSignal, udfdIntrSignalHandler, SIGINT); + + int r; + fnInfo("bind to pipe %s", global.listenPipeName); + if ((r = uv_pipe_bind(&global.listeningPipe, listenPipeName))) { + fnError("Bind error %s", uv_err_name(r)); + removeListeningPipe(); + return -1; + } + if ((r = uv_listen((uv_stream_t *)&global.listeningPipe, 128, udfdOnNewConnection))) { + fnError("Listen error %s", uv_err_name(r)); + removeListeningPipe(); + return -2; + } + return 0; +} - int r; - if ((r = uv_pipe_bind(&server, "udf.sock"))) { - debugPrint("Bind error %s\n", uv_err_name(r)); - removeListeningPipe(0); - return 1; - } - if ((r = uv_listen((uv_stream_t *) &server, 128, udfdOnNewConnection))) { - debugPrint("Listen error %s", uv_err_name(r)); - return 2; - } - uv_run(loop, UV_RUN_DEFAULT); - uv_loop_close(loop); +static int32_t udfdRun() { + global.udfsHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + uv_mutex_init(&global.udfsMutex); + + // TOOD: client rpc to fetch udf function info from mnode + if (udfdOpenClientRpc() != 0) { + fnError("open rpc connection to mnode failure"); + return -1; + } + + if (udfdUvInit() != 0) { + fnError("uv init failure"); + return -2; + } + + fnInfo("start the udfd"); + int code = uv_run(global.loop, UV_RUN_DEFAULT); + fnInfo("udfd stopped. result: %s", uv_err_name(code)); + int codeClose = uv_loop_close(global.loop); + fnDebug("uv loop close. result: %s", uv_err_name(codeClose)); + udfdCloseClientRpc(); + uv_mutex_destroy(&global.udfsMutex); + taosHashCleanup(global.udfsHash); + return code; +} + +int main(int argc, char *argv[]) { + if (!taosCheckSystemIsSmallEnd()) { + printf("failed to start since on non-small-end machines\n"); + return -1; + } + + if (udfdParseArgs(argc, argv) != 0) { + printf("failed to start since parse args error\n"); + return -1; + } + + if (global.printVersion) { + udfdPrintVersion(); + return 0; + } + + if (udfdInitLog() != 0) { + printf("failed to start since init log error\n"); + return -1; + } + + if (taosInitCfg(configDir, NULL, NULL, NULL, 0) != 0) { + fnError("failed to start since read config error"); + return -1; + } + + return udfdRun(); } diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index bd742d23d07ba5ae1467dcfc3fb658c8cd9bc4ef..0727a4a1d26db2678c1bb63a7c193e0a91fd2321 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -5,9 +5,11 @@ #include "uv.h" #include "os.h" #include "tudf.h" +#include "tdatablock.h" int main(int argc, char *argv[]) { - startUdfService(); + UdfcHandle udfc; + udfcOpen(1, &udfc); uv_sleep(1000); char path[256] = {0}; size_t cwdSize = 256; @@ -18,29 +20,42 @@ int main(int argc, char *argv[]) { } fprintf(stdout, "current working directory:%s\n", path); strcat(path, "/libudf1.so"); - SUdfInfo udfInfo = {.udfName="udf1", .path=path}; - UdfHandle handle; - setupUdf(&udfInfo, &handle); + UdfcFuncHandle handle; + SEpSet epSet; + setupUdf(udfc, "udf1", &epSet, &handle); - //char state[5000000] = "state"; - //char input[5000000] = "input"; - int dataSize = 500; - int callCount = 2; - if (argc > 1) dataSize = atoi(argv[1]); - if (argc > 2) callCount = atoi(argv[2]); - char *state = taosMemoryMalloc(dataSize); - char *input = taosMemoryMalloc(dataSize); - SUdfDataBlock blockInput = {.data = input, .size = dataSize}; - SUdfDataBlock blockOutput; - char* newState; - int32_t newStateSize; - for (int l = 0; l < callCount; ++l) { - callUdf(handle, 0, state, dataSize, blockInput, &newState, &newStateSize, &blockOutput); + SSDataBlock block = {0}; + SSDataBlock* pBlock = █ + pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + pBlock->info.numOfCols = 1; + pBlock->info.rows = 4; + char data[16] = {0}; + char bitmap[4] = {0}; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData colInfo = {0}; + colInfo.info.type = TSDB_DATA_TYPE_INT; + colInfo.info.bytes = sizeof(int32_t); + colInfo.info.colId = 1; + colInfo.pData = data; + colInfo.nullbitmap = bitmap; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + colDataAppendInt32(&colInfo, j, &j); + } + taosArrayPush(pBlock->pDataBlock, &colInfo); + } + + SScalarParam input = {0}; + input.numOfRows = pBlock->info.rows; + input.columnData = taosArrayGet(pBlock->pDataBlock, 0); + SScalarParam output = {0}; + callUdfScalarFunc(handle, &input, 1 , &output); + + SColumnInfoData *col = output.columnData; + for (int32_t i = 0; i < output.numOfRows; ++i) { + fprintf(stderr, "%d\t%d\n" , i, *(int32_t*)(col->pData + i *sizeof(int32_t))); } - taosMemoryFree(state); - taosMemoryFree(input); teardownUdf(handle); - stopUdfService(); + udfcClose(udfc); } diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c index dc88e8cf3e194532a34414b9362711141b0b8a9d..94cab9fee96377c170c70d45ec9a85f29a4976e3 100644 --- a/source/libs/function/test/udf1.c +++ b/source/libs/function/test/udf1.c @@ -2,20 +2,68 @@ #include #include -#include "os.h" #include "tudf.h" -void udf1(int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, - char **newState, int32_t *newStateSize, SUdfDataBlock *output) { - fprintf(stdout, "%s, step:%d\n", "udf function called", step); - char *newStateBuf = taosMemoryMalloc(stateSize); - memcpy(newStateBuf, state, stateSize); - *newState = newStateBuf; - *newStateSize = stateSize; - - char *outputBuf = taosMemoryMalloc(input.size); - memcpy(outputBuf, input.data, input.size); - output->data = outputBuf; - output->size = input.size; - return; +#undef malloc +#define malloc malloc +#undef free +#define free free + +int32_t udf1_setup() { + return 0; +} + +int32_t udf1_teardown() { + return 0; +} + +int32_t udf1(SUdfDataBlock block, SUdfColumn *resultCol) { + SUdfColumnData *resultData = &resultCol->colData; + resultData->numOfRows = block.numOfRows; + SUdfColumnData *srcData = &block.udfCols[0]->colData; + resultData->varLengthColumn = srcData->varLengthColumn; + + if (resultData->varLengthColumn) { + resultData->varLenCol.varOffsetsLen = srcData->varLenCol.varOffsetsLen; + resultData->varLenCol.varOffsets = malloc(resultData->varLenCol.varOffsetsLen); + memcpy(resultData->varLenCol.varOffsets, srcData->varLenCol.varOffsets, srcData->varLenCol.varOffsetsLen); + + resultData->varLenCol.payloadLen = srcData->varLenCol.payloadLen; + resultData->varLenCol.payload = malloc(resultData->varLenCol.payloadLen); + memcpy(resultData->varLenCol.payload, srcData->varLenCol.payload, srcData->varLenCol.payloadLen); + } else { + resultData->fixLenCol.nullBitmapLen = srcData->fixLenCol.nullBitmapLen; + resultData->fixLenCol.nullBitmap = malloc(resultData->fixLenCol.nullBitmapLen); + memcpy(resultData->fixLenCol.nullBitmap, srcData->fixLenCol.nullBitmap, srcData->fixLenCol.nullBitmapLen); + + resultData->fixLenCol.dataLen = srcData->fixLenCol.dataLen; + resultData->fixLenCol.data = malloc(resultData->fixLenCol.dataLen); + memcpy(resultData->fixLenCol.data, srcData->fixLenCol.data, srcData->fixLenCol.dataLen); + for (int32_t i = 0; i < resultData->numOfRows; ++i) { + *(resultData->fixLenCol.data + i * sizeof(int32_t)) = 88; + } + } + + SUdfColumnMeta *meta = &resultCol->colMeta; + meta->bytes = 4; + meta->type = TSDB_DATA_TYPE_INT; + meta->scale = 0; + meta->precision = 0; + return 0; } + +int32_t udf1_free(SUdfColumn *col) { + SUdfColumnData *data = &col->colData; + if (data->varLengthColumn) { + free(data->varLenCol.varOffsets); + data->varLenCol.varOffsets = NULL; + free(data->varLenCol.payload); + data->varLenCol.payload = NULL; + } else { + free(data->fixLenCol.nullBitmap); + data->fixLenCol.nullBitmap = NULL; + free(data->fixLenCol.data); + data->fixLenCol.data = NULL; + } + return 0; +} \ No newline at end of file diff --git a/source/libs/index/inc/indexFstRegex.h b/source/libs/index/inc/indexFstRegex.h index 8fb545533682c84476b55a447827a5034127f6e5..2bf9c9b791397d634cc0704d5f556448d796c6aa 100644 --- a/source/libs/index/inc/indexFstRegex.h +++ b/source/libs/index/inc/indexFstRegex.h @@ -31,6 +31,9 @@ extern "C" { typedef enum { MATCH, JUMP, SPLIT, RANGE } InstType; typedef struct MatchValue { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } MatchValue; typedef struct JumpValue { uint32_t step; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 058c33b725f84dfa509b0f8dacb6a6ff18562dfe..554b8092a20a17eda8958a6f99ce6d8e8aa9c77c 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -748,7 +748,7 @@ static SArray* tfileGetFileList(const char* path) { sprintf(buf, "%s/%s", path, file); taosArrayPush(files, &buf); } - taosCloseDir(pDir); + taosCloseDir(&pDir); taosArraySort(files, tfileCompare); tfileRmExpireFile(files); diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index ee9da50ab222e754903cb921215024b208e5cb0c..0162b754ee4c936da63b48fbc7bf06055f8802cb 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -485,7 +485,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) { std::string colVal("ab"); SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; + SIndexTermQuery query = { term, QUERY_TERM}; SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); fObj->Get(&query, result); @@ -625,7 +625,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colVal("v3"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; + SIndexTermQuery query = { term, QUERY_TERM }; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -640,7 +640,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colVal("v2"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; + SIndexTermQuery query = { term, QUERY_TERM }; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 9b0452425a7be6fe51f9543ac9e1dc2db15d8871..f789d2313635b4afbda199d509d532193cdd0261 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -108,7 +108,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); - for (uint i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { colVal[i % colVal.size()] = '0' + i % 128; SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); diff --git a/source/libs/monitor/src/monMsg.c b/source/libs/monitor/src/monMsg.c index adacbf479be27f60e3eb376007730cdf5bd5ef1f..24ea474ceaf39193e5df674573eca47864ba6f29 100644 --- a/source/libs/monitor/src/monMsg.c +++ b/source/libs/monitor/src/monMsg.c @@ -473,7 +473,7 @@ int32_t tSerializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo) for (int32_t i = 0; i < taosArrayGetSize(pInfo->pVloads); ++i) { SVnodeLoad *pLoad = taosArrayGet(pInfo->pVloads, i); if (tEncodeI32(&encoder, pLoad->vgId) < 0) return -1; - if (tEncodeI8(&encoder, pLoad->role) < 0) return -1; + if (tEncodeI32(&encoder, pLoad->syncState) < 0) return -1; if (tEncodeI64(&encoder, pLoad->numOfTables) < 0) return -1; if (tEncodeI64(&encoder, pLoad->numOfTimeSeries) < 0) return -1; if (tEncodeI64(&encoder, pLoad->totalStorage) < 0) return -1; @@ -507,7 +507,7 @@ int32_t tDeserializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInf for (int32_t i = 0; i < arraySize; ++i) { SVnodeLoad load = {0}; if (tDecodeI32(&decoder, &load.vgId) < 0) return -1; - if (tDecodeI8(&decoder, &load.role) < 0) return -1; + if (tDecodeI32(&decoder, &load.syncState) < 0) return -1; if (tDecodeI64(&decoder, &load.numOfTables) < 0) return -1; if (tDecodeI64(&decoder, &load.numOfTimeSeries) < 0) return -1; if (tDecodeI64(&decoder, &load.totalStorage) < 0) return -1; @@ -530,3 +530,30 @@ void tFreeSMonVloadInfo(SMonVloadInfo *pInfo) { taosArrayDestroy(pInfo->pVloads); pInfo->pVloads = NULL; } + +int32_t tSerializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInfo) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI8(&encoder, pInfo->isMnode) < 0) return -1; + if (tEncodeI32(&encoder, pInfo->load.syncState) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInfo) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI8(&decoder, &pInfo->isMnode) < 0) return -1; + if (tDecodeI32(&decoder, &pInfo->load.syncState) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} \ No newline at end of file diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index d4457a056e26d54ac60cfa7fe2141ee947a37643..bc9614b16905ccdd6ede25efe1daa0c877238460 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -149,7 +149,9 @@ SNodeptr nodesMakeNode(ENodeType type) { case QUERY_NODE_RESET_QUERY_CACHE_STMT: return makeNode(type, sizeof(SNode)); case QUERY_NODE_COMPACT_STMT: + break; case QUERY_NODE_CREATE_FUNCTION_STMT: + return makeNode(type, sizeof(SCreateFunctionStmt)); case QUERY_NODE_DROP_FUNCTION_STMT: break; case QUERY_NODE_CREATE_STREAM_STMT: @@ -167,6 +169,7 @@ SNodeptr nodesMakeNode(ENodeType type) { case QUERY_NODE_SHOW_QNODES_STMT: case QUERY_NODE_SHOW_SNODES_STMT: case QUERY_NODE_SHOW_BNODES_STMT: + case QUERY_NODE_SHOW_CLUSTER_STMT: case QUERY_NODE_SHOW_DATABASES_STMT: case QUERY_NODE_SHOW_FUNCTIONS_STMT: case QUERY_NODE_SHOW_INDEXES_STMT: @@ -1006,6 +1009,7 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) { case OP_TYPE_NOT_LIKE: case OP_TYPE_MATCH: case OP_TYPE_NMATCH: + case OP_TYPE_JSON_CONTAINS: case OP_TYPE_IS_NULL: case OP_TYPE_IS_NOT_NULL: case OP_TYPE_IS_TRUE: @@ -1024,7 +1028,6 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) { bool nodesIsJsonOp(const SOperatorNode* pOp) { switch (pOp->opType) { case OP_TYPE_JSON_GET_VALUE: - case OP_TYPE_JSON_CONTAINS: return true; default: break; diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 85f9b3399c0ff51d564264d92473bc3bcafdd4d3..7cb64b78e30ca21e6e8c94129437b07e64c0c2e8 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -53,6 +53,7 @@ typedef enum EDatabaseOptionType { DB_OPTION_VGROUPS, DB_OPTION_SINGLE_STABLE, DB_OPTION_STREAM_MODE, + DB_OPTION_STRICT, DB_OPTION_RETENTIONS } EDatabaseOptionType; @@ -166,7 +167,7 @@ SNode* createExplainStmt(SAstCreateContext* pCxt, bool analyze, SNode* pOptions, SNode* createDescribeStmt(SAstCreateContext* pCxt, SNode* pRealTable); SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt); SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups); -SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize); +SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize); SNode* createDropFunctionStmt(SAstCreateContext* pCxt, const SToken* pFuncName); SNode* createStreamOptions(SAstCreateContext* pCxt); SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pStreamName, SNode* pRealTable, SNode* pOptions, SNode* pQuery); diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index cf87f57d4f3fa4b45373785be22afeed32757260..ed7267655c61f9cf3cec9297d0f3bd33b1a67eaf 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -89,7 +89,7 @@ static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) { (int32_t)TD_BITMAP_BYTES(pTableInfo->numOfColumns - 1); } -static FORCE_INLINE void getSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, +static FORCE_INLINE void getSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, int32_t *toffset, col_id_t *colIdx) { col_id_t schemaIdx = 0; if (IS_DATA_COL_ORDERED(spd)) { @@ -131,7 +131,6 @@ static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks* int32_t schemaIdxCompar(const void *lhs, const void *rhs); int32_t boundIdxCompar(const void *lhs, const void *rhs); void setBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols); -void destroyBoundColumnInfo(SParsedDataColInfo* pColList); void destroyBlockArrayList(SArray* pDataBlockList); void destroyBlockHashmap(SHashObj* pDataBlockHash); int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); @@ -139,5 +138,7 @@ int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq); int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** pVgDataBlocks); +int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq); +int32_t allocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize); #endif // TDENGINE_DATABLOCKMGT_H diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index ea01caf9d785f38ccb4c0c61e0de649e80ef0ceb..87348e292ebb1cc92a2fed126e4976ebd9e4da9b 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -21,6 +21,8 @@ extern "C" { #endif #include "parser.h" +#include "parToken.h" +#include "parUtil.h" int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery); int32_t parse(SParseContext* pParseCxt, SQuery** pQuery); diff --git a/source/libs/parser/inc/parToken.h b/source/libs/parser/inc/parToken.h index 1122bafd12f21f5ded91b43b4d593781a2bd4b97..d8a2c22fd8a29cd00673f8a71277dab58a430aa9 100644 --- a/source/libs/parser/inc/parToken.h +++ b/source/libs/parser/inc/parToken.h @@ -20,7 +20,6 @@ extern "C" { #endif -#include "os.h" #include "ttokendef.h" // used to denote the minimum unite in sql parsing diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 742ab303d3b419c46f344815c04ee7e2f379728c..460a089802a2629f38abbb56100d95298c6ac6f1 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -47,6 +47,7 @@ SSchema *getTableTagSchema(const STableMeta* pTableMeta); int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta); STableComInfo getTableInfo(const STableMeta* pTableMeta); +int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* errMsg, int16_t startColId); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 93e56424df3b2d438a52638b4636320999df69d6..9ff36c702f59b687df0c1bffd512db2b1733f3b2 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -161,6 +161,7 @@ db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { ((SDatabaseOptions*)B)->pSingleStable = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { ((SDatabaseOptions*)B)->pStreamMode = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } db_options(A) ::= db_options(B) RETENTIONS retention_list(C). { ((SDatabaseOptions*)B)->pRetentions = C; A = B; } +db_options(A) ::= db_options(B) STRICT NK_INTEGER(C). { ((SDatabaseOptions*)B)->pStrict = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; } alter_db_options(A) ::= alter_db_option(B). { A = createDatabaseOptions(pCxt); A = setDatabaseAlterOption(pCxt, A, &B); } alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setDatabaseAlterOption(pCxt, B, &C); } @@ -175,6 +176,7 @@ alter_db_option(A) ::= WAL NK_INTEGER(B). alter_db_option(A) ::= QUORUM NK_INTEGER(B). { A.type = DB_OPTION_QUORUM; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } alter_db_option(A) ::= REPLICA NK_INTEGER(B). { A.type = DB_OPTION_REPLICA; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } +alter_db_option(A) ::= STRICT NK_INTEGER(B). { A.type = DB_OPTION_STRICT; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } %type integer_list { SNodeList* } %destructor integer_list { nodesDestroyList($$); } @@ -344,6 +346,7 @@ cmd ::= SHOW TOPICS. cmd ::= SHOW VARIABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLE_STMT, NULL, NULL); } cmd ::= SHOW BNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT, NULL, NULL); } cmd ::= SHOW SNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT, NULL, NULL); } +cmd ::= SHOW CLUSTER. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT, NULL, NULL); } db_name_cond_opt(A) ::= . { A = createDefaultDatabaseCondValue(pCxt); } db_name_cond_opt(A) ::= db_name(B) NK_DOT. { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } @@ -359,7 +362,7 @@ from_db_opt(A) ::= FROM db_name(B). %type func_name_list { SNodeList* } %destructor func_name_list { nodesDestroyList($$); } func_name_list(A) ::= func_name(B). { A = createNodeList(pCxt, B); } -func_name_list(A) ::= func_name_list(B) NK_COMMA col_name(C). { A = addNodeToList(pCxt, B, C); } +func_name_list(A) ::= func_name_list(B) NK_COMMA func_name(C). { A = addNodeToList(pCxt, B, C); } func_name(A) ::= function_name(B). { A = createFunctionNode(pCxt, &B, NULL); } @@ -411,8 +414,8 @@ explain_options(A) ::= explain_options(B) RATIO NK_FLOAT(C). cmd ::= COMPACT VNODES IN NK_LP integer_list(A) NK_RP. { pCxt->pRootNode = createCompactStmt(pCxt, A); } /************************************************ create/drop function ************************************************/ -cmd ::= CREATE agg_func_opt(A) FUNCTION function_name(B) - AS NK_STRING(C) OUTPUTTYPE type_name(D) bufsize_opt(E). { pCxt->pRootNode = createCreateFunctionStmt(pCxt, A, &B, &C, D, E); } +cmd ::= CREATE agg_func_opt(A) FUNCTION not_exists_opt(F) function_name(B) + AS NK_STRING(C) OUTPUTTYPE type_name(D) bufsize_opt(E). { pCxt->pRootNode = createCreateFunctionStmt(pCxt, F, A, &B, &C, D, E); } cmd ::= DROP FUNCTION function_name(A). { pCxt->pRootNode = createDropFunctionStmt(pCxt, &A); } %type agg_func_opt { bool } @@ -834,6 +837,8 @@ query_expression(A) ::= query_expression_body(A) ::= query_primary(B). { A = B; } query_expression_body(A) ::= query_expression_body(B) UNION ALL query_expression_body(D). { A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); } +query_expression_body(A) ::= + query_expression_body(B) UNION query_expression_body(D). { A = createSetOperator(pCxt, SET_OP_TYPE_UNION, B, D); } query_primary(A) ::= query_specification(B). { A = B; } //query_primary(A) ::= diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 0bc28beef9b510a3e02de53dd1ffd8986b197f02..ea4e8cae50c0e667779323c83d153c9286b90506 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -48,6 +48,7 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) { } static void trimEscape(SToken* pName) { + // todo need to deal with `ioo``ii` -> ioo`ii if (NULL != pName && pName->n > 1 && '`' == pName->z[0]) { pName->z += 1; pName->n -= 2; @@ -677,9 +678,6 @@ SNode* setDatabaseAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOp case DB_OPTION_PRECISION: ((SDatabaseOptions*)pOptions)->pPrecision = pAlterOption->pVal; break; - case DB_OPTION_QUORUM: - ((SDatabaseOptions*)pOptions)->pQuorum = pAlterOption->pVal; - break; case DB_OPTION_REPLICA: ((SDatabaseOptions*)pOptions)->pReplica = pAlterOption->pVal; break; @@ -1184,10 +1182,21 @@ SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups) { return pStmt; } -SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize) { - SNode* pStmt = nodesMakeNode(QUERY_NODE_CREATE_FUNCTION_STMT); +SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, + bool ignoreExists, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize) { + if (pLibPath->n <= 2) { + pCxt->valid = false; + return NULL; + } + SCreateFunctionStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_FUNCTION_STMT); CHECK_OUT_OF_MEM(pStmt); - return pStmt; + pStmt->ignoreExists = ignoreExists; + strncpy(pStmt->funcName, pFuncName->z, pFuncName->n); + pStmt->isAgg = aggFunc; + strncpy(pStmt->libraryPath, pLibPath->z + 1, pLibPath->n - 2); + pStmt->outputDt = dataType; + pStmt->bufSize = bufSize; + return (SNode*)pStmt; } SNode* createDropFunctionStmt(SAstCreateContext* pCxt, const SToken* pFuncName) { diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 2eeff04ea2ff18ece6b34bc1609eb8a78d9ceb72..fcc15b197f238d268a4789903ace235eab5fc001 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "os.h" #include "parInt.h" #include "parAst.h" diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 1e8b732b6a4d52546ae52ad1e66a184232803a6e..25f978d98819c10b983b8c069d1793b42dc4ed77 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "os.h" #include "parInsertData.h" #include "parInt.h" #include "parToken.h" @@ -40,29 +41,21 @@ sToken = tStrGetToken(pSql, &index, false); \ } while (0) -#define CHECK_CODE(expr) \ - do { \ - int32_t code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - return code; \ - } \ - } while (0) - typedef struct SInsertParseContext { - SParseContext* pComCxt; // input - char* pSql; // input - SMsgBuf msg; // input - STableMeta* pTableMeta; // each table - SParsedDataColInfo tags; // each table - SKVRowBuilder tagsBuilder; // each table - SVCreateTbReq createTblReq; // each table - SHashObj* pVgroupsHashObj; // global - SHashObj* pTableBlockHashObj; // global - SHashObj* pSubTableHashObj; // global - SArray* pTableDataBlocks; // global - SArray* pVgDataBlocks; // global - int32_t totalNum; + SParseContext* pComCxt; // input + char *pSql; // input + SMsgBuf msg; // input + STableMeta* pTableMeta; // each table + SParsedDataColInfo tags; // each table + SKVRowBuilder tagsBuilder; // each table + SVCreateTbReq createTblReq; // each table + SHashObj* pVgroupsHashObj; // global + SHashObj* pTableBlockHashObj; // global + SHashObj* pSubTableHashObj; // global + SArray* pVgDataBlocks; // global + int32_t totalNum; SVnodeModifOpStmt* pOutput; + SStmtCallback* pStmtCb; } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); @@ -70,6 +63,29 @@ typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; +typedef struct SKvParam { + SKVRowBuilder *builder; + SSchema *schema; + char buf[TSDB_MAX_TAGS_LEN]; +} SKvParam; + +typedef struct SMemParam { + SRowBuilder* rb; + SSchema* schema; + int32_t toffset; + col_id_t colIdx; +} SMemParam; + + +#define CHECK_CODE(expr) \ + do { \ + int32_t code = expr; \ + if (TSDB_CODE_SUCCESS != code) { \ + return code; \ + } \ + } while (0) + + static int32_t skipInsertInto(SInsertParseContext* pCxt) { SToken sToken; NEXT_TOKEN(pCxt->pSql, sToken); @@ -163,7 +179,8 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD return TSDB_CODE_SUCCESS; } -static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { + +static int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf) { const char* msg1 = "name too long"; const char* msg2 = "invalid database name"; const char* msg3 = "db is not specified"; @@ -179,7 +196,7 @@ static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pPar strncpy(name, pTableName->z, dbLen); dbLen = strdequote(name); - code = tNameSetDbName(pName, pParseCtx->acctId, name, dbLen); + code = tNameSetDbName(pName, acctId, name, dbLen); if (code != TSDB_CODE_SUCCESS) { return buildInvalidOperationMsg(pMsgBuf, msg1); } @@ -204,11 +221,11 @@ static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pPar strncpy(name, pTableName->z, pTableName->n); strdequote(name); - if (pParseCtx->db == NULL) { + if (dbName == NULL) { return buildInvalidOperationMsg(pMsgBuf, msg3); } - code = tNameSetDbName(pName, pParseCtx->acctId, pParseCtx->db, strlen(pParseCtx->db)); + code = tNameSetDbName(pName, acctId, dbName, strlen(dbName)); if (code != TSDB_CODE_SUCCESS) { code = buildInvalidOperationMsg(pMsgBuf, msg2); return code; @@ -225,8 +242,8 @@ static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pPar static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SToken* pTname, bool isStb) { SParseContext* pBasicCtx = pCxt->pComCxt; - SName name = {0}; - createSName(&name, pTname, pBasicCtx, &pCxt->msg); + SName name = {0}; + createSName(&name, pTname, pBasicCtx->acctId, pBasicCtx->db, &pCxt->msg); if (isStb) { CHECK_CODE(catalogGetSTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); @@ -299,7 +316,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) { return TSDB_CODE_SUCCESS; } -static int32_t checkTimestamp(STableDataBlocks* pDataBlocks, const char* start) { +int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) { // once the data block is disordered, we do NOT keep previous timestamp any more if (!pDataBlocks->ordered) { return TSDB_CODE_SUCCESS; @@ -397,35 +414,15 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); } - if (IS_NUMERIC_TYPE(type) && pToken->n == 0) { - return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z); - } - // Remove quotation marks if (TK_NK_STRING == pToken->type) { if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); } - // delete escape character: \\, \', \" - char delim = pToken->z[0]; - int32_t cnt = 0; - int32_t j = 0; - for (uint32_t k = 1; k < pToken->n - 1; ++k) { - if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) { - tmpTokenBuf[j] = pToken->z[k + 1]; - cnt++; - j++; - k++; - continue; - } - tmpTokenBuf[j] = pToken->z[k]; - j++; - } - - tmpTokenBuf[j] = 0; + int32_t len = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW); pToken->z = tmpTokenBuf; - pToken->n -= 2 + cnt; + pToken->n = len; } return TSDB_CODE_SUCCESS; @@ -461,7 +458,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int if (isNullStr(pToken)) { if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { - return buildSyntaxErrMsg(pMsgBuf, "primary timestamp can not be null", pToken->z); + return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z); } return func(pMsgBuf, NULL, 0, param); @@ -603,6 +600,13 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return func(pMsgBuf, pToken->z, pToken->n, param); } + case TSDB_DATA_TYPE_JSON: { + if(pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z); + } + return func(pMsgBuf, pToken->z, pToken->n, param); + } + case TSDB_DATA_TYPE_TIMESTAMP: { int64_t tmpVal; if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) { @@ -616,13 +620,6 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return TSDB_CODE_FAILED; } -typedef struct SMemParam { - SRowBuilder* rb; - SSchema* schema; - int32_t toffset; - col_id_t colIdx; -} SMemParam; - static FORCE_INLINE int32_t MemRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param) { SMemParam* pa = (SMemParam*)param; SRowBuilder* rb = pa->rb; @@ -723,27 +720,32 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } - memset(&pColList->boundColumns[pColList->numOfBound], 0, - sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); + if(pColList->numOfCols > pColList->numOfBound){ + memset(&pColList->boundColumns[pColList->numOfBound], 0, + sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); + } return TSDB_CODE_SUCCESS; } -typedef struct SKvParam { - SKVRowBuilder* builder; - SSchema* schema; - char buf[TSDB_MAX_TAGS_LEN]; -} SKvParam; - -static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param) { - SKvParam* pa = (SKvParam*)param; +static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, void *param) { + SKvParam* pa = (SKvParam*) param; int8_t type = pa->schema->type; int16_t colId = pa->schema->colId; + if(TSDB_DATA_TYPE_JSON == type){ + return parseJsontoTagData(value, pa->builder, pMsgBuf, colId); + } + + if (value == NULL) { // it is a null data + // tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx); + return TSDB_CODE_SUCCESS; + } + if (TSDB_DATA_TYPE_BINARY == type) { STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len); - tdAddColToKVRow(pa->builder, colId, type, pa->buf); + tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf)); } else if (TSDB_DATA_TYPE_NCHAR == type) { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' int32_t output = 0; @@ -751,25 +753,24 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, voi char buf[512] = {0}; snprintf(buf, tListLen(buf), "%s", strerror(errno)); return buildSyntaxErrMsg(pMsgBuf, buf, value); - ; } varDataSetLen(pa->buf, output); - tdAddColToKVRow(pa->builder, colId, type, pa->buf); + tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf)); } else { - tdAddColToKVRow(pa->builder, colId, type, value); + tdAddColToKVRow(pa->builder, colId, value, TYPE_BYTES[type]); } return TSDB_CODE_SUCCESS; } -static int32_t buildCreateTbReq(SInsertParseContext* pCxt, const SName* pName, SKVRow row) { +static int32_t buildCreateTbReq(SVCreateTbReq *pTbReq, const SName* pName, SKVRow row, int64_t suid) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pName, dbFName); - pCxt->createTblReq.type = TD_CHILD_TABLE; - pCxt->createTblReq.name = strdup(pName->tname); - pCxt->createTblReq.ctbCfg.suid = pCxt->pTableMeta->suid; - pCxt->createTblReq.ctbCfg.pTag = row; + pTbReq->type = TD_CHILD_TABLE; + pTbReq->name = strdup(pName->tname); + pTbReq->ctbCfg.suid = suid; + pTbReq->ctbCfg.pTag = row; return TSDB_CODE_SUCCESS; } @@ -781,23 +782,42 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } SKvParam param = {.builder = &pCxt->tagsBuilder}; - SToken sToken; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" + SToken sToken; + bool isParseBindParam = false; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i] - 1]; // colId starts with 1 + + if (sToken.type == TK_NK_QUESTION) { + isParseBindParam = true; + if (NULL == pCxt->pStmtCb) { + return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); + } + + continue; + } + + if (isParseBindParam) { + return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values"); + } + + SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i] - 1]; // colId starts with 1 param.schema = pTagSchema; CHECK_CODE( parseValueToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); } + if (isParseBindParam) { + return TSDB_CODE_SUCCESS; + } + SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder); if (NULL == row) { return buildInvalidOperationMsg(&pCxt->msg, "tag value expected"); } tdSortKVRowByColIdx(row); - return buildCreateTbReq(pCxt, pName, row); + return buildCreateTbReq(&pCxt->createTblReq, pName, row, pCxt->pTableMeta->suid); } static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { @@ -821,7 +841,7 @@ static int32_t storeTableMeta(SHashObj* pHash, const char* pName, int32_t len, S // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) { SName name; - createSName(&name, pTbnameToken, pCxt->pComCxt, &pCxt->msg); + createSName(&name, pTbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&name, tbFName); int32_t len = strlen(tbFName); @@ -866,8 +886,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) return TSDB_CODE_SUCCESS; } -static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, int32_t* len, - char* tmpTokenBuf) { +static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, bool* gotRow, char* tmpTokenBuf) { SParsedDataColInfo* spd = &pDataBlocks->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlocks->rowBuilder; STSRow* row = (STSRow*)(pDataBlocks->pData + pDataBlocks->size); // skip the SSubmitBlk header @@ -882,16 +901,27 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, for (int i = 0; i < spd->numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); SSchema* pSchema = &schema[spd->boundColumns[i] - 1]; + + if (sToken.type == TK_NK_QUESTION) { + isParseBindParam = true; + if (NULL == pCxt->pStmtCb) { + return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); + } + + continue; + } + + if (isParseBindParam) { + return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and values"); + } + param.schema = pSchema; - getSTSRowAppendInfo(schema, pBuilder->rowType, spd, i, ¶m.toffset, ¶m.colIdx); + getSTSRowAppendInfo(pBuilder->rowType, spd, i, ¶m.toffset, ¶m.colIdx); CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, timePrec, tmpTokenBuf, MemRowAppend, ¶m, &pCxt->msg)); if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { TSKEY tsKey = TD_ROW_KEY(row); - if (checkTimestamp(pDataBlocks, (const char*)&tsKey) != TSDB_CODE_SUCCESS) { - buildSyntaxErrMsg(&pCxt->msg, "client time/server time can not be mixed up", sToken.z); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } + checkTimestamp(pDataBlocks, (const char*)&tsKey); } } @@ -905,6 +935,8 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, } } } + + *gotRow = true; } // *len = pBuilder->extendedRowSize; @@ -935,20 +967,24 @@ static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlo maxRows = tSize; } - int32_t len = 0; - CHECK_CODE(parseOneRow(pCxt, pDataBlock, tinfo.precision, &len, tmpTokenBuf)); - pDataBlock->size += extendedRowSize; // len; + bool gotRow = false; + CHECK_CODE(parseOneRow(pCxt, pDataBlock, tinfo.precision, &gotRow, tmpTokenBuf)); + if (gotRow) { + pDataBlock->size += extendedRowSize; //len; + } NEXT_TOKEN(pCxt->pSql, sToken); if (TK_NK_RP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, ") expected", sToken.z); } - (*numOfRows)++; + if (gotRow) { + (*numOfRows)++; + } } - if (0 == (*numOfRows)) { - return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); + if (0 == (*numOfRows) && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { + return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); } return TSDB_CODE_SUCCESS; } @@ -970,7 +1006,7 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da return TSDB_CODE_SUCCESS; } -static void destroyCreateSubTbReq(SVCreateTbReq* pReq) { +void destroyCreateSubTbReq(SVCreateTbReq* pReq) { taosMemoryFreeClear(pReq->name); taosMemoryFreeClear(pReq->ctbCfg.pTag); } @@ -1004,7 +1040,6 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { taosHashCleanup(pCxt->pVgroupsHashObj); destroyBlockHashmap(pCxt->pTableBlockHashObj); - destroyBlockArrayList(pCxt->pTableDataBlocks); destroyBlockArrayList(pCxt->pVgDataBlocks); } @@ -1014,23 +1049,41 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; static int32_t parseInsertBody(SInsertParseContext* pCxt) { + int32_t tbNum = 0; + // for each table while (1) { - destroyInsertParseContextForTable(pCxt); - SToken sToken; + char *tbName = NULL; + // pSql -> tb_name ... NEXT_TOKEN(pCxt->pSql, sToken); // no data in the sql string anymore. if (sToken.n == 0) { - if (0 == pCxt->totalNum) { + if (0 == pCxt->totalNum && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { return buildInvalidOperationMsg(&pCxt->msg, "no data in sql"); - ; } break; } + if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && tbNum > 0) { + return buildInvalidOperationMsg(&pCxt->msg, "single table allowed in one stmt");; + } + + destroyInsertParseContextForTable(pCxt); + + if (TK_NK_QUESTION == sToken.type) { + if (pCxt->pStmtCb) { + CHECK_CODE((*pCxt->pStmtCb->getTbNameFn)(pCxt->pStmtCb->pStmt, &tbName)); + + sToken.z = tbName; + sToken.n = strlen(tbName); + } else { + return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); + } + } + SToken tbnameToken = sToken; NEXT_TOKEN(pCxt->pSql, sToken); @@ -1056,7 +1109,9 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (TK_VALUES == sToken.type) { // pSql -> (field1_value, ...) [(field1_value2, ...) ...] CHECK_CODE(parseValuesClause(pCxt, dataBuf)); - pCxt->pOutput->insertType = TSDB_QUERY_TYPE_INSERT; + TSDB_QUERY_SET_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_INSERT); + + tbNum++; continue; } @@ -1069,14 +1124,32 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { } // todo pCxt->pOutput->insertType = TSDB_QUERY_TYPE_FILE_INSERT; + + tbNum++; continue; } return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); } + + if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { + SParsedDataColInfo *tags = taosMemoryMalloc(sizeof(pCxt->tags)); + if (NULL == tags) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); + (*pCxt->pStmtCb->setBindInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags); + memset(&pCxt->tags, 0, sizeof(pCxt->tags)); + + (*pCxt->pStmtCb->setExecInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj); + pCxt->pVgroupsHashObj = NULL; + pCxt->pTableBlockHashObj = NULL; + + return TSDB_CODE_SUCCESS; + } + // merge according to vgId - if (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && - taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { + if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); } return buildOutput(pCxt); @@ -1090,29 +1163,43 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { // [...]; int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { SInsertParseContext context = { - .pComCxt = pContext, - .pSql = (char*)pContext->pSql, - .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, - .pTableMeta = NULL, - .pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false), - .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), - .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false), - .totalNum = 0, - .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT)}; - - if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pSubTableHashObj || - NULL == context.pOutput) { + .pComCxt = pContext, + .pSql = (char*) pContext->pSql, + .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, + .pTableMeta = NULL, + .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false), + .totalNum = 0, + .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT), + .pStmtCb = pContext->pStmtCb + }; + + if (pContext->pStmtCb && *pQuery) { + (*pContext->pStmtCb->getExecInfoFn)(pContext->pStmtCb->pStmt, &context.pVgroupsHashObj, &context.pTableBlockHashObj); + } else { + context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + context.pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + } + + if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || + NULL == context.pSubTableHashObj || NULL == context.pOutput) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - *pQuery = taosMemoryCalloc(1, sizeof(SQuery)); + if (pContext->pStmtCb) { + TSDB_QUERY_SET_TYPE(context.pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT); + } + if (NULL == *pQuery) { - return TSDB_CODE_OUT_OF_MEMORY; + *pQuery = taosMemoryCalloc(1, sizeof(SQuery)); + if (NULL == *pQuery) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE; + (*pQuery)->haveResultSet = false; + (*pQuery)->msgType = TDMT_VND_SUBMIT; + (*pQuery)->pRoot = (SNode*)context.pOutput; } - (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE; - (*pQuery)->haveResultSet = false; - (*pQuery)->msgType = TDMT_VND_SUBMIT; - (*pQuery)->pRoot = (SNode*)context.pOutput; + context.pOutput->payloadType = PAYLOAD_TYPE_KV; int32_t code = skipInsertInto(&context); @@ -1122,3 +1209,317 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { destroyInsertParseContext(&context); return code; } + + +int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen) { + SMsgBuf msg = {.buf = msgBuf, .len =msgBufLen}; + SToken sToken; + int32_t code = 0; + char *tbName = NULL; + + NEXT_TOKEN(pTableName, sToken); + + if (sToken.n == 0) { + return buildInvalidOperationMsg(&msg, "empty table name"); + } + + code = createSName(pName, &sToken, acctId, dbName, &msg); + if (code) { + return code; + } + + NEXT_TOKEN(pTableName, sToken); + + if (sToken.n > 0) { + return buildInvalidOperationMsg(&msg, "table name format is wrong"); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) { + SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pQuery->pRoot; + int32_t code = 0; + SInsertParseContext insertCtx = { + .pVgroupsHashObj = pVgHash, + .pTableBlockHashObj = pBlockHash, + .pOutput = (SVnodeModifOpStmt*)pQuery->pRoot, + }; + + // merge according to vgId + if (taosHashGetSize(insertCtx.pTableBlockHashObj) > 0) { + CHECK_CODE(mergeTableDataBlocks(insertCtx.pTableBlockHashObj, modifyNode->payloadType, &insertCtx.pVgDataBlocks)); + } + + CHECK_CODE(buildOutput(&insertCtx)); + + return TSDB_CODE_SUCCESS; +} + +int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen){ + STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + if (NULL == tags) { + return TSDB_CODE_QRY_APP_ERROR; + } + + SKVRowBuilder tagBuilder; + if (tdInitKVRowBuilder(&tagBuilder) < 0) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + SKvParam param = {.builder = &tagBuilder}; + + for (int c = 0; c < tags->numOfBound; ++c) { + if (bind[c].is_null && bind[c].is_null[0]) { + KvRowAppend(&pBuf, NULL, 0, ¶m); + continue; + } + + SSchema* pTagSchema = &pSchema[tags->boundColumns[c] - 1]; // colId starts with 1 + param.schema = pTagSchema; + + int32_t colLen = pTagSchema->bytes; + if (IS_VAR_DATA_TYPE(pTagSchema->type)) { + colLen = bind[c].length[0]; + } + + CHECK_CODE(KvRowAppend(&pBuf, (char *)bind[c].buffer, colLen, ¶m)); + } + + SKVRow row = tdGetKVRowFromBuilder(&tagBuilder); + if (NULL == row) { + tdDestroyKVRowBuilder(&tagBuilder); + return buildInvalidOperationMsg(&pBuf, "tag value expected"); + } + tdSortKVRowByColIdx(row); + + SVCreateTbReq tbReq = {0}; + CHECK_CODE(buildCreateTbReq(&tbReq, pName, row, suid)); + CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq)); + + destroyCreateSubTbReq(&tbReq); + tdDestroyKVRowBuilder(&tagBuilder); + + return TSDB_CODE_SUCCESS; +} + + +int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen) { + STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; + SRowBuilder* pBuilder = &pDataBlock->rowBuilder; + SMemParam param = {.rb = pBuilder}; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + int32_t rowNum = bind->num; + + CHECK_CODE(initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); + + CHECK_CODE(allocateMemForSize(pDataBlock, extendedRowSize * bind->num)); + + for (int32_t r = 0; r < bind->num; ++r) { + STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header + tdSRowResetBuf(pBuilder, row); + + for (int c = 0; c < spd->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1]; + + if (bind[c].buffer_type != pColSchema->type) { + return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); + } + + if (bind[c].num != rowNum) { + return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + } + + param.schema = pColSchema; + getSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); + + if (bind[c].is_null && bind[c].is_null[r]) { + if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL"); + } + + CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, ¶m)); + } else { + int32_t colLen = pColSchema->bytes; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + colLen = bind[c].length[r]; + } + + CHECK_CODE(MemRowAppend(&pBuf, (char *)bind[c].buffer + bind[c].buffer_length * r, colLen, ¶m)); + } + + if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { + TSKEY tsKey = TD_ROW_KEY(row); + checkTimestamp(pDataBlock, (const char *)&tsKey); + } + } + + // set the null value for the columns that do not assign values + if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { + for (int32_t i = 0; i < spd->numOfCols; ++i) { + if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE + tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, + spd->cols[i].toffset); + } + } + } + + pDataBlock->size += extendedRowSize; + } + + SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData); + if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, bind->num)) { + return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767"); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum) { + STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; + SRowBuilder* pBuilder = &pDataBlock->rowBuilder; + SMemParam param = {.rb = pBuilder}; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + bool rowStart = (0 == colIdx); + bool rowEnd = ((colIdx + 1) == spd->numOfBound); + + if (rowStart) { + CHECK_CODE(initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); + CHECK_CODE(allocateMemForSize(pDataBlock, extendedRowSize * bind->num)); + } + + for (int32_t r = 0; r < bind->num; ++r) { + STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size + extendedRowSize * r); // skip the SSubmitBlk header + if (rowStart) { + tdSRowResetBuf(pBuilder, row); + } else { + tdSRowGetBuf(pBuilder, row); + } + + SSchema* pColSchema = &pSchema[spd->boundColumns[colIdx] - 1]; + + if (bind->num != rowNum) { + return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + } + + param.schema = pColSchema; + getSTSRowAppendInfo(pBuilder->rowType, spd, colIdx, ¶m.toffset, ¶m.colIdx); + + if (bind->is_null && bind->is_null[r]) { + if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL"); + } + + CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, ¶m)); + } else { + if (bind->buffer_type != pColSchema->type) { + return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); + } + + int32_t colLen = pColSchema->bytes; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + colLen = bind->length[r]; + } + + CHECK_CODE(MemRowAppend(&pBuf, (char *)bind->buffer + bind->buffer_length * r, colLen, ¶m)); + } + + if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { + TSKEY tsKey = TD_ROW_KEY(row); + checkTimestamp(pDataBlock, (const char *)&tsKey); + } + + // set the null value for the columns that do not assign values + if (rowEnd && (spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { + for (int32_t i = 0; i < spd->numOfCols; ++i) { + if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE + tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, + spd->cols[i].toffset); + } + } + } + } + + if (rowEnd) { + pDataBlock->size += extendedRowSize * bind->num; + + SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData); + if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, bind->num)) { + return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767"); + } + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t buildBoundFields(SParsedDataColInfo *boundInfo, SSchema *pSchema, int32_t *fieldNum, TAOS_FIELD** fields) { + if (fields) { + *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD)); + if (NULL == *fields) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < boundInfo->numOfBound; ++i) { + SSchema* pTagSchema = &pSchema[boundInfo->boundColumns[i] - 1]; + strcpy((*fields)[i].name, pTagSchema->name); + (*fields)[i].type = pTagSchema->type; + (*fields)[i].bytes = pTagSchema->bytes; + } + } + + *fieldNum = boundInfo->numOfBound; + + return TSDB_CODE_SUCCESS; +} + + +int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields) { + STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; + SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + if (NULL == tags) { + return TSDB_CODE_QRY_APP_ERROR; + } + + SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + if (tags->numOfBound <= 0) { + *fieldNum = 0; + *fields = NULL; + + return TSDB_CODE_SUCCESS; + } + + CHECK_CODE(buildBoundFields(tags, pSchema, fieldNum, fields)); + + return TSDB_CODE_SUCCESS; +} + +int32_t qBuildStmtColFields(void *pBlock, int32_t *fieldNum, TAOS_FIELD** fields) { + STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + if (pDataBlock->boundColumnInfo.numOfBound <= 0) { + *fieldNum = 0; + if (fields) { + *fields = NULL; + } + + return TSDB_CODE_SUCCESS; + } + + CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields)); + + return TSDB_CODE_SUCCESS; +} + + + diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 088b25d5448665de9d84ae94c023ee8fef41e711..bf30915fcbdbfff64abee067f419f887d6018e16 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -18,6 +18,7 @@ #include "catalog.h" #include "parUtil.h" #include "querynodes.h" +#include "parInt.h" #define IS_RAW_PAYLOAD(t) \ (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert @@ -102,7 +103,13 @@ int32_t boundIdxCompar(const void *lhs, const void *rhs) { } } -void destroyBoundColumnInfo(SParsedDataColInfo* pColList) { +void destroyBoundColumnInfo(void* pBoundInfo) { + if (NULL == pBoundInfo) { + return; + } + + SParsedDataColInfo* pColList = (SParsedDataColInfo*)pBoundInfo; + taosMemoryFreeClear(pColList->boundColumns); taosMemoryFreeClear(pColList->cols); taosMemoryFreeClear(pColList->colIdxInfo); @@ -149,7 +156,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star return TSDB_CODE_SUCCESS; } -static int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { +int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { int32_t len = tSerializeSVCreateTbReq(NULL, pCreateTbReq); if (pBlocks->nAllocSize - pBlocks->size < len) { pBlocks->nAllocSize += len + pBlocks->rowSize; @@ -506,6 +513,28 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p return TSDB_CODE_SUCCESS; } +int32_t allocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize) { + size_t remain = pDataBlock->nAllocSize - pDataBlock->size; + uint32_t nAllocSizeOld = pDataBlock->nAllocSize; + + // expand the allocated size + if (remain < allSize) { + pDataBlock->nAllocSize = (pDataBlock->size + allSize) * 1.5; + + char *tmp = taosMemoryRealloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); + if (tmp != NULL) { + pDataBlock->pData = tmp; + memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); + } else { + // do nothing, if allocate more memory failed + pDataBlock->nAllocSize = nAllocSizeOld; + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows) { size_t remain = pDataBlock->nAllocSize - pDataBlock->size; const int factor = 5; @@ -541,3 +570,84 @@ int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo pColInfo->boundNullLen); return TSDB_CODE_SUCCESS; } + + +int32_t qResetStmtDataBlock(void* block, bool keepBuf) { + STableDataBlocks* pBlock = (STableDataBlocks*)block; + + if (keepBuf) { + taosMemoryFreeClear(pBlock->pData); + pBlock->pData = taosMemoryMalloc(TSDB_PAYLOAD_SIZE); + if (NULL == pBlock->pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memset(pBlock->pData, 0, sizeof(SSubmitBlk)); + } else { + pBlock->pData = NULL; + } + + pBlock->ordered = true; + pBlock->prevTS = INT64_MIN; + pBlock->size = sizeof(SSubmitBlk); + pBlock->tsSource = -1; + pBlock->numOfTables = 1; + pBlock->nAllocSize = TSDB_PAYLOAD_SIZE; + pBlock->headerSize = pBlock->size; + + memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder)); + + return TSDB_CODE_SUCCESS; +} + + +int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) { + *pDst = taosMemoryMalloc(sizeof(STableDataBlocks)); + if (NULL == *pDst) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + memcpy(*pDst, pSrc, sizeof(STableDataBlocks)); + ((STableDataBlocks*)(*pDst))->cloned = true; + + return qResetStmtDataBlock(*pDst, false); +} + +int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc) { + int32_t code = qCloneStmtDataBlock(pDst, pSrc); + if (code) { + return code; + } + + STableDataBlocks *pBlock = (STableDataBlocks*)*pDst; + pBlock->pData = taosMemoryMalloc(pBlock->nAllocSize); + if (NULL == pBlock->pData) { + qFreeStmtDataBlock(pBlock); + return TSDB_CODE_OUT_OF_MEMORY; + } + + memset(pBlock->pData, 0, sizeof(SSubmitBlk)); + + return TSDB_CODE_SUCCESS; +} + + +void qFreeStmtDataBlock(void* pDataBlock) { + if (pDataBlock == NULL) { + return; + } + + taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pData); + taosMemoryFreeClear(pDataBlock); +} + +void qDestroyStmtDataBlock(void* pBlock) { + if (pBlock == NULL) { + return; + } + + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + + pDataBlock->cloned = false; + destroyDataBlock(pDataBlock); +} + diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 2a191b7bf2ec51438f2a8385d125539d2806599a..666cdbb6db804c615c1e73b19cc633a36ba73310 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -52,6 +52,7 @@ static SKeyword keywordTable[] = { {"CACHE", TK_CACHE}, {"CACHELAST", TK_CACHELAST}, {"CAST", TK_CAST}, + {"CLUSTER", TK_CLUSTER}, {"COLUMN", TK_COLUMN}, {"COMMENT", TK_COMMENT}, {"COMP", TK_COMP}, @@ -168,6 +169,7 @@ static SKeyword keywordTable[] = { {"STREAM", TK_STREAM}, {"STREAMS", TK_STREAMS}, {"STREAM_MODE", TK_STREAM_MODE}, + {"STRICT", TK_STRICT}, {"SYNCDB", TK_SYNCDB}, {"TABLE", TK_TABLE}, {"TABLES", TK_TABLES}, @@ -247,7 +249,6 @@ static SKeyword keywordTable[] = { // {"BEFORE", TK_BEFORE}, // {"BEGIN", TK_BEGIN}, // {"CASCADE", TK_CASCADE}, - // {"CLUSTER", TK_CLUSTER}, // {"CONFLICT", TK_CONFLICT}, // {"COPY", TK_COPY}, // {"DEFERRED", TK_DEFERRED}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 30fc0c69444354122eed3b46ca96003b2b532f67..2aedc37c24cb6a266bb6388ebf03f69179f8feb5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -49,12 +49,14 @@ static bool afterGroupBy(ESqlClause clause) { return clause > SQL_CLAUSE_GROUP_B static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; } -#define generateDealNodeErrMsg(pCxt, code, ...) \ - ({ \ - generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__); \ - pCxt->errCode = code; \ - DEAL_RES_ERROR; \ - }) +enum EDealRes generateDealNodeErrMsg(STranslateContext* pCxt, int32_t code, ...) { + va_list ap; + va_start(ap, code); + generateSyntaxErrMsg(&pCxt->msgBuf, code, ap); + va_end(ap); + pCxt->errCode = code; + return DEAL_RES_ERROR; +} static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel); @@ -198,6 +200,71 @@ static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int return code; } +static int32_t getDBCfg(STranslateContext* pCxt, const char* pDbName, SDbCfgInfo* pInfo) { + SParseContext* pParCxt = pCxt->pParseCxt; + SName name; + tNameSetDbName(&name, pCxt->pParseCxt->acctId, pDbName, strlen(pDbName)); + char dbFname[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, dbFname); + int32_t code = collectUseDatabaseImpl(dbFname, pCxt->pDbs); + if (TSDB_CODE_SUCCESS == code) { + code = catalogGetDBCfg(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, dbFname, pInfo); + } + if (TSDB_CODE_SUCCESS != code) { + parserError("catalogGetDBCfg error, code:%s, dbFName:%s", tstrerror(code), dbFname); + } + return code; +} + +static int32_t initTranslateContext(SParseContext* pParseCxt, STranslateContext* pCxt) { + pCxt->pParseCxt = pParseCxt; + pCxt->errCode = TSDB_CODE_SUCCESS; + pCxt->msgBuf.buf = pParseCxt->pMsg; + pCxt->msgBuf.len = pParseCxt->msgLen; + pCxt->pNsLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + pCxt->currLevel = 0; + pCxt->currClause = 0; + pCxt->pDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pCxt->pTables = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (NULL == pCxt->pNsLevel || NULL == pCxt->pDbs || NULL == pCxt->pTables) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t resetTranslateNamespace(STranslateContext* pCxt) { + if (NULL != pCxt->pNsLevel) { + size_t size = taosArrayGetSize(pCxt->pNsLevel); + for (size_t i = 0; i < size; ++i) { + taosArrayDestroy(taosArrayGetP(pCxt->pNsLevel, i)); + } + taosArrayDestroy(pCxt->pNsLevel); + } + pCxt->pNsLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + if (NULL == pCxt->pNsLevel) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return TSDB_CODE_SUCCESS; +} + +static void destroyTranslateContext(STranslateContext* pCxt) { + if (NULL != pCxt->pNsLevel) { + size_t size = taosArrayGetSize(pCxt->pNsLevel); + for (size_t i = 0; i < size; ++i) { + taosArrayDestroy(taosArrayGetP(pCxt->pNsLevel, i)); + } + taosArrayDestroy(pCxt->pNsLevel); + } + + if (NULL != pCxt->pCmdMsg) { + taosMemoryFreeClear(pCxt->pCmdMsg->pMsg); + taosMemoryFreeClear(pCxt->pCmdMsg); + } + + taosHashCleanup(pCxt->pDbs); + taosHashCleanup(pCxt->pTables); +} + static bool belongTable(const char* currentDb, const SColumnNode* pCol, const STableNode* pTable) { int cmp = 0; if ('\0' != pCol->dbName[0]) { @@ -211,9 +278,11 @@ static bool belongTable(const char* currentDb, const SColumnNode* pCol, const ST return (0 == cmp); } -static SNodeList* getProjectList(SNode* pNode) { +static SNodeList* getProjectList(const SNode* pNode) { if (QUERY_NODE_SELECT_STMT == nodeType(pNode)) { return ((SSelectStmt*)pNode)->pProjectionList; + } else if (QUERY_NODE_SET_OPERATOR == nodeType(pNode)) { + return ((SSetOperator*)pNode)->pProjectionList; } return NULL; } @@ -493,14 +562,21 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; } } else if (nodesIsComparisonOp(pOp)) { - if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || + if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } + if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) { + ((SExprNode*)pOp->pRight)->resType = ((SExprNode*)pOp->pLeft)->resType; + } pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - } else { - // todo json operator + } else if (nodesIsJsonOp(pOp)){ + if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + } + pOp->node.resType.type = TSDB_DATA_TYPE_JSON; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes; } return DEAL_RES_CONTINUE; } @@ -514,7 +590,8 @@ static EDealRes haveAggFunction(SNode* pNode, void* pContext) { } static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { - if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { + SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet}; + if (TSDB_CODE_SUCCESS != fmGetFuncInfo(¶m, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName); } pCxt->errCode = fmGetFuncResultType(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len); @@ -749,15 +826,18 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { case QUERY_NODE_REAL_TABLE: { SRealTableNode* pRealTable = (SRealTableNode*)pTable; pRealTable->ratio = (NULL != pCxt->pExplainOpt ? pCxt->pExplainOpt->ratio : 1.0); - SName name; - code = getTableMetaImpl( - pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name), - &(pRealTable->pMeta)); - if (TSDB_CODE_SUCCESS != code) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pRealTable->table.tableName); + // The SRealTableNode created through ROLLUP already has STableMeta. + if (NULL == pRealTable->pMeta) { + SName name; + code = getTableMetaImpl( + pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name), + &(pRealTable->pMeta)); + if (TSDB_CODE_SUCCESS != code) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pRealTable->table.tableName); + } + code = setTableVgroupList(pCxt, &name, pRealTable); } pRealTable->table.precision = pRealTable->pMeta->tableInfo.precision; - code = setTableVgroupList(pCxt, &name, pRealTable); if (TSDB_CODE_SUCCESS == code) { code = addNamespace(pCxt, pRealTable); } @@ -1296,6 +1376,86 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { return code; } +static SNode* createSetOperProject(SNode* pNode) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return NULL; + } + pCol->node.resType = ((SExprNode*)pNode)->resType; + strcpy(pCol->colName, ((SExprNode*)pNode)->aliasName); + strcpy(pCol->node.aliasName, pCol->colName); + return (SNode*)pCol; +} + +static bool dataTypeEqual(const SDataType* l, const SDataType* r) { + return (l->type == r->type && l->bytes == l->bytes && l->precision == r->precision && l->scale == l->scale); +} + +static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType dt, SNode** pCast) { + SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pFunc->functionName, "cast"); + pFunc->node.resType = dt; + if (TSDB_CODE_SUCCESS != nodesListMakeAppend(&pFunc->pParameterList, pExpr)) { + nodesDestroyNode(pFunc); + return TSDB_CODE_OUT_OF_MEMORY; + } + if (DEAL_RES_ERROR == translateFunction(pCxt, pFunc)) { + nodesClearList(pFunc->pParameterList); + pFunc->pParameterList = NULL; + nodesDestroyNode(pFunc); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pExpr)->aliasName); + } + *pCast = (SNode*)pFunc; + return TSDB_CODE_SUCCESS; +} + +static int32_t translateSetOperatorImpl(STranslateContext* pCxt, SSetOperator* pSetOperator) { + SNodeList* pLeftProjections = getProjectList(pSetOperator->pLeft); + SNodeList* pRightProjections = getProjectList(pSetOperator->pRight); + if (LIST_LENGTH(pLeftProjections) != LIST_LENGTH(pRightProjections)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCORRECT_NUM_OF_COL); + } + + SNode* pLeft = NULL; + SNode* pRight = NULL; + FORBOTH(pLeft, pLeftProjections, pRight, pRightProjections) { + SExprNode* pLeftExpr = (SExprNode*)pLeft; + SExprNode* pRightExpr = (SExprNode*)pRight; + if (!dataTypeEqual(&pLeftExpr->resType, &pRightExpr->resType)) { + SNode* pRightFunc = NULL; + int32_t code = createCastFunc(pCxt, pRight, pLeftExpr->resType, &pRightFunc); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + REPLACE_LIST2_NODE(pRightFunc); + pRightExpr = (SExprNode*)pRightFunc; + } + strcpy(pRightExpr->aliasName, pLeftExpr->aliasName); + pRightExpr->aliasName[strlen(pLeftExpr->aliasName)] = '\0'; + if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, createSetOperProject(pLeft))) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateSetOperator(STranslateContext* pCxt, SSetOperator* pSetOperator) { + int32_t code = translateQuery(pCxt, pSetOperator->pLeft); + if (TSDB_CODE_SUCCESS == code) { + code = resetTranslateNamespace(pCxt); + } + if (TSDB_CODE_SUCCESS == code) { + code = translateQuery(pCxt, pSetOperator->pRight); + } + if (TSDB_CODE_SUCCESS == code) { + code = translateSetOperatorImpl(pCxt, pSetOperator); + } + return code; +} + static int64_t getUnitPerMinute(uint8_t precision) { switch (precision) { case TSDB_TIME_PRECISION_MILLI: @@ -1350,21 +1510,22 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS pReq->daysToKeep0 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 0), TSDB_DEFAULT_KEEP); pReq->daysToKeep1 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 1), TSDB_DEFAULT_KEEP); pReq->daysToKeep2 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 2), TSDB_DEFAULT_KEEP); - pReq->minRows = GET_OPTION_VAL(pStmt->pOptions->pMinRowsPerBlock, TSDB_DEFAULT_MIN_ROW_FBLOCK); - pReq->maxRows = GET_OPTION_VAL(pStmt->pOptions->pMaxRowsPerBlock, TSDB_DEFAULT_MAX_ROW_FBLOCK); + pReq->minRows = GET_OPTION_VAL(pStmt->pOptions->pMinRowsPerBlock, TSDB_DEFAULT_MINROWS_FBLOCK); + pReq->maxRows = GET_OPTION_VAL(pStmt->pOptions->pMaxRowsPerBlock, TSDB_DEFAULT_MAXROWS_FBLOCK); pReq->commitTime = -1; pReq->fsyncPeriod = GET_OPTION_VAL(pStmt->pOptions->pFsyncPeriod, TSDB_DEFAULT_FSYNC_PERIOD); pReq->walLevel = GET_OPTION_VAL(pStmt->pOptions->pWalLevel, TSDB_DEFAULT_WAL_LEVEL); pReq->precision = GET_OPTION_VAL(pStmt->pOptions->pPrecision, TSDB_TIME_PRECISION_MILLI); pReq->compression = GET_OPTION_VAL(pStmt->pOptions->pCompressionLevel, TSDB_DEFAULT_COMP_LEVEL); - pReq->replications = GET_OPTION_VAL(pStmt->pOptions->pReplica, TSDB_DEFAULT_DB_REPLICA_OPTION); - pReq->quorum = GET_OPTION_VAL(pStmt->pOptions->pQuorum, TSDB_DEFAULT_DB_QUORUM_OPTION); + pReq->replications = GET_OPTION_VAL(pStmt->pOptions->pReplica, TSDB_DEFAULT_DB_REPLICA); pReq->update = -1; pReq->cacheLastRow = GET_OPTION_VAL(pStmt->pOptions->pCachelast, TSDB_DEFAULT_CACHE_LAST_ROW); pReq->ignoreExist = pStmt->ignoreExists; - pReq->streamMode = GET_OPTION_VAL(pStmt->pOptions->pStreamMode, TSDB_DEFAULT_DB_STREAM_MODE_OPTION); - pReq->ttl = GET_OPTION_VAL(pStmt->pOptions->pTtl, TSDB_DEFAULT_DB_TTL_OPTION); - pReq->singleSTable = GET_OPTION_VAL(pStmt->pOptions->pSingleStable, TSDB_DEFAULT_DB_SINGLE_STABLE_OPTION); + pReq->streamMode = GET_OPTION_VAL(pStmt->pOptions->pStreamMode, TSDB_DEFAULT_DB_STREAM_MODE); + pReq->ttl = GET_OPTION_VAL(pStmt->pOptions->pTtl, TSDB_DEFAULT_DB_TTL); + pReq->singleSTable = GET_OPTION_VAL(pStmt->pOptions->pSingleStable, TSDB_DEFAULT_DB_SINGLE_STABLE); + pReq->strict = GET_OPTION_VAL(pStmt->pOptions->pStrict, TSDB_DEFAULT_DB_STRICT); + return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq); } @@ -1431,8 +1592,8 @@ static int32_t checkTtlOption(STranslateContext* pCxt, SValueNode* pVal) { return pCxt->errCode; } int64_t val = pVal->datum.i; - if (val < TSDB_MIN_DB_TTL_OPTION) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_TTL_OPTION, val, TSDB_MIN_DB_TTL_OPTION); + if (val < TSDB_MIN_DB_TTL) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_TTL_OPTION, val, TSDB_MIN_DB_TTL); } } return TSDB_CODE_SUCCESS; @@ -1539,12 +1700,12 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, SDatabaseOptions* p code = checkRangeOption(pCxt, "fsyncPeriod", pOptions->pFsyncPeriod, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD); } if (TSDB_CODE_SUCCESS == code) { - code = checkRangeOption(pCxt, "maxRowsPerBlock", pOptions->pMaxRowsPerBlock, TSDB_MIN_MAX_ROW_FBLOCK, - TSDB_MAX_MAX_ROW_FBLOCK); + code = checkRangeOption(pCxt, "maxRowsPerBlock", pOptions->pMaxRowsPerBlock, TSDB_MIN_MAXROWS_FBLOCK, + TSDB_MAX_MAXROWS_FBLOCK); } if (TSDB_CODE_SUCCESS == code) { - code = checkRangeOption(pCxt, "minRowsPerBlock", pOptions->pMinRowsPerBlock, TSDB_MIN_MIN_ROW_FBLOCK, - TSDB_MAX_MIN_ROW_FBLOCK); + code = checkRangeOption(pCxt, "minRowsPerBlock", pOptions->pMinRowsPerBlock, TSDB_MIN_MINROWS_FBLOCK, + TSDB_MAX_MINROWS_FBLOCK); } if (TSDB_CODE_SUCCESS == code) { code = checkKeepOption(pCxt, pOptions->pKeep); @@ -1553,11 +1714,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, SDatabaseOptions* p code = checkDbPrecisionOption(pCxt, pOptions->pPrecision); } if (TSDB_CODE_SUCCESS == code) { - code = checkRangeOption(pCxt, "quorum", pOptions->pQuorum, TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION); - } - if (TSDB_CODE_SUCCESS == code) { - code = checkDbEnumOption(pCxt, "replications", pOptions->pReplica, TSDB_MIN_DB_REPLICA_OPTION, - TSDB_MAX_DB_REPLICA_OPTION); + code = checkDbEnumOption(pCxt, "replications", pOptions->pReplica, TSDB_MIN_DB_REPLICA, TSDB_MAX_DB_REPLICA); } if (TSDB_CODE_SUCCESS == code) { code = checkTtlOption(pCxt, pOptions->pTtl); @@ -1569,16 +1726,19 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, SDatabaseOptions* p code = checkRangeOption(pCxt, "vgroups", pOptions->pNumOfVgroups, TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbEnumOption(pCxt, "singleStable", pOptions->pSingleStable, TSDB_MIN_DB_SINGLE_STABLE_OPTION, - TSDB_MAX_DB_SINGLE_STABLE_OPTION); + code = checkDbEnumOption(pCxt, "singleStable", pOptions->pSingleStable, TSDB_DB_SINGLE_STABLE_ON, + TSDB_DB_SINGLE_STABLE_OFF); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbEnumOption(pCxt, "streamMode", pOptions->pStreamMode, TSDB_MIN_DB_STREAM_MODE_OPTION, - TSDB_MAX_DB_STREAM_MODE_OPTION); + code = + checkDbEnumOption(pCxt, "streamMode", pOptions->pStreamMode, TSDB_DB_STREAM_MODE_OFF, TSDB_DB_STREAM_MODE_ON); } if (TSDB_CODE_SUCCESS == code) { code = checkDbRetentionsOption(pCxt, pOptions->pRetentions); } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbEnumOption(pCxt, "strict", pOptions->pStrict, TSDB_DB_STRICT_OFF, TSDB_DB_STRICT_ON); + } return code; } @@ -1640,7 +1800,7 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, pReq->daysToKeep2 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 2), -1); pReq->fsyncPeriod = GET_OPTION_VAL(pStmt->pOptions->pFsyncPeriod, -1); pReq->walLevel = GET_OPTION_VAL(pStmt->pOptions->pWalLevel, -1); - pReq->quorum = GET_OPTION_VAL(pStmt->pOptions->pQuorum, -1); + pReq->strict = GET_OPTION_VAL(pStmt->pOptions->pQuorum, -1); pReq->cacheLastRow = GET_OPTION_VAL(pStmt->pOptions->pCachelast, -1); pReq->replications = GET_OPTION_VAL(pStmt->pOptions->pReplica, -1); return; @@ -1744,6 +1904,17 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS return TSDB_CODE_SUCCESS; } +static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt) { + SNode* pNode; + FOREACH(pNode, pStmt->pTags) { + SColumnDefNode* pCol = (SColumnDefNode*)pNode; + if(pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1){ + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) { if (NULL == pFuncs) { return TSDB_CODE_SUCCESS; @@ -1753,7 +1924,8 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_ROLLUP_OPTION); } SFunctionNode* pFunc = nodesListGetNode(pFuncs, 0); - if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { + SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet}; + if (TSDB_CODE_SUCCESS != fmGetFuncInfo(¶m, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName); } return TSDB_CODE_SUCCESS; @@ -1779,6 +1951,9 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt if (TSDB_CODE_SUCCESS == code) { code = checkRangeOption(pCxt, "delay", pStmt->pOptions->pDelay, TSDB_MIN_DB_DELAY, TSDB_MAX_DB_DELAY); } + if (TSDB_CODE_SUCCESS == code) { + code = checkTableTags(pCxt, pStmt); + } return code; } @@ -1789,35 +1964,259 @@ static int32_t getAggregationMethod(SNodeList* pFuncs) { return ((SFunctionNode*)nodesListGetNode(pFuncs, 0))->funcId; } -static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { - int32_t code = checkCreateTable(pCxt, pStmt); - if (TSDB_CODE_SUCCESS != code) { +static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchema) { + int8_t flags = 0; + if (pCol->sma) { + flags |= SCHEMA_SMA_ON; + } + pSchema->colId = colId; + pSchema->type = pCol->dataType.type; + pSchema->bytes = calcTypeBytes(pCol->dataType); + pSchema->flags = flags; + strcpy(pSchema->name, pCol->colName); +} + +typedef struct SSampleAstInfo { + const char* pDbName; + const char* pTableName; + SNodeList* pFuncs; + SNode* pInterval; + SNode* pOffset; + SNode* pSliding; + STableMeta* pRollupTableMeta; +} SSampleAstInfo; + +static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, char** pAst, int32_t* pLen) { + SSelectStmt* pSelect = nodesMakeNode(QUERY_NODE_SELECT_STMT); + if (NULL == pSelect) { + return TSDB_CODE_OUT_OF_MEMORY; + } + sprintf(pSelect->stmtName, "%p", pSelect); + + SRealTableNode* pTable = nodesMakeNode(QUERY_NODE_REAL_TABLE); + if (NULL == pTable) { + nodesDestroyNode(pSelect); + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pTable->table.dbName, pInfo->pDbName); + strcpy(pTable->table.tableName, pInfo->pTableName); + TSWAP(pTable->pMeta, pInfo->pRollupTableMeta, STableMeta*); + pSelect->pFromTable = (SNode*)pTable; + + TSWAP(pSelect->pProjectionList, pInfo->pFuncs, SNodeList*); + SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pSelect->pProjectionList || NULL == pFunc) { + nodesDestroyNode(pSelect); + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pFunc->functionName, "_wstartts"); + nodesListPushFront(pSelect->pProjectionList, pFunc); + SNode* pProject = NULL; + FOREACH(pProject, pSelect->pProjectionList) { sprintf(((SExprNode*)pProject)->aliasName, "#%p", pProject); } + + SIntervalWindowNode* pInterval = nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); + if (NULL == pInterval) { + nodesDestroyNode(pSelect); + return TSDB_CODE_OUT_OF_MEMORY; + } + pSelect->pWindow = (SNode*)pInterval; + TSWAP(pInterval->pInterval, pInfo->pInterval, SNode*); + TSWAP(pInterval->pOffset, pInfo->pOffset, SNode*); + TSWAP(pInterval->pSliding, pInfo->pSliding, SNode*); + pInterval->pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pInterval->pCol) { + nodesDestroyNode(pSelect); + return TSDB_CODE_OUT_OF_MEMORY; + } + ((SColumnNode*)pInterval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + strcpy(((SColumnNode*)pInterval->pCol)->colName, PK_TS_COL_INTERNAL_NAME); + + int32_t code = translateQuery(pCxt, (SNode*)pSelect); + if (TSDB_CODE_SUCCESS == code) { + code = nodesNodeToString(pSelect, false, pAst, pLen); + } + nodesDestroyNode(pSelect); + return code; +} + +static void clearSampleAstInfo(SSampleAstInfo* pInfo) { + nodesDestroyList(pInfo->pFuncs); + nodesDestroyNode(pInfo->pInterval); + nodesDestroyNode(pInfo->pOffset); + nodesDestroyNode(pInfo->pSliding); +} + +static SNode* makeIntervalVal(SRetention* pRetension, int8_t precision) { + SValueNode* pVal = nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == pVal) { + return NULL; + } + int64_t timeVal = convertTimeFromPrecisionToUnit(pRetension->freq, precision, pRetension->freqUnit); + char buf[20] = {0}; + int32_t len = snprintf(buf, sizeof(buf), "%"PRId64"%c", timeVal, pRetension->freqUnit); + pVal->literal = strndup(buf, len); + if (NULL == pVal->literal) { + nodesDestroyNode(pVal); + return NULL; + } + pVal->isDuration = true; + pVal->node.resType.type = TSDB_DATA_TYPE_BIGINT; + pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + pVal->node.resType.precision = precision; + return (SNode*)pVal; +} + +static SNode* createColumnFromDef(SColumnDefNode* pDef) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return NULL; + } + strcpy(pCol->colName, pDef->colName); + return (SNode*)pCol; +} + +static SNode* createRollupFunc(SNode* pSrcFunc, SColumnDefNode* pColDef) { + SFunctionNode* pFunc = nodesCloneNode(pSrcFunc); + if (NULL == pFunc) { + return NULL; + } + if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createColumnFromDef(pColDef))) { + nodesDestroyNode(pFunc); + return NULL; + } + return (SNode*)pFunc; +} + +static SNodeList* createRollupFuncs(SCreateTableStmt* pStmt) { + SNodeList* pFuncs = nodesMakeList(); + if (NULL == pFuncs) { + return NULL; + } + + SNode* pFunc = NULL; + FOREACH(pFunc, pStmt->pOptions->pFuncs) { + SNode* pCol = NULL; + bool primaryKey = true; + FOREACH(pCol, pStmt->pCols) { + if (primaryKey) { + primaryKey = false; + continue; + } + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pFuncs, createRollupFunc(pFunc, (SColumnDefNode*)pCol))) { + nodesDestroyList(pFuncs); + return NULL; + } + } + } + + return pFuncs; +} + +static STableMeta* createRollupTableMeta(SCreateTableStmt* pStmt, int8_t precision) { + int32_t numOfField = LIST_LENGTH(pStmt->pCols) + LIST_LENGTH(pStmt->pTags); + STableMeta* pMeta = taosMemoryCalloc(1, sizeof(STableMeta) + numOfField * sizeof(SSchema)); + if (NULL == pMeta) { + return NULL; + } + pMeta->tableType = TSDB_SUPER_TABLE; + pMeta->tableInfo.numOfTags = LIST_LENGTH(pStmt->pTags); + pMeta->tableInfo.precision = precision; + pMeta->tableInfo.numOfColumns = LIST_LENGTH(pStmt->pCols); + + int32_t index = 0; + SNode* pCol = NULL; + FOREACH(pCol, pStmt->pCols) { + toSchema((SColumnDefNode*)pCol, index + 1, pMeta->schema + index); + ++index; + } + SNode* pTag = NULL; + FOREACH(pTag, pStmt->pTags) { + toSchema((SColumnDefNode*)pTag, index + 1, pMeta->schema + index); + ++index; + } + + return pMeta; +} + +static int32_t buildSampleAstInfoByTable(STranslateContext* pCxt, + SCreateTableStmt* pStmt, SRetention* pRetension, int8_t precision, SSampleAstInfo* pInfo) { + pInfo->pDbName = pStmt->dbName; + pInfo->pTableName = pStmt->tableName; + pInfo->pFuncs = createRollupFuncs(pStmt); + pInfo->pInterval = makeIntervalVal(pRetension, precision); + pInfo->pRollupTableMeta = createRollupTableMeta(pStmt, precision); + if (NULL == pInfo->pFuncs || NULL == pInfo->pInterval || NULL == pInfo->pRollupTableMeta) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t getRollupAst(STranslateContext* pCxt, + SCreateTableStmt* pStmt, SRetention* pRetension, int8_t precision, char** pAst, int32_t* pLen) { + SSampleAstInfo info = {0}; + int32_t code = buildSampleAstInfoByTable(pCxt, pStmt, pRetension, precision, &info); + if (TSDB_CODE_SUCCESS == code) { + code = buildSampleAst(pCxt, &info, pAst, pLen); + } + clearSampleAstInfo(&info); + return code; +} + +static int32_t buildRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt, SMCreateStbReq* pReq) { + SDbCfgInfo dbCfg = {0}; + int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg); + int32_t num = taosArrayGetSize(dbCfg.pRetensions); + if (TSDB_CODE_SUCCESS != code || num < 2) { return code; } + for (int32_t i = 1; i < num; ++i) { + SRetention *pRetension = taosArrayGet(dbCfg.pRetensions, i); + STranslateContext cxt = {0}; + initTranslateContext(pCxt->pParseCxt, &cxt); + code = getRollupAst(&cxt, pStmt, pRetension, dbCfg.precision, + 1 == i ? &pReq->pAst1 : &pReq->pAst2, 1 == i ? &pReq->ast1Len : &pReq->ast2Len); + destroyTranslateContext(&cxt); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; +} - SMCreateStbReq createReq = {0}; - createReq.igExists = pStmt->ignoreExists; - createReq.aggregationMethod = getAggregationMethod(pStmt->pOptions->pFuncs); - createReq.xFilesFactor = GET_OPTION_VAL(pStmt->pOptions->pFilesFactor, TSDB_DEFAULT_DB_FILE_FACTOR); - createReq.delay = GET_OPTION_VAL(pStmt->pOptions->pDelay, TSDB_DEFAULT_DB_DELAY); - columnDefNodeToField(pStmt->pCols, &createReq.pColumns); - columnDefNodeToField(pStmt->pTags, &createReq.pTags); - createReq.numOfColumns = LIST_LENGTH(pStmt->pCols); - createReq.numOfTags = LIST_LENGTH(pStmt->pTags); +static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStmt, SMCreateStbReq* pReq) { + pReq->igExists = pStmt->ignoreExists; + pReq->aggregationMethod = getAggregationMethod(pStmt->pOptions->pFuncs); + pReq->xFilesFactor = GET_OPTION_VAL(pStmt->pOptions->pFilesFactor, TSDB_DEFAULT_DB_FILE_FACTOR); + pReq->delay = GET_OPTION_VAL(pStmt->pOptions->pDelay, TSDB_DEFAULT_DB_DELAY); + columnDefNodeToField(pStmt->pCols, &pReq->pColumns); + columnDefNodeToField(pStmt->pTags, &pReq->pTags); + pReq->numOfColumns = LIST_LENGTH(pStmt->pCols); + pReq->numOfTags = LIST_LENGTH(pStmt->pTags); if (NULL == pStmt->pOptions->pSma) { - columnDefNodeToField(pStmt->pCols, &createReq.pSmas); - createReq.numOfSmas = createReq.numOfColumns; + columnDefNodeToField(pStmt->pCols, &pReq->pSmas); + pReq->numOfSmas = pReq->numOfColumns; } else { - columnNodeToField(pStmt->pOptions->pSma, &createReq.pSmas); - createReq.numOfSmas = LIST_LENGTH(pStmt->pOptions->pSma); + columnNodeToField(pStmt->pOptions->pSma, &pReq->pSmas); + pReq->numOfSmas = LIST_LENGTH(pStmt->pOptions->pSma); } SName tableName = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; strcpy(tableName.dbname, pStmt->dbName); strcpy(tableName.tname, pStmt->tableName); - tNameExtractFullName(&tableName, createReq.name); + tNameExtractFullName(&tableName, pReq->name); - code = buildCmdMsg(pCxt, TDMT_MND_CREATE_STB, (FSerializeFunc)tSerializeSMCreateStbReq, &createReq); + return buildRollupAst(pCxt, pStmt, pReq); +} + +static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { + SMCreateStbReq createReq = {0}; + int32_t code = checkCreateTable(pCxt, pStmt); + if (TSDB_CODE_SUCCESS == code) { + code = buildCreateStbReq(pCxt, pStmt, &createReq); + } + if (TSDB_CODE_SUCCESS == code) { + code = buildCmdMsg(pCxt, TDMT_MND_CREATE_STB, (FSerializeFunc)tSerializeSMCreateStbReq, &createReq); + } tFreeSMCreateStbReq(&createReq); return code; } @@ -1983,20 +2382,20 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt } static int32_t nodeTypeToShowType(ENodeType nt) { - // switch (nt) { - // case QUERY_NODE_SHOW_CONNECTIONS_STMT: - // return TSDB_MGMT_TABLE_CONNS; - // case QUERY_NODE_SHOW_LICENCE_STMT: - // return TSDB_MGMT_TABLE_GRANTS; - // case QUERY_NODE_SHOW_QUERIES_STMT: - // return TSDB_MGMT_TABLE_QUERIES; - // case QUERY_NODE_SHOW_TOPICS_STMT: - // return 0; // todo - // case QUERY_NODE_SHOW_VARIABLE_STMT: - // return TSDB_MGMT_TABLE_VARIABLES; - // default: - // break; - // } + switch (nt) { + case QUERY_NODE_SHOW_CONNECTIONS_STMT: + return TSDB_MGMT_TABLE_CONNS; + case QUERY_NODE_SHOW_LICENCE_STMT: + return TSDB_MGMT_TABLE_GRANTS; + case QUERY_NODE_SHOW_QUERIES_STMT: + return TSDB_MGMT_TABLE_QUERIES; + case QUERY_NODE_SHOW_TOPICS_STMT: + return 0; // todo + case QUERY_NODE_SHOW_VARIABLE_STMT: + return 0; // todo + default: + break; + } return 0; } @@ -2027,57 +2426,28 @@ static int32_t getSmaIndexExpr(STranslateContext* pCxt, SCreateIndexStmt* pStmt, return nodesListToString(pStmt->pOptions->pFuncs, false, pExpr, pLen); } -static int32_t getSmaIndexBuildAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, char** pAst, int32_t* pLen) { - SSelectStmt* pSelect = nodesMakeNode(QUERY_NODE_SELECT_STMT); - if (NULL == pSelect) { - return TSDB_CODE_OUT_OF_MEMORY; - } - sprintf(pSelect->stmtName, "%p", pSelect); - - SRealTableNode* pTable = nodesMakeNode(QUERY_NODE_REAL_TABLE); - if (NULL == pTable) { - nodesDestroyNode(pSelect); +static int32_t buildSampleAstInfoByIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SSampleAstInfo* pInfo) { + pInfo->pDbName = pCxt->pParseCxt->db; + pInfo->pTableName = pStmt->tableName; + pInfo->pFuncs = nodesCloneList(pStmt->pOptions->pFuncs); + pInfo->pInterval = nodesCloneNode(pStmt->pOptions->pInterval); + pInfo->pOffset = nodesCloneNode(pStmt->pOptions->pOffset); + pInfo->pSliding = nodesCloneNode(pStmt->pOptions->pSliding); + if (NULL == pInfo->pFuncs || NULL == pInfo->pInterval || + (NULL != pStmt->pOptions->pOffset && NULL == pInfo->pOffset) || + (NULL != pStmt->pOptions->pSliding && NULL == pInfo->pSliding)) { return TSDB_CODE_OUT_OF_MEMORY; } - strcpy(pTable->table.dbName, pCxt->pParseCxt->db); - strcpy(pTable->table.tableName, pStmt->tableName); - pSelect->pFromTable = (SNode*)pTable; - - pSelect->pProjectionList = nodesCloneList(pStmt->pOptions->pFuncs); - SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); - if (NULL == pSelect->pProjectionList || NULL == pFunc) { - nodesDestroyNode(pSelect); - return TSDB_CODE_OUT_OF_MEMORY; - } - strcpy(pFunc->functionName, "_wstartts"); - nodesListPushFront(pSelect->pProjectionList, pFunc); - SNode* pProject = NULL; - FOREACH(pProject, pSelect->pProjectionList) { sprintf(((SExprNode*)pProject)->aliasName, "#sma_%p", pProject); } - - SIntervalWindowNode* pInterval = nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); - if (NULL == pInterval) { - nodesDestroyNode(pSelect); - return TSDB_CODE_OUT_OF_MEMORY; - } - pSelect->pWindow = (SNode*)pInterval; - pInterval->pCol = nodesMakeNode(QUERY_NODE_COLUMN); - pInterval->pInterval = nodesCloneNode(pStmt->pOptions->pInterval); - pInterval->pOffset = nodesCloneNode(pStmt->pOptions->pOffset); - pInterval->pSliding = nodesCloneNode(pStmt->pOptions->pSliding); - if (NULL == pInterval->pCol || NULL == pInterval->pInterval || - (NULL != pStmt->pOptions->pOffset && NULL == pInterval->pOffset) || - (NULL != pStmt->pOptions->pSliding && NULL == pInterval->pSliding)) { - nodesDestroyNode(pSelect); - return TSDB_CODE_OUT_OF_MEMORY; - } - ((SColumnNode*)pInterval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - strcpy(((SColumnNode*)pInterval->pCol)->colName, PK_TS_COL_INTERNAL_NAME); + return TSDB_CODE_SUCCESS; +} - int32_t code = translateQuery(pCxt, (SNode*)pSelect); +static int32_t getSmaIndexAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, char** pAst, int32_t* pLen) { + SSampleAstInfo info = {0}; + int32_t code = buildSampleAstInfoByIndex(pCxt, pStmt, &info); if (TSDB_CODE_SUCCESS == code) { - code = nodesNodeToString(pSelect, false, pAst, pLen); + code = buildSampleAst(pCxt, &info, pAst, pLen); } - nodesDestroyNode(pSelect); + clearSampleAstInfo(&info); return code; } @@ -2106,7 +2476,7 @@ static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStm code = getSmaIndexExpr(pCxt, pStmt, &pReq->expr, &pReq->exprLen); } if (TSDB_CODE_SUCCESS == code) { - code = getSmaIndexBuildAst(pCxt, pStmt, &pReq->ast, &pReq->astLen); + code = getSmaIndexAst(pCxt, pStmt, &pReq->ast, &pReq->astLen); } return code; @@ -2129,10 +2499,12 @@ static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt tFreeSMCreateSmaReq(&createSmaReq); return code; } + static int32_t buildCreateFullTextReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SMCreateFullTextReq* pReq) { // impl later return 0; } + static int32_t translateCreateFullTextIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { SMCreateFullTextReq createFTReq = {0}; int32_t code = buildCreateFullTextReq(pCxt, pStmt, &createFTReq); @@ -2340,12 +2712,64 @@ static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pSt return TSDB_CODE_SUCCESS; } +static int32_t readFromFile(char* pName, int32_t *len, char **buf) { + int64_t filesize = 0; + if (taosStatFile(pName, &filesize, NULL) < 0) { + return TAOS_SYSTEM_ERROR(errno); + } + + *len = filesize; + + if (*len <= 0) { + return TSDB_CODE_TSC_FILE_EMPTY; + } + + *buf = taosMemoryCalloc(1, *len); + if (*buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + TdFilePtr tfile = taosOpenFile(pName, O_RDONLY | O_BINARY); + if (NULL == tfile) { + taosMemoryFreeClear(*buf); + return TAOS_SYSTEM_ERROR(errno); + } + + int64_t s = taosReadFile(tfile, *buf, *len); + if (s != *len) { + taosCloseFile(&tfile); + taosMemoryFreeClear(*buf); + return TSDB_CODE_TSC_APP_ERROR; + } + taosCloseFile(&tfile); + return TSDB_CODE_SUCCESS; +} + +static int32_t translateCreateFunction(STranslateContext* pCxt, SCreateFunctionStmt* pStmt) { + SCreateFuncReq req = {0}; + strcpy(req.name, pStmt->funcName); + req.igExists = pStmt->ignoreExists; + req.funcType = pStmt->isAgg ? TSDB_FUNC_TYPE_AGGREGATE : TSDB_FUNC_TYPE_SCALAR; + req.scriptType = TSDB_FUNC_SCRIPT_BIN_LIB; + req.outputType = pStmt->outputDt.type; + req.outputLen = pStmt->outputDt.bytes; + req.bufSize = pStmt->bufSize; + int32_t code = readFromFile(pStmt->libraryPath, &req.codeLen, &req.pCode); + if (TSDB_CODE_SUCCESS == code) { + code = buildCmdMsg(pCxt, TDMT_MND_CREATE_FUNC, (FSerializeFunc)tSerializeSCreateFuncReq, &req); + } + return code; +} + static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pNode)) { case QUERY_NODE_SELECT_STMT: code = translateSelect(pCxt, (SSelectStmt*)pNode); break; + case QUERY_NODE_SET_OPERATOR: + code = translateSetOperator(pCxt, (SSetOperator*)pNode); + break; case QUERY_NODE_CREATE_DATABASE_STMT: code = translateCreateDatabase(pCxt, (SCreateDatabaseStmt*)pNode); break; @@ -2438,6 +2862,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_DROP_STREAM_STMT: code = translateDropStream(pCxt, (SDropStreamStmt*)pNode); break; + case QUERY_NODE_CREATE_FUNCTION_STMT: + code = translateCreateFunction(pCxt, (SCreateFunctionStmt*)pNode); + break; default: break; } @@ -2455,8 +2882,8 @@ static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { return code; } -static int32_t extractSelectResultSchema(const SSelectStmt* pSelect, int32_t* numOfCols, SSchema** pSchema) { - *numOfCols = LIST_LENGTH(pSelect->pProjectionList); +static int32_t extractQueryResultSchema(const SNodeList* pProjections, int32_t* numOfCols, SSchema** pSchema) { + *numOfCols = LIST_LENGTH(pProjections); *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); if (NULL == (*pSchema)) { return TSDB_CODE_OUT_OF_MEMORY; @@ -2464,7 +2891,7 @@ static int32_t extractSelectResultSchema(const SSelectStmt* pSelect, int32_t* nu SNode* pNode; int32_t index = 0; - FOREACH(pNode, pSelect->pProjectionList) { + FOREACH(pNode, pProjections) { SExprNode* pExpr = (SExprNode*)pNode; (*pSchema)[index].type = pExpr->resType.type; (*pSchema)[index].bytes = pExpr->resType.bytes; @@ -2523,7 +2950,8 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS switch (nodeType(pRoot)) { case QUERY_NODE_SELECT_STMT: - return extractSelectResultSchema((SSelectStmt*)pRoot, numOfCols, pSchema); + case QUERY_NODE_SET_OPERATOR: + return extractQueryResultSchema(getProjectList(pRoot), numOfCols, pSchema); case QUERY_NODE_EXPLAIN_STMT: return extractExplainResultSchema(numOfCols, pSchema); case QUERY_NODE_DESCRIBE_STMT: @@ -2535,24 +2963,6 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS return TSDB_CODE_FAILED; } -static void destroyTranslateContext(STranslateContext* pCxt) { - if (NULL != pCxt->pNsLevel) { - size_t size = taosArrayGetSize(pCxt->pNsLevel); - for (size_t i = 0; i < size; ++i) { - taosArrayDestroy(taosArrayGetP(pCxt->pNsLevel, i)); - } - taosArrayDestroy(pCxt->pNsLevel); - } - - if (NULL != pCxt->pCmdMsg) { - taosMemoryFreeClear(pCxt->pCmdMsg->pMsg); - taosMemoryFreeClear(pCxt->pCmdMsg); - } - - taosHashCleanup(pCxt->pDbs); - taosHashCleanup(pCxt->pTables); -} - static const char* getSysDbName(ENodeType type) { switch (type) { case QUERY_NODE_SHOW_DATABASES_STMT: @@ -2570,6 +2980,7 @@ static const char* getSysDbName(ENodeType type) { case QUERY_NODE_SHOW_BNODES_STMT: case QUERY_NODE_SHOW_SNODES_STMT: case QUERY_NODE_SHOW_LICENCE_STMT: + case QUERY_NODE_SHOW_CLUSTER_STMT: return TSDB_INFORMATION_SCHEMA_DB; case QUERY_NODE_SHOW_CONNECTIONS_STMT: case QUERY_NODE_SHOW_QUERIES_STMT: @@ -2612,9 +3023,12 @@ static const char* getSysTableName(ENodeType type) { return TSDB_INS_TABLE_SNODES; case QUERY_NODE_SHOW_LICENCE_STMT: return TSDB_INS_TABLE_LICENCES; + case QUERY_NODE_SHOW_CLUSTER_STMT: + return TSDB_INS_TABLE_CLUSTER; case QUERY_NODE_SHOW_CONNECTIONS_STMT: + return TSDB_PERFS_TABLE_CONNECTIONS; case QUERY_NODE_SHOW_QUERIES_STMT: - // todo + return TSDB_PERFS_TABLE_QUERIES; default: break; } @@ -2739,18 +3153,6 @@ typedef struct SVgroupTablesBatch { char dbName[TSDB_DB_NAME_LEN]; } SVgroupTablesBatch; -static void toSchemaEx(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchema) { - int8_t flags = 0; - if (pCol->sma) { - flags |= SCHEMA_SMA_ON; - } - pSchema->colId = colId; - pSchema->type = pCol->dataType.type; - pSchema->bytes = calcTypeBytes(pCol->dataType); - pSchema->flags = flags; - strcpy(pSchema->name, pCol->colName); -} - static void destroyCreateTbReq(SVCreateTbReq* pReq) { taosMemoryFreeClear(pReq->name); taosMemoryFreeClear(pReq->ntbCfg.pSchema); @@ -2798,7 +3200,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* SNode* pCol; col_id_t index = 0; FOREACH(pCol, pStmt->pCols) { - toSchemaEx((SColumnDefNode*)pCol, index + 1, req.ntbCfg.pSchema + index); + toSchema((SColumnDefNode*)pCol, index + 1, req.ntbCfg.pSchema + index); ++index; } if (TSDB_CODE_SUCCESS != buildSmaParam(pStmt->pOptions, &req)) { @@ -2952,17 +3354,25 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema, SKVRowBuilder* pBuilder) { + if(pSchema->type == TSDB_DATA_TYPE_JSON){ + if(pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); + } + + return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId); + } + if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { return pCxt->errCode; } - SVariant var; - valueNodeToVariant(pVal, &var); - char tagVal[TSDB_MAX_TAGS_LEN] = {0}; - int32_t code = taosVariantDump(&var, tagVal, pSchema->type, true); - if (TSDB_CODE_SUCCESS == code) { - tdAddColToKVRow(pBuilder, pSchema->colId, pSchema->type, tagVal); + + if(pVal->node.resType.type == TSDB_DATA_TYPE_NULL){ + // todo + }else{ + tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]); } - return code; + + return TSDB_CODE_SUCCESS; } static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta, @@ -3139,6 +3549,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_SHOW_SNODES_STMT: case QUERY_NODE_SHOW_CONNECTIONS_STMT: case QUERY_NODE_SHOW_QUERIES_STMT: + case QUERY_NODE_SHOW_CLUSTER_STMT: code = rewriteShow(pCxt, pQuery); break; case QUERY_NODE_CREATE_TABLE_STMT: @@ -3163,6 +3574,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { switch (nodeType(pQuery->pRoot)) { case QUERY_NODE_SELECT_STMT: + case QUERY_NODE_SET_OPERATOR: case QUERY_NODE_EXPLAIN_STMT: pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->haveResultSet = true; @@ -3224,19 +3636,11 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { } int32_t translate(SParseContext* pParseCxt, SQuery* pQuery) { - STranslateContext cxt = { - .pParseCxt = pParseCxt, - .errCode = TSDB_CODE_SUCCESS, - .msgBuf = {.buf = pParseCxt->pMsg, .len = pParseCxt->msgLen}, - .pNsLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES), - .currLevel = 0, - .currClause = 0, - .pDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK), - .pTables = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK)}; - if (NULL == cxt.pNsLevel) { - return TSDB_CODE_OUT_OF_MEMORY; + STranslateContext cxt = {0}; + int32_t code = initTranslateContext(pParseCxt, &cxt); + if (TSDB_CODE_SUCCESS == code) { + code = fmFuncMgtInit(); } - int32_t code = fmFuncMgtInit(); if (TSDB_CODE_SUCCESS == code) { code = rewriteQuery(&cxt, pQuery); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 51e0b2a3280db171b4bd6ba28e79fa07ab765100..075caa868c6c7a15446164413c364dcc0995fb46 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -14,6 +14,7 @@ */ #include "parUtil.h" +#include "cJSON.h" static char* getSyntaxErrFormat(int32_t errCode) { switch (errCode) { @@ -115,6 +116,10 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "sliding value no larger than the interval value"; case TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL: return "sliding value can not less than 1% of interval value"; + case TSDB_CODE_PAR_ONLY_ONE_JSON_TAG: + return "Only one tag if there is a json tag"; + case TSDB_CODE_PAR_INCORRECT_NUM_OF_COL: + return "Query block has incorrect number of result columns"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: @@ -215,24 +220,178 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta) { } int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) { - // delete escape character: \\, \', \" + if (len <=0 || dlen <= 0) return 0; + char delim = src[0]; - int32_t cnt = 0; int32_t j = 0; for (uint32_t k = 1; k < len - 1; ++k) { if (j >= dlen) { - break; + dst[j - 1] = '\0'; + return j; } - if (src[k] == '\\' || (src[k] == delim && src[k + 1] == delim)) { + if (src[k] == delim && src[k + 1] == delim) { // deal with "", '' dst[j] = src[k + 1]; - cnt++; j++; k++; continue; } + + if (src[k] == '\\') { // deal with escape character + if(src[k+1] == 'n'){ + dst[j] = '\n'; + }else if(src[k+1] == 'r'){ + dst[j] = '\r'; + }else if(src[k+1] == 't'){ + dst[j] = '\t'; + }else if(src[k+1] == '\\'){ + dst[j] = '\\'; + }else if(src[k+1] == '\''){ + dst[j] = '\''; + }else if(src[k+1] == '"'){ + dst[j] = '"'; + }else if(src[k+1] == '%' || src[k+1] == '_'){ + dst[j++] = src[k]; + dst[j] = src[k+1]; + }else{ + dst[j] = src[k+1]; + } + j++; + k++; + continue; + } + dst[j] = src[k]; j++; } dst[j] = '\0'; return j; } + +static bool isValidateTag(char *input) { + if (!input) return false; + for (size_t i = 0; i < strlen(input); ++i) { + if (isprint(input[i]) == 0) return false; + } + return true; +} + +int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){ + // set json NULL data + uint8_t jsonNULL = TSDB_DATA_TYPE_NULL; + int jsonIndex = startColId + 1; + if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){ + tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES); + return TSDB_CODE_SUCCESS; + } + + // set json real data + cJSON *root = cJSON_Parse(json); + if (root == NULL){ + return buildSyntaxErrMsg(pMsgBuf, "json parse error", json); + } + + int size = cJSON_GetArraySize(root); + if(!cJSON_IsObject(root)){ + return buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json); + } + + int retCode = 0; + char *tagKV = NULL; + SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + for(int i = 0; i < size; i++) { + cJSON* item = cJSON_GetArrayItem(root, i); + if (!item) { + qError("json inner error:%d", i); + retCode = buildSyntaxErrMsg(pMsgBuf, "json inner error", json); + goto end; + } + + char *jsonKey = item->string; + if(!isValidateTag(jsonKey)){ + retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey); + goto end; + } +// if(strlen(jsonKey) > TSDB_MAX_JSON_KEY_LEN){ +// tscError("json key too long error"); +// retCode = tscSQLSyntaxErrMsg(errMsg, "json key too long, more than 256", NULL); +// goto end; +// } + size_t keyLen = strlen(jsonKey); + if(keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL){ + continue; + } + // key: keyLen + VARSTR_HEADER_SIZE, value type: CHAR_BYTES, value reserved: LONG_BYTES + tagKV = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + LONG_BYTES, 1); + if(!tagKV) { + retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + strncpy(varDataVal(tagKV), jsonKey, keyLen); + varDataSetLen(tagKV, keyLen); + if(taosHashGetSize(keyHash) == 0){ + uint8_t jsonNotNULL = TSDB_DATA_TYPE_JSON; + tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES); // add json type + } + taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless + + if(item->type == cJSON_String){ // add json value format: type|data + char *jsonValue = item->valuestring; + int32_t valLen = (int32_t)strlen(jsonValue); + int32_t totalLen = keyLen + VARSTR_HEADER_SIZE + valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES; + char *tmp = taosMemoryRealloc(tagKV, totalLen); + if(!tmp) { + retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + tagKV = tmp; + char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); + char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); + *valueType = TSDB_DATA_TYPE_NCHAR; + if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(valueData), + (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { + qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno)); + retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); + goto end; + } + + varDataSetLen(valueData, valLen); + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, totalLen); + }else if(item->type == cJSON_Number){ + if(!isfinite(item->valuedouble)){ + qError("json value is invalidate"); + retCode = buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json); + goto end; + } + char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); + char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); + *valueType = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_DOUBLE; + if(*valueType== TSDB_DATA_TYPE_DOUBLE) *((double *)valueData) = item->valuedouble; + else if(*valueType == TSDB_DATA_TYPE_BIGINT) *((int64_t *)valueData) = item->valueint; + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES +LONG_BYTES); + }else if(item->type == cJSON_True || item->type == cJSON_False){ + char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); + char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); + *valueType = TSDB_DATA_TYPE_BOOL; + *valueData = (char)(item->valueint); + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + CHAR_BYTES); + }else if(item->type == cJSON_NULL){ + char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); + *valueType = TSDB_DATA_TYPE_NULL; + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); + } + else{ + retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json); + goto end; + } + } + + if(taosHashGetSize(keyHash) == 0){ // set json NULL true + tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES); + } + +end: + taosMemoryFree(tagKV); + taosHashCleanup(keyHash); + cJSON_Delete(root); + return retCode; +} \ No newline at end of file diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index ebe76cc1293f16783517c51e3e2b509be4849ca8..f17322063adc8a176fd0e61115ef81dc39a5121f 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -13,12 +13,17 @@ * along with this program. If not, see . */ +#include "os.h" #include "parser.h" #include "parInt.h" #include "parToken.h" -static bool isInsertSql(const char* pStr, size_t length) { +bool isInsertSql(const char* pStr, size_t length) { + if (NULL == pStr) { + return false; + } + int32_t index = 0; do { @@ -68,4 +73,4 @@ void qDestroyQuery(SQuery* pQueryNode) { int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { return extractResultSchema(pRoot, numOfCols, pSchema); -} \ No newline at end of file +} diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 110f4f9fb595f6cded70977bbd563c2b50ee20c7..bf23058f7c745a4ec695549b1362e6bd30452a76 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -32,6 +32,7 @@ #include #include +#include "os.h" #include "functionMgt.h" #include "nodes.h" #include "parToken.h" @@ -100,24 +101,24 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 334 +#define YYNOCODE 336 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - SAlterOption yy29; - EJoinType yy164; - ENullOrder yy209; - SDataType yy388; - EOperatorType yy416; - SNode* yy456; - SToken yy537; - EOrder yy626; - SNodeList* yy632; - EFillMode yy646; - bool yy649; - int32_t yy652; + int32_t yy4; + bool yy89; + EFillMode yy102; + SDataType yy112; + SAlterOption yy221; + ENullOrder yy361; + EJoinType yy372; + SNodeList* yy376; + EOperatorType yy380; + EOrder yy386; + SNode* yy392; + SToken yy449; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -132,17 +133,17 @@ typedef union { #define ParseCTX_PARAM #define ParseCTX_FETCH #define ParseCTX_STORE -#define YYNSTATE 568 -#define YYNRULE 433 -#define YYNTOKEN 220 -#define YY_MAX_SHIFT 567 -#define YY_MIN_SHIFTREDUCE 843 -#define YY_MAX_SHIFTREDUCE 1275 -#define YY_ERROR_ACTION 1276 -#define YY_ACCEPT_ACTION 1277 -#define YY_NO_ACTION 1278 -#define YY_MIN_REDUCE 1279 -#define YY_MAX_REDUCE 1711 +#define YYNSTATE 572 +#define YYNRULE 437 +#define YYNTOKEN 222 +#define YY_MAX_SHIFT 571 +#define YY_MIN_SHIFTREDUCE 850 +#define YY_MAX_SHIFTREDUCE 1286 +#define YY_ERROR_ACTION 1287 +#define YY_ACCEPT_ACTION 1288 +#define YY_NO_ACTION 1289 +#define YY_MIN_REDUCE 1290 +#define YY_MAX_REDUCE 1726 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -209,534 +210,553 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (1863) +#define YY_ACTTAB_COUNT (1960) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 1390, 1690, 1388, 1563, 268, 26, 203, 323, 482, 1277, - /* 10 */ 469, 1549, 33, 31, 1689, 1563, 1492, 77, 1688, 1302, - /* 20 */ 277, 1549, 1096, 285, 380, 1545, 1552, 1579, 34, 32, - /* 30 */ 30, 29, 28, 1399, 466, 1545, 1551, 248, 1094, 1579, - /* 40 */ 1549, 481, 53, 417, 465, 1535, 466, 441, 1535, 1301, - /* 50 */ 12, 33, 31, 1218, 1545, 1551, 465, 1102, 481, 277, - /* 60 */ 1535, 1096, 293, 1395, 1535, 445, 125, 1564, 468, 1566, - /* 70 */ 1567, 464, 104, 459, 1, 482, 359, 1094, 72, 1564, - /* 80 */ 468, 1566, 1567, 464, 321, 459, 1377, 418, 1629, 12, - /* 90 */ 482, 1232, 249, 1625, 1535, 36, 1102, 564, 62, 77, - /* 100 */ 1399, 1690, 33, 31, 1690, 123, 387, 1291, 102, 1095, - /* 110 */ 277, 1704, 1096, 1, 136, 1399, 1375, 136, 1688, 1392, - /* 120 */ 22, 1688, 443, 132, 1636, 1637, 1690, 1641, 1094, 1280, - /* 130 */ 34, 32, 30, 29, 28, 359, 564, 1331, 1448, 136, - /* 140 */ 12, 1117, 481, 1688, 267, 1579, 1097, 1102, 1095, 1446, - /* 150 */ 87, 127, 466, 86, 85, 84, 83, 82, 81, 80, - /* 160 */ 79, 78, 1440, 469, 1, 518, 280, 1100, 1101, 1491, - /* 170 */ 1145, 1146, 1147, 1148, 1149, 1150, 1151, 461, 1156, 1157, - /* 180 */ 1158, 1159, 1160, 1161, 1162, 1097, 434, 564, 431, 395, - /* 190 */ 316, 389, 315, 560, 559, 394, 36, 137, 101, 1095, - /* 200 */ 390, 388, 137, 391, 55, 266, 1100, 1101, 174, 1145, - /* 210 */ 1146, 1147, 1148, 1149, 1150, 1151, 461, 1156, 1157, 1158, - /* 220 */ 1159, 1160, 1161, 1162, 482, 482, 482, 441, 482, 931, - /* 230 */ 33, 31, 452, 322, 330, 331, 1097, 358, 277, 880, - /* 240 */ 1096, 879, 137, 34, 32, 30, 29, 28, 933, 1399, - /* 250 */ 1399, 1399, 104, 1399, 436, 432, 1094, 1100, 1101, 881, - /* 260 */ 1145, 1146, 1147, 1148, 1149, 1150, 1151, 461, 1156, 1157, - /* 270 */ 1158, 1159, 1160, 1161, 1162, 1102, 33, 31, 1163, 385, - /* 280 */ 384, 250, 345, 143, 277, 1300, 1096, 1328, 102, 521, - /* 290 */ 87, 1371, 7, 86, 85, 84, 83, 82, 81, 80, - /* 300 */ 79, 78, 1094, 133, 1636, 1637, 1132, 1641, 281, 51, - /* 310 */ 1119, 395, 50, 389, 1180, 564, 121, 394, 137, 137, - /* 320 */ 101, 1102, 390, 388, 1401, 391, 1242, 1095, 147, 146, - /* 330 */ 1535, 317, 260, 34, 32, 30, 29, 28, 7, 540, - /* 340 */ 539, 538, 537, 292, 879, 536, 535, 534, 107, 529, - /* 350 */ 528, 527, 526, 525, 524, 523, 522, 114, 100, 53, - /* 360 */ 378, 564, 383, 1181, 1097, 428, 1240, 1241, 1243, 1244, - /* 370 */ 1690, 124, 99, 1095, 261, 1357, 259, 258, 453, 382, - /* 380 */ 1394, 1186, 435, 136, 386, 1100, 1101, 1688, 1145, 1146, - /* 390 */ 1147, 1148, 1149, 1150, 1151, 461, 1156, 1157, 1158, 1159, - /* 400 */ 1160, 1161, 1162, 441, 33, 31, 393, 392, 441, 532, - /* 410 */ 1097, 1120, 277, 311, 1096, 137, 25, 275, 1175, 1176, - /* 420 */ 1177, 1178, 1179, 1183, 1184, 1185, 515, 514, 104, 456, - /* 430 */ 1094, 1100, 1101, 104, 1145, 1146, 1147, 1148, 1149, 1150, - /* 440 */ 1151, 461, 1156, 1157, 1158, 1159, 1160, 1161, 1162, 1102, - /* 450 */ 33, 31, 445, 482, 409, 287, 69, 1096, 277, 120, - /* 460 */ 1096, 1448, 1396, 121, 102, 1121, 7, 282, 905, 102, - /* 470 */ 105, 1401, 1446, 1094, 9, 8, 1094, 1391, 1399, 134, - /* 480 */ 1636, 1637, 449, 1641, 196, 1636, 440, 906, 439, 564, - /* 490 */ 1376, 1690, 1102, 1690, 1299, 1102, 34, 32, 30, 29, - /* 500 */ 28, 1095, 1318, 1118, 136, 1298, 136, 518, 1688, 1297, - /* 510 */ 1688, 1296, 1, 969, 505, 504, 503, 973, 502, 975, - /* 520 */ 976, 501, 978, 498, 396, 984, 495, 986, 987, 492, - /* 530 */ 489, 247, 564, 1117, 288, 564, 482, 508, 1097, 1535, - /* 540 */ 338, 1482, 1484, 350, 1095, 1516, 1172, 1095, 1194, 1295, - /* 550 */ 1535, 106, 351, 100, 1535, 513, 1535, 383, 1313, 1100, - /* 560 */ 1101, 1399, 1145, 1146, 1147, 1148, 1149, 1150, 1151, 461, - /* 570 */ 1156, 1157, 1158, 1159, 1160, 1161, 1162, 516, 1479, 386, - /* 580 */ 398, 1097, 482, 198, 1097, 145, 34, 32, 30, 29, - /* 590 */ 28, 479, 225, 1448, 1535, 1429, 512, 511, 510, 289, - /* 600 */ 509, 520, 1100, 1101, 1446, 1100, 1101, 1399, 1145, 1146, - /* 610 */ 1147, 1148, 1149, 1150, 1151, 461, 1156, 1157, 1158, 1159, - /* 620 */ 1160, 1161, 1162, 1563, 250, 349, 450, 1294, 344, 343, - /* 630 */ 342, 341, 340, 1384, 337, 336, 335, 334, 333, 329, - /* 640 */ 328, 327, 326, 325, 324, 448, 482, 1579, 34, 32, - /* 650 */ 30, 29, 28, 482, 444, 480, 567, 1180, 1182, 290, - /* 660 */ 482, 1386, 217, 1293, 465, 533, 531, 121, 1535, 291, - /* 670 */ 221, 1399, 1535, 98, 1563, 1401, 1187, 1290, 1399, 556, - /* 680 */ 1289, 552, 548, 544, 220, 1399, 73, 1564, 468, 1566, - /* 690 */ 1567, 464, 1122, 459, 1648, 1213, 1629, 6, 1579, 1643, - /* 700 */ 270, 1625, 131, 1288, 1287, 444, 1181, 1448, 1535, 1286, - /* 710 */ 70, 23, 1285, 215, 199, 465, 24, 1382, 1483, 1535, - /* 720 */ 424, 1656, 1535, 1640, 1186, 1535, 34, 32, 30, 29, - /* 730 */ 28, 30, 29, 28, 1448, 1284, 1217, 73, 1564, 468, - /* 740 */ 1566, 1567, 464, 478, 459, 1447, 1643, 1629, 1535, 1535, - /* 750 */ 1225, 270, 1625, 131, 1535, 1563, 1119, 1535, 400, 25, - /* 760 */ 275, 1175, 1176, 1177, 1178, 1179, 1183, 1184, 1185, 1283, - /* 770 */ 1639, 423, 1657, 408, 180, 1272, 1524, 1311, 1643, 1579, - /* 780 */ 1535, 34, 32, 30, 29, 28, 466, 173, 1168, 1074, - /* 790 */ 403, 176, 1282, 177, 1119, 397, 465, 121, 308, 401, - /* 800 */ 1535, 172, 1638, 165, 167, 1402, 163, 166, 1082, 1083, - /* 810 */ 184, 1563, 300, 169, 1535, 460, 168, 310, 73, 1564, - /* 820 */ 468, 1566, 1567, 464, 507, 459, 171, 45, 1629, 170, - /* 830 */ 44, 416, 270, 1625, 1702, 1579, 1374, 1535, 407, 1292, - /* 840 */ 122, 1563, 466, 1663, 1132, 231, 9, 8, 1358, 112, - /* 850 */ 1213, 405, 465, 420, 1271, 447, 1535, 229, 47, 187, - /* 860 */ 1274, 1275, 1239, 189, 35, 1579, 200, 1441, 1188, 35, - /* 870 */ 148, 1563, 466, 1152, 73, 1564, 468, 1566, 1567, 464, - /* 880 */ 1556, 459, 465, 429, 1629, 1105, 1535, 410, 270, 1625, - /* 890 */ 1702, 377, 1554, 1104, 35, 1579, 193, 106, 1057, 1686, - /* 900 */ 68, 513, 466, 442, 73, 1564, 468, 1566, 1567, 464, - /* 910 */ 64, 459, 465, 206, 1629, 1659, 1535, 208, 270, 1625, - /* 920 */ 1702, 445, 1580, 516, 109, 202, 1563, 110, 474, 1647, - /* 930 */ 1216, 214, 2, 71, 238, 1564, 468, 1566, 1567, 464, - /* 940 */ 1117, 459, 512, 511, 510, 112, 509, 47, 487, 962, - /* 950 */ 1579, 957, 990, 295, 299, 255, 1108, 466, 1066, 222, - /* 960 */ 1690, 49, 48, 320, 1107, 142, 931, 465, 144, 332, - /* 970 */ 314, 1535, 257, 136, 110, 1481, 1563, 1688, 994, 111, - /* 980 */ 339, 112, 256, 1001, 306, 1000, 302, 298, 139, 74, - /* 990 */ 1564, 468, 1566, 1567, 464, 347, 459, 346, 110, 1629, - /* 1000 */ 1579, 348, 113, 1628, 1625, 1126, 353, 466, 352, 354, - /* 1010 */ 1125, 149, 152, 356, 355, 1124, 357, 465, 360, 137, - /* 1020 */ 160, 1535, 155, 130, 1563, 52, 158, 1123, 379, 376, - /* 1030 */ 381, 372, 368, 364, 159, 1563, 97, 1389, 265, 74, - /* 1040 */ 1564, 468, 1566, 1567, 464, 1102, 459, 162, 1579, 1629, - /* 1050 */ 1520, 1385, 164, 455, 1625, 463, 115, 116, 1387, 1579, - /* 1060 */ 54, 175, 1383, 157, 223, 465, 466, 117, 118, 1535, - /* 1070 */ 411, 419, 415, 412, 179, 1122, 465, 182, 421, 1670, - /* 1080 */ 1535, 422, 1660, 430, 472, 1669, 427, 245, 1564, 468, - /* 1090 */ 1566, 1567, 464, 462, 459, 457, 1601, 185, 125, 1564, - /* 1100 */ 468, 1566, 1567, 464, 188, 459, 1563, 269, 284, 283, - /* 1110 */ 5, 433, 438, 103, 426, 4, 1644, 1121, 1110, 1213, - /* 1120 */ 156, 1650, 151, 37, 153, 271, 451, 454, 16, 1490, - /* 1130 */ 1579, 1489, 195, 470, 1103, 129, 1563, 466, 471, 1610, - /* 1140 */ 194, 150, 446, 1703, 1563, 192, 475, 465, 476, 279, - /* 1150 */ 210, 1535, 477, 1102, 212, 63, 1400, 61, 224, 1687, - /* 1160 */ 1579, 201, 1372, 1705, 226, 219, 563, 466, 1579, 74, - /* 1170 */ 1564, 468, 1566, 1567, 464, 466, 459, 465, 128, 1629, - /* 1180 */ 485, 1535, 43, 232, 1626, 465, 233, 228, 230, 1535, - /* 1190 */ 1529, 1563, 425, 483, 1528, 294, 1525, 296, 297, 241, - /* 1200 */ 1564, 468, 1566, 1567, 464, 1106, 459, 246, 1564, 468, - /* 1210 */ 1566, 1567, 464, 1090, 459, 1579, 1091, 140, 1523, 303, - /* 1220 */ 301, 304, 466, 305, 1522, 307, 1521, 309, 1506, 141, - /* 1230 */ 312, 313, 465, 1069, 1068, 1500, 1535, 437, 1040, 274, - /* 1240 */ 1499, 1563, 1111, 319, 318, 1498, 1497, 1474, 1473, 1472, - /* 1250 */ 1471, 1470, 1469, 1468, 246, 1564, 468, 1566, 1567, 464, - /* 1260 */ 1467, 459, 1466, 1114, 1465, 1579, 1464, 1463, 1462, 1461, - /* 1270 */ 1460, 1459, 463, 108, 1458, 1457, 1456, 1455, 1454, 1563, - /* 1280 */ 1453, 1042, 465, 1452, 1451, 1450, 1535, 1449, 1330, 1514, - /* 1290 */ 1563, 1508, 1496, 1487, 1378, 898, 1329, 154, 1327, 361, - /* 1300 */ 363, 362, 1325, 1579, 245, 1564, 468, 1566, 1567, 464, - /* 1310 */ 466, 459, 1323, 1602, 1579, 367, 371, 365, 366, 369, - /* 1320 */ 465, 466, 370, 1321, 1535, 1310, 1309, 276, 373, 1306, - /* 1330 */ 1380, 465, 1006, 374, 530, 1535, 161, 1009, 278, 1279, - /* 1340 */ 1379, 76, 246, 1564, 468, 1566, 1567, 464, 1563, 459, - /* 1350 */ 1319, 375, 930, 246, 1564, 468, 1566, 1567, 464, 929, - /* 1360 */ 459, 1563, 928, 96, 95, 94, 93, 92, 91, 90, - /* 1370 */ 89, 88, 1579, 927, 532, 924, 923, 262, 1314, 466, - /* 1380 */ 263, 399, 1312, 264, 402, 1579, 1305, 404, 1304, 465, - /* 1390 */ 406, 75, 466, 1535, 1513, 1076, 1507, 413, 1495, 1563, - /* 1400 */ 1494, 1486, 465, 181, 3, 56, 1535, 119, 1563, 13, - /* 1410 */ 126, 234, 1564, 468, 1566, 1567, 464, 190, 459, 35, - /* 1420 */ 41, 14, 186, 1579, 240, 1564, 468, 1566, 1567, 464, - /* 1430 */ 466, 459, 1579, 46, 1238, 191, 178, 20, 414, 466, - /* 1440 */ 465, 1231, 40, 57, 1535, 21, 1563, 1210, 1554, 465, - /* 1450 */ 183, 1209, 39, 1535, 1265, 15, 197, 58, 1260, 135, - /* 1460 */ 1259, 1563, 242, 1564, 468, 1566, 1567, 464, 272, 459, - /* 1470 */ 1579, 235, 1564, 468, 1566, 1567, 464, 466, 459, 1264, - /* 1480 */ 1263, 273, 8, 1173, 17, 1579, 1155, 465, 458, 138, - /* 1490 */ 1154, 1535, 466, 27, 204, 1140, 1153, 10, 18, 19, - /* 1500 */ 38, 467, 465, 205, 1236, 1485, 1535, 11, 207, 243, - /* 1510 */ 1564, 468, 1566, 1567, 464, 211, 459, 64, 209, 473, - /* 1520 */ 59, 60, 213, 1553, 236, 1564, 468, 1566, 1567, 464, - /* 1530 */ 1563, 459, 1112, 42, 216, 484, 991, 486, 1563, 286, - /* 1540 */ 488, 490, 988, 491, 493, 985, 979, 496, 494, 497, - /* 1550 */ 499, 977, 968, 500, 1579, 983, 982, 981, 65, 980, - /* 1560 */ 1003, 466, 1579, 1002, 506, 66, 67, 999, 1563, 466, - /* 1570 */ 996, 465, 896, 517, 937, 1535, 519, 919, 218, 465, - /* 1580 */ 918, 917, 916, 1535, 915, 914, 912, 913, 932, 934, - /* 1590 */ 909, 908, 1579, 244, 1564, 468, 1566, 1567, 464, 466, - /* 1600 */ 459, 237, 1564, 468, 1566, 1567, 464, 1326, 459, 465, - /* 1610 */ 907, 904, 903, 1535, 902, 901, 1563, 541, 542, 1324, - /* 1620 */ 1322, 543, 545, 546, 547, 550, 549, 1320, 551, 553, - /* 1630 */ 554, 1575, 1564, 468, 1566, 1567, 464, 1308, 459, 555, - /* 1640 */ 1579, 557, 558, 1307, 1303, 561, 562, 466, 1098, 566, - /* 1650 */ 227, 565, 1278, 1278, 1563, 1278, 1278, 465, 1278, 1278, - /* 1660 */ 1278, 1535, 1278, 1563, 1278, 1278, 1278, 1278, 1278, 1278, - /* 1670 */ 1278, 1278, 1278, 1278, 1278, 1278, 1278, 1278, 1579, 1574, - /* 1680 */ 1564, 468, 1566, 1567, 464, 466, 459, 1579, 1278, 1278, - /* 1690 */ 1278, 1278, 1278, 1278, 466, 465, 1278, 1278, 1278, 1535, - /* 1700 */ 1278, 1563, 1278, 1278, 465, 1278, 1278, 1278, 1535, 1278, - /* 1710 */ 1278, 1278, 1278, 1278, 1278, 1278, 1563, 1573, 1564, 468, - /* 1720 */ 1566, 1567, 464, 1278, 459, 1579, 253, 1564, 468, 1566, - /* 1730 */ 1567, 464, 466, 459, 1278, 1278, 1278, 1278, 1278, 1278, - /* 1740 */ 1579, 1278, 465, 1278, 1278, 1278, 1535, 466, 1278, 1278, - /* 1750 */ 1278, 1278, 1278, 1278, 1563, 1278, 1278, 465, 1278, 1278, - /* 1760 */ 1278, 1535, 1278, 1278, 252, 1564, 468, 1566, 1567, 464, - /* 1770 */ 1278, 459, 1278, 1278, 1278, 1278, 1278, 1278, 1579, 254, - /* 1780 */ 1564, 468, 1566, 1567, 464, 466, 459, 1278, 1278, 1278, - /* 1790 */ 1278, 1278, 1563, 1278, 1278, 465, 1278, 1278, 1278, 1535, - /* 1800 */ 1278, 1278, 1278, 1278, 1278, 1278, 1278, 1278, 1278, 1278, - /* 1810 */ 1278, 1278, 1278, 1278, 1278, 1278, 1579, 251, 1564, 468, - /* 1820 */ 1566, 1567, 464, 466, 459, 1278, 1278, 1278, 1278, 1278, - /* 1830 */ 1278, 1278, 1278, 465, 1278, 1278, 1278, 1535, 1278, 1278, - /* 1840 */ 1278, 1278, 1278, 1278, 1278, 1278, 1278, 1278, 1278, 1278, - /* 1850 */ 1278, 1278, 1278, 1278, 1278, 239, 1564, 468, 1566, 1567, - /* 1860 */ 464, 1278, 459, + /* 0 */ 270, 1593, 287, 472, 1577, 485, 282, 325, 469, 1505, + /* 10 */ 1288, 1401, 33, 31, 78, 1577, 124, 1563, 1302, 1563, + /* 20 */ 279, 383, 1106, 34, 32, 30, 29, 28, 1593, 1563, + /* 30 */ 1412, 1559, 1565, 1559, 1565, 469, 125, 250, 1104, 1593, + /* 40 */ 1369, 1705, 438, 1559, 1566, 468, 469, 1389, 445, 1549, + /* 50 */ 12, 33, 31, 1228, 1704, 54, 468, 1112, 1702, 279, + /* 60 */ 1549, 1106, 484, 295, 319, 449, 1577, 243, 1578, 471, + /* 70 */ 1580, 1581, 467, 105, 462, 1, 1408, 1104, 73, 1578, + /* 80 */ 471, 1580, 1581, 467, 525, 462, 1384, 290, 1643, 12, + /* 90 */ 1593, 484, 251, 1639, 1496, 1498, 1112, 448, 568, 26, + /* 100 */ 205, 362, 1705, 1705, 1705, 441, 421, 468, 107, 103, + /* 110 */ 1105, 1549, 516, 485, 1, 137, 137, 137, 1129, 1702, + /* 120 */ 1702, 1702, 323, 447, 133, 1650, 1651, 445, 1655, 74, + /* 130 */ 1578, 471, 1580, 1581, 467, 519, 462, 568, 1412, 1643, + /* 140 */ 54, 144, 485, 272, 1639, 132, 36, 940, 1107, 1105, + /* 150 */ 422, 324, 105, 100, 515, 514, 513, 201, 512, 1127, + /* 160 */ 439, 1407, 1130, 428, 1670, 435, 942, 1412, 52, 1110, + /* 170 */ 1111, 51, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 464, + /* 180 */ 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1107, 103, 1705, + /* 190 */ 1461, 34, 32, 30, 29, 28, 269, 9, 8, 138, + /* 200 */ 69, 1459, 137, 134, 1650, 1651, 1702, 1655, 1110, 1111, + /* 210 */ 65, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 464, 1166, + /* 220 */ 1167, 1168, 1169, 1170, 1171, 1172, 33, 31, 138, 252, + /* 230 */ 227, 440, 436, 1442, 279, 249, 1106, 1127, 1577, 399, + /* 240 */ 63, 393, 521, 1461, 341, 398, 70, 353, 102, 284, + /* 250 */ 394, 392, 1104, 395, 1459, 1142, 354, 318, 391, 317, + /* 260 */ 106, 1405, 1593, 1190, 12, 33, 31, 1404, 1227, 469, + /* 270 */ 138, 1112, 1204, 279, 22, 1106, 1657, 56, 268, 468, + /* 280 */ 1252, 175, 24, 1549, 34, 32, 30, 29, 28, 1, + /* 290 */ 1657, 1104, 34, 32, 30, 29, 28, 30, 29, 28, + /* 300 */ 1654, 74, 1578, 471, 1580, 1581, 467, 128, 462, 138, + /* 310 */ 1112, 1643, 568, 1191, 1653, 272, 1639, 1717, 1453, 432, + /* 320 */ 1250, 1251, 1253, 1254, 1105, 413, 1677, 1390, 7, 1290, + /* 330 */ 352, 1196, 1128, 347, 346, 345, 344, 343, 138, 340, + /* 340 */ 339, 338, 337, 336, 332, 331, 330, 329, 328, 327, + /* 350 */ 326, 568, 59, 98, 97, 96, 95, 94, 93, 92, + /* 360 */ 91, 90, 1107, 1105, 1705, 1705, 25, 277, 1185, 1186, + /* 370 */ 1187, 1188, 1189, 1193, 1194, 1195, 362, 137, 1703, 1403, + /* 380 */ 1131, 1702, 1702, 1110, 1111, 6, 1155, 1156, 1157, 1158, + /* 390 */ 1159, 1160, 1161, 464, 1166, 1167, 1168, 1169, 1170, 1171, + /* 400 */ 1172, 1107, 524, 484, 452, 34, 32, 30, 29, 28, + /* 410 */ 89, 564, 563, 88, 87, 86, 85, 84, 83, 82, + /* 420 */ 81, 80, 1110, 1111, 1549, 1155, 1156, 1157, 1158, 1159, + /* 430 */ 1160, 1161, 464, 1166, 1167, 1168, 1169, 1170, 1171, 1172, + /* 440 */ 33, 31, 1173, 252, 1291, 536, 1313, 283, 279, 313, + /* 450 */ 1106, 289, 138, 485, 472, 122, 262, 101, 36, 122, + /* 460 */ 1506, 386, 333, 1414, 1226, 89, 1104, 1414, 88, 87, + /* 470 */ 86, 85, 84, 83, 82, 81, 80, 1190, 1412, 33, + /* 480 */ 31, 48, 101, 1387, 389, 1112, 386, 279, 1312, 1106, + /* 490 */ 1577, 1549, 34, 32, 30, 29, 28, 485, 485, 263, + /* 500 */ 200, 261, 260, 7, 385, 1104, 78, 334, 511, 389, + /* 510 */ 388, 387, 292, 390, 1593, 34, 32, 30, 29, 28, + /* 520 */ 122, 469, 1412, 1412, 1112, 485, 568, 1191, 1414, 1311, + /* 530 */ 1461, 468, 1493, 1549, 361, 1549, 291, 485, 1105, 146, + /* 540 */ 1223, 1459, 7, 1192, 107, 1196, 1409, 138, 516, 453, + /* 550 */ 1412, 397, 396, 74, 1578, 471, 1580, 1581, 467, 1461, + /* 560 */ 462, 1197, 1412, 1643, 1342, 568, 1132, 272, 1639, 1717, + /* 570 */ 1460, 519, 348, 1397, 1549, 1657, 1107, 1105, 1700, 1242, + /* 580 */ 25, 277, 1185, 1186, 1187, 1188, 1189, 1193, 1194, 1195, + /* 590 */ 515, 514, 513, 1577, 512, 1538, 23, 1110, 1111, 1652, + /* 600 */ 1155, 1156, 1157, 1158, 1159, 1160, 1161, 464, 1166, 1167, + /* 610 */ 1168, 1169, 1170, 1171, 1172, 1107, 399, 1593, 393, 148, + /* 620 */ 147, 1283, 398, 1310, 448, 102, 166, 394, 392, 164, + /* 630 */ 395, 302, 518, 517, 468, 391, 1110, 1111, 1549, 1155, + /* 640 */ 1156, 1157, 1158, 1159, 1160, 1161, 464, 1166, 1167, 1168, + /* 650 */ 1169, 1170, 1171, 1172, 33, 31, 74, 1578, 471, 1580, + /* 660 */ 1581, 467, 279, 462, 1106, 1577, 1643, 1388, 1549, 485, + /* 670 */ 272, 1639, 132, 34, 32, 30, 29, 28, 1530, 485, + /* 680 */ 1104, 1309, 122, 485, 445, 887, 186, 886, 482, 1593, + /* 690 */ 1415, 1671, 293, 451, 1412, 1399, 469, 537, 535, 1112, + /* 700 */ 1282, 1235, 485, 886, 1412, 888, 468, 1129, 1412, 105, + /* 710 */ 1549, 483, 485, 1308, 1307, 310, 521, 1, 1339, 381, + /* 720 */ 1306, 219, 1305, 1304, 1301, 1395, 1549, 1412, 74, 1578, + /* 730 */ 471, 1580, 1581, 467, 312, 462, 168, 1412, 1643, 167, + /* 740 */ 568, 455, 272, 1639, 1717, 103, 34, 32, 30, 29, + /* 750 */ 28, 1300, 1105, 1661, 1299, 1298, 178, 1297, 1549, 1549, + /* 760 */ 135, 1650, 1651, 1296, 1655, 1549, 459, 1549, 1549, 1549, + /* 770 */ 544, 543, 542, 541, 294, 463, 540, 539, 538, 108, + /* 780 */ 533, 532, 531, 530, 529, 528, 527, 526, 115, 522, + /* 790 */ 1107, 1178, 1497, 1498, 1662, 1223, 1549, 1129, 510, 1549, + /* 800 */ 1549, 121, 1549, 1295, 1092, 1093, 1294, 1293, 1549, 113, + /* 810 */ 1142, 1110, 1111, 424, 1155, 1156, 1157, 1158, 1159, 1160, + /* 820 */ 1161, 464, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1570, + /* 830 */ 978, 508, 507, 506, 982, 505, 984, 985, 504, 987, + /* 840 */ 501, 1568, 993, 498, 995, 996, 495, 492, 1549, 123, + /* 850 */ 1577, 1549, 1549, 170, 233, 1329, 169, 172, 1324, 411, + /* 860 */ 171, 1322, 1115, 1577, 47, 189, 231, 420, 1249, 191, + /* 870 */ 9, 8, 409, 1114, 1593, 1285, 1286, 400, 35, 149, + /* 880 */ 402, 469, 1198, 405, 1303, 1182, 35, 1593, 456, 35, + /* 890 */ 1162, 468, 208, 1067, 469, 1549, 210, 110, 1370, 111, + /* 900 */ 449, 477, 113, 216, 468, 47, 971, 202, 1549, 966, + /* 910 */ 433, 1454, 414, 240, 1578, 471, 1580, 1581, 467, 490, + /* 920 */ 462, 111, 195, 999, 380, 1003, 75, 1578, 471, 1580, + /* 930 */ 1581, 467, 446, 462, 1118, 1673, 1643, 1594, 112, 1705, + /* 940 */ 1642, 1639, 1010, 72, 1577, 1117, 204, 912, 2, 301, + /* 950 */ 161, 113, 137, 131, 111, 1009, 1702, 1127, 114, 379, + /* 960 */ 297, 375, 371, 367, 160, 257, 913, 259, 1593, 940, + /* 970 */ 224, 335, 50, 49, 322, 469, 143, 1495, 1076, 342, + /* 980 */ 145, 316, 350, 349, 351, 468, 355, 1136, 356, 1549, + /* 990 */ 150, 55, 357, 258, 158, 308, 1577, 304, 300, 140, + /* 1000 */ 1135, 358, 153, 359, 1134, 360, 156, 75, 1578, 471, + /* 1010 */ 1580, 1581, 467, 53, 462, 363, 159, 1643, 1133, 571, + /* 1020 */ 1593, 458, 1639, 382, 384, 1402, 79, 466, 163, 267, + /* 1030 */ 138, 1398, 1112, 223, 165, 116, 99, 468, 117, 1400, + /* 1040 */ 1396, 1549, 560, 118, 556, 552, 548, 222, 119, 176, + /* 1050 */ 225, 1577, 157, 415, 152, 423, 154, 1534, 179, 247, + /* 1060 */ 1578, 471, 1580, 1581, 467, 465, 462, 460, 1615, 419, + /* 1070 */ 181, 425, 426, 151, 71, 1593, 416, 217, 1132, 434, + /* 1080 */ 184, 187, 469, 1684, 475, 1674, 431, 1683, 5, 190, + /* 1090 */ 442, 271, 468, 437, 430, 4, 1549, 1223, 104, 1131, + /* 1100 */ 1577, 273, 37, 1720, 457, 454, 197, 481, 1658, 1664, + /* 1110 */ 16, 473, 1624, 474, 126, 1578, 471, 1580, 1581, 467, + /* 1120 */ 194, 462, 130, 1504, 1593, 196, 478, 1503, 226, 212, + /* 1130 */ 281, 469, 479, 214, 480, 62, 427, 1701, 203, 182, + /* 1140 */ 1413, 468, 64, 488, 221, 1549, 1385, 228, 567, 129, + /* 1150 */ 230, 234, 1577, 43, 1084, 235, 177, 232, 450, 1718, + /* 1160 */ 445, 1543, 1542, 75, 1578, 471, 1580, 1581, 467, 296, + /* 1170 */ 462, 1539, 298, 1643, 299, 1100, 1593, 1101, 1640, 141, + /* 1180 */ 303, 1537, 1577, 469, 305, 105, 306, 307, 1536, 309, + /* 1190 */ 1535, 1520, 311, 468, 142, 314, 315, 1549, 1079, 1078, + /* 1200 */ 429, 1514, 1513, 320, 449, 321, 1593, 1512, 1511, 1050, + /* 1210 */ 1488, 1487, 1486, 469, 109, 248, 1578, 471, 1580, 1581, + /* 1220 */ 467, 103, 462, 468, 1485, 1484, 1483, 1549, 1482, 1481, + /* 1230 */ 1480, 1479, 1478, 1577, 1477, 1476, 198, 1650, 444, 1475, + /* 1240 */ 443, 1474, 1473, 1705, 1472, 126, 1578, 471, 1580, 1581, + /* 1250 */ 467, 1471, 462, 1470, 1469, 1468, 137, 1593, 1467, 1466, + /* 1260 */ 1702, 1465, 1052, 1577, 469, 1464, 1463, 1462, 1341, 1528, + /* 1270 */ 1522, 1510, 1501, 1391, 468, 162, 155, 1340, 1549, 1338, + /* 1280 */ 1336, 276, 365, 364, 369, 368, 366, 1593, 905, 1334, + /* 1290 */ 1719, 1577, 370, 374, 466, 372, 248, 1578, 471, 1580, + /* 1300 */ 1581, 467, 1332, 462, 468, 378, 1321, 1320, 1549, 1317, + /* 1310 */ 373, 1393, 1015, 1392, 1018, 1593, 376, 377, 1330, 264, + /* 1320 */ 77, 939, 469, 938, 534, 937, 247, 1578, 471, 1580, + /* 1330 */ 1581, 467, 468, 462, 936, 1616, 1549, 935, 932, 278, + /* 1340 */ 1577, 931, 286, 285, 1325, 1323, 265, 403, 266, 1106, + /* 1350 */ 406, 1316, 1120, 408, 248, 1578, 471, 1580, 1581, 467, + /* 1360 */ 536, 462, 1315, 410, 1593, 1104, 76, 1527, 1113, 1521, + /* 1370 */ 1086, 469, 120, 1509, 417, 180, 1508, 1500, 1577, 57, + /* 1380 */ 183, 468, 3, 13, 1112, 1549, 35, 1112, 280, 1577, + /* 1390 */ 14, 188, 41, 1248, 127, 38, 46, 192, 185, 11, + /* 1400 */ 418, 1241, 1593, 248, 1578, 471, 1580, 1581, 467, 469, + /* 1410 */ 462, 1220, 193, 1593, 20, 1568, 199, 58, 1271, 468, + /* 1420 */ 469, 21, 40, 1549, 1219, 568, 39, 15, 486, 1276, + /* 1430 */ 468, 1270, 274, 1275, 1549, 1274, 275, 1105, 8, 1577, + /* 1440 */ 1116, 236, 1578, 471, 1580, 1581, 467, 136, 462, 461, + /* 1450 */ 17, 1165, 242, 1578, 471, 1580, 1581, 467, 1164, 462, + /* 1460 */ 1183, 27, 10, 1593, 1163, 18, 139, 1150, 206, 470, + /* 1470 */ 469, 19, 476, 1499, 209, 1107, 207, 1246, 1121, 211, + /* 1480 */ 468, 60, 213, 61, 1549, 215, 42, 65, 1122, 1577, + /* 1490 */ 1567, 1000, 489, 288, 218, 487, 1110, 1111, 491, 1124, + /* 1500 */ 1577, 997, 244, 1578, 471, 1580, 1581, 467, 493, 462, + /* 1510 */ 494, 977, 496, 1593, 994, 497, 499, 988, 500, 502, + /* 1520 */ 469, 992, 986, 503, 1593, 509, 991, 990, 989, 1012, + /* 1530 */ 468, 469, 66, 67, 1549, 68, 1008, 1005, 903, 1011, + /* 1540 */ 520, 468, 928, 946, 523, 1549, 926, 220, 1577, 925, + /* 1550 */ 924, 923, 237, 1578, 471, 1580, 1581, 467, 922, 462, + /* 1560 */ 921, 1577, 404, 245, 1578, 471, 1580, 1581, 467, 920, + /* 1570 */ 462, 919, 1593, 943, 941, 916, 915, 412, 914, 469, + /* 1580 */ 911, 910, 909, 908, 1337, 1593, 545, 546, 1335, 468, + /* 1590 */ 547, 174, 469, 1549, 407, 549, 550, 551, 1333, 401, + /* 1600 */ 553, 554, 468, 1331, 555, 173, 1549, 557, 558, 559, + /* 1610 */ 1319, 238, 1578, 471, 1580, 1581, 467, 561, 462, 562, + /* 1620 */ 1318, 1314, 565, 566, 246, 1578, 471, 1580, 1581, 467, + /* 1630 */ 1577, 462, 45, 1108, 229, 44, 569, 570, 1289, 1577, + /* 1640 */ 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1650 */ 1289, 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, + /* 1660 */ 1289, 469, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1670 */ 469, 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, + /* 1680 */ 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, 1577, + /* 1690 */ 1289, 1289, 1289, 239, 1578, 471, 1580, 1581, 467, 1289, + /* 1700 */ 462, 1289, 1589, 1578, 471, 1580, 1581, 467, 1289, 462, + /* 1710 */ 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1720 */ 469, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1730 */ 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, 1577, + /* 1740 */ 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1750 */ 1577, 1289, 1588, 1578, 471, 1580, 1581, 467, 1289, 462, + /* 1760 */ 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1770 */ 469, 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, + /* 1780 */ 468, 469, 1289, 1289, 1549, 1289, 1289, 1289, 1289, 1289, + /* 1790 */ 1289, 468, 1289, 1289, 1289, 1549, 1289, 1289, 1577, 1289, + /* 1800 */ 1289, 1289, 1587, 1578, 471, 1580, 1581, 467, 1289, 462, + /* 1810 */ 1289, 1577, 1289, 255, 1578, 471, 1580, 1581, 467, 1289, + /* 1820 */ 462, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, 469, + /* 1830 */ 1289, 1289, 1289, 1289, 1289, 1593, 1289, 1289, 1289, 468, + /* 1840 */ 1289, 1289, 469, 1549, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1850 */ 1289, 1289, 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, + /* 1860 */ 1289, 254, 1578, 471, 1580, 1581, 467, 1289, 462, 1289, + /* 1870 */ 1289, 1289, 1289, 1289, 256, 1578, 471, 1580, 1581, 467, + /* 1880 */ 1577, 462, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1577, + /* 1890 */ 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1900 */ 1289, 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, + /* 1910 */ 1289, 469, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1920 */ 469, 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, + /* 1930 */ 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, 1289, + /* 1940 */ 1289, 1289, 1289, 253, 1578, 471, 1580, 1581, 467, 1289, + /* 1950 */ 462, 1289, 241, 1578, 471, 1580, 1581, 467, 1289, 462, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 223, 312, 248, 223, 251, 297, 298, 229, 229, 220, - /* 10 */ 264, 268, 12, 13, 325, 223, 270, 238, 329, 223, - /* 20 */ 20, 268, 22, 251, 245, 282, 283, 247, 12, 13, - /* 30 */ 14, 15, 16, 254, 254, 282, 283, 259, 38, 247, - /* 40 */ 268, 20, 231, 229, 264, 268, 254, 229, 268, 223, - /* 50 */ 50, 12, 13, 14, 282, 283, 264, 57, 20, 20, - /* 60 */ 268, 22, 273, 252, 268, 273, 286, 287, 288, 289, - /* 70 */ 290, 291, 254, 293, 74, 229, 49, 38, 286, 287, - /* 80 */ 288, 289, 290, 291, 238, 293, 0, 273, 296, 50, - /* 90 */ 229, 75, 300, 301, 268, 74, 57, 97, 228, 238, - /* 100 */ 254, 312, 12, 13, 312, 222, 245, 224, 290, 109, - /* 110 */ 20, 331, 22, 74, 325, 254, 0, 325, 329, 249, - /* 120 */ 2, 329, 304, 305, 306, 307, 312, 309, 38, 0, - /* 130 */ 12, 13, 14, 15, 16, 49, 97, 0, 247, 325, - /* 140 */ 50, 20, 20, 329, 253, 247, 146, 57, 109, 258, - /* 150 */ 21, 246, 254, 24, 25, 26, 27, 28, 29, 30, - /* 160 */ 31, 32, 257, 264, 74, 49, 267, 167, 168, 270, - /* 170 */ 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, - /* 180 */ 180, 181, 182, 183, 184, 146, 288, 97, 136, 52, - /* 190 */ 145, 54, 147, 226, 227, 58, 74, 197, 61, 109, - /* 200 */ 63, 64, 197, 66, 155, 156, 167, 168, 159, 170, - /* 210 */ 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, - /* 220 */ 181, 182, 183, 184, 229, 229, 229, 229, 229, 38, - /* 230 */ 12, 13, 71, 238, 238, 238, 146, 238, 20, 20, - /* 240 */ 22, 22, 197, 12, 13, 14, 15, 16, 57, 254, - /* 250 */ 254, 254, 254, 254, 202, 203, 38, 167, 168, 40, - /* 260 */ 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, - /* 270 */ 180, 181, 182, 183, 184, 57, 12, 13, 14, 233, - /* 280 */ 234, 50, 67, 47, 20, 223, 22, 0, 290, 235, - /* 290 */ 21, 237, 74, 24, 25, 26, 27, 28, 29, 30, - /* 300 */ 31, 32, 38, 305, 306, 307, 75, 309, 239, 73, - /* 310 */ 20, 52, 76, 54, 83, 97, 247, 58, 197, 197, - /* 320 */ 61, 57, 63, 64, 255, 66, 167, 109, 113, 114, - /* 330 */ 268, 273, 35, 12, 13, 14, 15, 16, 74, 52, - /* 340 */ 53, 54, 55, 56, 22, 58, 59, 60, 61, 62, - /* 350 */ 63, 64, 65, 66, 67, 68, 69, 70, 61, 231, - /* 360 */ 38, 97, 65, 132, 146, 206, 207, 208, 209, 210, - /* 370 */ 312, 232, 244, 109, 77, 236, 79, 80, 217, 82, - /* 380 */ 252, 150, 20, 325, 87, 167, 168, 329, 170, 171, - /* 390 */ 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, - /* 400 */ 182, 183, 184, 229, 12, 13, 233, 234, 229, 71, - /* 410 */ 146, 20, 20, 75, 22, 197, 185, 186, 187, 188, - /* 420 */ 189, 190, 191, 192, 193, 194, 233, 234, 254, 50, - /* 430 */ 38, 167, 168, 254, 170, 171, 172, 173, 174, 175, - /* 440 */ 176, 177, 178, 179, 180, 181, 182, 183, 184, 57, - /* 450 */ 12, 13, 273, 229, 273, 239, 228, 22, 20, 138, - /* 460 */ 22, 247, 238, 247, 290, 20, 74, 253, 38, 290, - /* 470 */ 242, 255, 258, 38, 1, 2, 38, 249, 254, 305, - /* 480 */ 306, 307, 71, 309, 305, 306, 307, 57, 309, 97, - /* 490 */ 0, 312, 57, 312, 223, 57, 12, 13, 14, 15, - /* 500 */ 16, 109, 0, 20, 325, 223, 325, 49, 329, 223, - /* 510 */ 329, 223, 74, 88, 89, 90, 91, 92, 93, 94, - /* 520 */ 95, 96, 97, 98, 22, 100, 101, 102, 103, 104, - /* 530 */ 105, 18, 97, 20, 256, 97, 229, 85, 146, 268, - /* 540 */ 27, 263, 264, 30, 109, 238, 167, 109, 75, 223, - /* 550 */ 268, 61, 39, 61, 268, 65, 268, 65, 0, 167, - /* 560 */ 168, 254, 170, 171, 172, 173, 174, 175, 176, 177, - /* 570 */ 178, 179, 180, 181, 182, 183, 184, 87, 254, 87, - /* 580 */ 22, 146, 229, 138, 146, 261, 12, 13, 14, 15, - /* 590 */ 16, 238, 240, 247, 268, 243, 106, 107, 108, 253, - /* 600 */ 110, 57, 167, 168, 258, 167, 168, 254, 170, 171, - /* 610 */ 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, - /* 620 */ 182, 183, 184, 223, 50, 112, 215, 223, 115, 116, - /* 630 */ 117, 118, 119, 248, 121, 122, 123, 124, 125, 126, - /* 640 */ 127, 128, 129, 130, 131, 3, 229, 247, 12, 13, - /* 650 */ 14, 15, 16, 229, 254, 238, 19, 83, 132, 239, - /* 660 */ 229, 248, 238, 223, 264, 233, 234, 247, 268, 238, - /* 670 */ 33, 254, 268, 36, 223, 255, 150, 223, 254, 42, - /* 680 */ 223, 44, 45, 46, 47, 254, 286, 287, 288, 289, - /* 690 */ 290, 291, 20, 293, 195, 196, 296, 43, 247, 284, - /* 700 */ 300, 301, 302, 223, 223, 254, 132, 247, 268, 223, - /* 710 */ 73, 185, 223, 76, 314, 264, 2, 248, 258, 268, - /* 720 */ 320, 321, 268, 308, 150, 268, 12, 13, 14, 15, - /* 730 */ 16, 14, 15, 16, 247, 223, 4, 286, 287, 288, - /* 740 */ 289, 290, 291, 106, 293, 258, 284, 296, 268, 268, - /* 750 */ 14, 300, 301, 302, 268, 223, 20, 268, 4, 185, - /* 760 */ 186, 187, 188, 189, 190, 191, 192, 193, 194, 223, - /* 770 */ 308, 134, 321, 19, 137, 139, 0, 0, 284, 247, - /* 780 */ 268, 12, 13, 14, 15, 16, 254, 33, 14, 152, - /* 790 */ 36, 154, 223, 248, 20, 41, 264, 247, 142, 22, - /* 800 */ 268, 47, 308, 78, 78, 255, 81, 81, 157, 158, - /* 810 */ 138, 223, 36, 78, 268, 248, 81, 161, 286, 287, - /* 820 */ 288, 289, 290, 291, 248, 293, 78, 73, 296, 81, - /* 830 */ 76, 276, 300, 301, 302, 247, 0, 268, 21, 224, - /* 840 */ 18, 223, 254, 311, 75, 23, 1, 2, 236, 71, - /* 850 */ 196, 34, 264, 75, 218, 213, 268, 35, 71, 71, - /* 860 */ 182, 183, 75, 75, 71, 247, 332, 257, 75, 71, - /* 870 */ 48, 223, 254, 75, 286, 287, 288, 289, 290, 291, - /* 880 */ 74, 293, 264, 323, 296, 38, 268, 280, 300, 301, - /* 890 */ 302, 226, 86, 38, 71, 247, 317, 61, 75, 311, - /* 900 */ 74, 65, 254, 310, 286, 287, 288, 289, 290, 291, - /* 910 */ 84, 293, 264, 71, 296, 285, 268, 75, 300, 301, - /* 920 */ 302, 273, 247, 87, 71, 326, 223, 71, 75, 311, - /* 930 */ 198, 75, 313, 111, 286, 287, 288, 289, 290, 291, - /* 940 */ 20, 293, 106, 107, 108, 71, 110, 71, 71, 75, - /* 950 */ 247, 75, 75, 229, 36, 281, 109, 254, 144, 274, - /* 960 */ 312, 139, 140, 141, 109, 143, 38, 264, 120, 229, - /* 970 */ 148, 268, 233, 325, 71, 229, 223, 329, 75, 71, - /* 980 */ 262, 71, 160, 75, 162, 75, 164, 165, 166, 286, - /* 990 */ 287, 288, 289, 290, 291, 132, 293, 260, 71, 296, - /* 1000 */ 247, 260, 75, 300, 301, 20, 278, 254, 229, 264, - /* 1010 */ 20, 231, 231, 254, 272, 20, 265, 264, 229, 197, - /* 1020 */ 33, 268, 231, 36, 223, 231, 231, 20, 225, 42, - /* 1030 */ 247, 44, 45, 46, 47, 223, 229, 247, 225, 286, - /* 1040 */ 287, 288, 289, 290, 291, 57, 293, 247, 247, 296, - /* 1050 */ 268, 247, 247, 300, 301, 254, 247, 247, 247, 247, - /* 1060 */ 73, 228, 247, 76, 278, 264, 254, 247, 247, 268, - /* 1070 */ 153, 272, 264, 277, 228, 20, 264, 228, 254, 322, - /* 1080 */ 268, 265, 285, 205, 204, 322, 268, 286, 287, 288, - /* 1090 */ 289, 290, 291, 292, 293, 294, 295, 269, 286, 287, - /* 1100 */ 288, 289, 290, 291, 269, 293, 223, 268, 12, 13, - /* 1110 */ 212, 268, 211, 254, 200, 199, 284, 20, 22, 196, - /* 1120 */ 133, 319, 135, 120, 137, 219, 214, 216, 74, 269, - /* 1130 */ 247, 269, 303, 268, 38, 316, 223, 254, 268, 299, - /* 1140 */ 315, 154, 330, 331, 223, 318, 135, 264, 266, 268, - /* 1150 */ 254, 268, 265, 57, 228, 74, 254, 228, 243, 328, - /* 1160 */ 247, 327, 237, 333, 229, 228, 225, 254, 247, 286, - /* 1170 */ 287, 288, 289, 290, 291, 254, 293, 264, 279, 296, - /* 1180 */ 250, 268, 275, 241, 301, 264, 241, 230, 221, 268, - /* 1190 */ 0, 223, 271, 97, 0, 64, 0, 38, 163, 286, - /* 1200 */ 287, 288, 289, 290, 291, 109, 293, 286, 287, 288, - /* 1210 */ 289, 290, 291, 38, 293, 247, 38, 38, 0, 38, - /* 1220 */ 163, 38, 254, 163, 0, 38, 0, 38, 0, 74, - /* 1230 */ 150, 149, 264, 109, 146, 0, 268, 324, 86, 271, - /* 1240 */ 0, 223, 146, 142, 53, 0, 0, 0, 0, 0, - /* 1250 */ 0, 0, 0, 0, 286, 287, 288, 289, 290, 291, - /* 1260 */ 0, 293, 0, 167, 0, 247, 0, 0, 0, 0, - /* 1270 */ 0, 0, 254, 120, 0, 0, 0, 0, 0, 223, - /* 1280 */ 0, 22, 264, 0, 0, 0, 268, 0, 0, 0, - /* 1290 */ 223, 0, 0, 0, 0, 51, 0, 43, 0, 38, - /* 1300 */ 43, 36, 0, 247, 286, 287, 288, 289, 290, 291, - /* 1310 */ 254, 293, 0, 295, 247, 43, 43, 38, 36, 38, - /* 1320 */ 264, 254, 36, 0, 268, 0, 0, 271, 38, 0, - /* 1330 */ 0, 264, 22, 36, 71, 268, 81, 38, 271, 0, - /* 1340 */ 0, 83, 286, 287, 288, 289, 290, 291, 223, 293, - /* 1350 */ 0, 43, 38, 286, 287, 288, 289, 290, 291, 38, - /* 1360 */ 293, 223, 38, 24, 25, 26, 27, 28, 29, 30, - /* 1370 */ 31, 32, 247, 38, 71, 38, 38, 22, 0, 254, - /* 1380 */ 22, 39, 0, 22, 38, 247, 0, 22, 0, 264, - /* 1390 */ 22, 20, 254, 268, 0, 38, 0, 22, 0, 223, - /* 1400 */ 0, 0, 264, 43, 71, 74, 268, 151, 223, 201, - /* 1410 */ 74, 286, 287, 288, 289, 290, 291, 74, 293, 71, - /* 1420 */ 71, 201, 75, 247, 286, 287, 288, 289, 290, 291, - /* 1430 */ 254, 293, 247, 138, 75, 71, 135, 74, 138, 254, - /* 1440 */ 264, 75, 138, 74, 268, 71, 223, 75, 86, 264, - /* 1450 */ 133, 75, 71, 268, 75, 71, 86, 4, 38, 86, - /* 1460 */ 38, 223, 286, 287, 288, 289, 290, 291, 38, 293, - /* 1470 */ 247, 286, 287, 288, 289, 290, 291, 254, 293, 38, - /* 1480 */ 38, 38, 2, 167, 71, 247, 75, 264, 74, 86, - /* 1490 */ 75, 268, 254, 74, 86, 22, 75, 74, 74, 74, - /* 1500 */ 195, 169, 264, 75, 75, 0, 268, 201, 74, 286, - /* 1510 */ 287, 288, 289, 290, 291, 43, 293, 84, 74, 136, - /* 1520 */ 74, 74, 133, 86, 286, 287, 288, 289, 290, 291, - /* 1530 */ 223, 293, 22, 74, 86, 85, 75, 38, 223, 38, - /* 1540 */ 74, 38, 75, 74, 38, 75, 75, 38, 74, 74, - /* 1550 */ 38, 75, 22, 74, 247, 99, 99, 99, 74, 99, - /* 1560 */ 38, 254, 247, 109, 87, 74, 74, 38, 223, 254, - /* 1570 */ 22, 264, 51, 50, 57, 268, 72, 38, 71, 264, - /* 1580 */ 38, 38, 38, 268, 38, 38, 22, 38, 38, 57, - /* 1590 */ 38, 38, 247, 286, 287, 288, 289, 290, 291, 254, - /* 1600 */ 293, 286, 287, 288, 289, 290, 291, 0, 293, 264, - /* 1610 */ 38, 38, 38, 268, 38, 38, 223, 38, 36, 0, - /* 1620 */ 0, 43, 38, 36, 43, 36, 38, 0, 43, 38, - /* 1630 */ 36, 286, 287, 288, 289, 290, 291, 0, 293, 43, - /* 1640 */ 247, 38, 37, 0, 0, 22, 21, 254, 22, 20, - /* 1650 */ 22, 21, 334, 334, 223, 334, 334, 264, 334, 334, - /* 1660 */ 334, 268, 334, 223, 334, 334, 334, 334, 334, 334, - /* 1670 */ 334, 334, 334, 334, 334, 334, 334, 334, 247, 286, - /* 1680 */ 287, 288, 289, 290, 291, 254, 293, 247, 334, 334, - /* 1690 */ 334, 334, 334, 334, 254, 264, 334, 334, 334, 268, - /* 1700 */ 334, 223, 334, 334, 264, 334, 334, 334, 268, 334, - /* 1710 */ 334, 334, 334, 334, 334, 334, 223, 286, 287, 288, - /* 1720 */ 289, 290, 291, 334, 293, 247, 286, 287, 288, 289, - /* 1730 */ 290, 291, 254, 293, 334, 334, 334, 334, 334, 334, - /* 1740 */ 247, 334, 264, 334, 334, 334, 268, 254, 334, 334, - /* 1750 */ 334, 334, 334, 334, 223, 334, 334, 264, 334, 334, - /* 1760 */ 334, 268, 334, 334, 286, 287, 288, 289, 290, 291, - /* 1770 */ 334, 293, 334, 334, 334, 334, 334, 334, 247, 286, - /* 1780 */ 287, 288, 289, 290, 291, 254, 293, 334, 334, 334, - /* 1790 */ 334, 334, 223, 334, 334, 264, 334, 334, 334, 268, - /* 1800 */ 334, 334, 334, 334, 334, 334, 334, 334, 334, 334, - /* 1810 */ 334, 334, 334, 334, 334, 334, 247, 286, 287, 288, - /* 1820 */ 289, 290, 291, 254, 293, 334, 334, 334, 334, 334, - /* 1830 */ 334, 334, 334, 264, 334, 334, 334, 268, 334, 334, - /* 1840 */ 334, 334, 334, 334, 334, 334, 334, 334, 334, 334, - /* 1850 */ 334, 334, 334, 334, 334, 286, 287, 288, 289, 290, - /* 1860 */ 291, 334, 293, 334, 334, 334, 334, 334, 334, 334, - /* 1870 */ 334, 334, + /* 0 */ 253, 249, 253, 266, 225, 231, 269, 231, 256, 272, + /* 10 */ 222, 250, 12, 13, 240, 225, 224, 270, 226, 270, + /* 20 */ 20, 247, 22, 12, 13, 14, 15, 16, 249, 270, + /* 30 */ 256, 284, 285, 284, 285, 256, 234, 261, 38, 249, + /* 40 */ 238, 314, 290, 284, 285, 266, 256, 0, 231, 270, + /* 50 */ 50, 12, 13, 14, 327, 233, 266, 57, 331, 20, + /* 60 */ 270, 22, 20, 275, 275, 275, 225, 288, 289, 290, + /* 70 */ 291, 292, 293, 256, 295, 75, 254, 38, 288, 289, + /* 80 */ 290, 291, 292, 293, 237, 295, 239, 258, 298, 50, + /* 90 */ 249, 20, 302, 303, 265, 266, 57, 256, 98, 299, + /* 100 */ 300, 49, 314, 314, 314, 326, 231, 266, 61, 292, + /* 110 */ 110, 270, 65, 231, 75, 327, 327, 327, 20, 331, + /* 120 */ 331, 331, 240, 306, 307, 308, 309, 231, 311, 288, + /* 130 */ 289, 290, 291, 292, 293, 88, 295, 98, 256, 298, + /* 140 */ 233, 47, 231, 302, 303, 304, 75, 38, 148, 110, + /* 150 */ 275, 240, 256, 246, 107, 108, 109, 316, 111, 20, + /* 160 */ 20, 254, 20, 322, 323, 138, 57, 256, 74, 169, + /* 170 */ 170, 77, 172, 173, 174, 175, 176, 177, 178, 179, + /* 180 */ 180, 181, 182, 183, 184, 185, 186, 148, 292, 314, + /* 190 */ 249, 12, 13, 14, 15, 16, 255, 1, 2, 199, + /* 200 */ 75, 260, 327, 307, 308, 309, 331, 311, 169, 170, + /* 210 */ 85, 172, 173, 174, 175, 176, 177, 178, 179, 180, + /* 220 */ 181, 182, 183, 184, 185, 186, 12, 13, 199, 50, + /* 230 */ 242, 204, 205, 245, 20, 18, 22, 20, 225, 52, + /* 240 */ 230, 54, 49, 249, 27, 58, 230, 30, 61, 255, + /* 250 */ 63, 64, 38, 66, 260, 76, 39, 147, 71, 149, + /* 260 */ 244, 251, 249, 84, 50, 12, 13, 251, 4, 256, + /* 270 */ 199, 57, 76, 20, 2, 22, 286, 157, 158, 266, + /* 280 */ 169, 161, 2, 270, 12, 13, 14, 15, 16, 75, + /* 290 */ 286, 38, 12, 13, 14, 15, 16, 14, 15, 16, + /* 300 */ 310, 288, 289, 290, 291, 292, 293, 248, 295, 199, + /* 310 */ 57, 298, 98, 134, 310, 302, 303, 304, 259, 208, + /* 320 */ 209, 210, 211, 212, 110, 275, 313, 0, 75, 0, + /* 330 */ 113, 152, 20, 116, 117, 118, 119, 120, 199, 122, + /* 340 */ 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + /* 350 */ 133, 98, 4, 24, 25, 26, 27, 28, 29, 30, + /* 360 */ 31, 32, 148, 110, 314, 314, 187, 188, 189, 190, + /* 370 */ 191, 192, 193, 194, 195, 196, 49, 327, 327, 225, + /* 380 */ 20, 331, 331, 169, 170, 43, 172, 173, 174, 175, + /* 390 */ 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, + /* 400 */ 186, 148, 57, 20, 72, 12, 13, 14, 15, 16, + /* 410 */ 21, 228, 229, 24, 25, 26, 27, 28, 29, 30, + /* 420 */ 31, 32, 169, 170, 270, 172, 173, 174, 175, 176, + /* 430 */ 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, + /* 440 */ 12, 13, 14, 50, 0, 72, 225, 241, 20, 76, + /* 450 */ 22, 241, 199, 231, 266, 249, 35, 61, 75, 249, + /* 460 */ 272, 65, 240, 257, 200, 21, 38, 257, 24, 25, + /* 470 */ 26, 27, 28, 29, 30, 31, 32, 84, 256, 12, + /* 480 */ 13, 3, 61, 0, 88, 57, 65, 20, 225, 22, + /* 490 */ 225, 270, 12, 13, 14, 15, 16, 231, 231, 78, + /* 500 */ 140, 80, 81, 75, 83, 38, 240, 240, 86, 88, + /* 510 */ 235, 236, 241, 247, 249, 12, 13, 14, 15, 16, + /* 520 */ 249, 256, 256, 256, 57, 231, 98, 134, 257, 225, + /* 530 */ 249, 266, 256, 270, 240, 270, 255, 231, 110, 263, + /* 540 */ 198, 260, 75, 134, 61, 152, 240, 199, 65, 217, + /* 550 */ 256, 235, 236, 288, 289, 290, 291, 292, 293, 249, + /* 560 */ 295, 152, 256, 298, 0, 98, 20, 302, 303, 304, + /* 570 */ 260, 88, 67, 250, 270, 286, 148, 110, 313, 76, + /* 580 */ 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, + /* 590 */ 107, 108, 109, 225, 111, 0, 187, 169, 170, 310, + /* 600 */ 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, + /* 610 */ 182, 183, 184, 185, 186, 148, 52, 249, 54, 114, + /* 620 */ 115, 141, 58, 225, 256, 61, 79, 63, 64, 82, + /* 630 */ 66, 36, 235, 236, 266, 71, 169, 170, 270, 172, + /* 640 */ 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, + /* 650 */ 183, 184, 185, 186, 12, 13, 288, 289, 290, 291, + /* 660 */ 292, 293, 20, 295, 22, 225, 298, 0, 270, 231, + /* 670 */ 302, 303, 304, 12, 13, 14, 15, 16, 240, 231, + /* 680 */ 38, 225, 249, 231, 231, 20, 140, 22, 240, 249, + /* 690 */ 257, 323, 240, 215, 256, 250, 256, 235, 236, 57, + /* 700 */ 220, 14, 231, 22, 256, 40, 266, 20, 256, 256, + /* 710 */ 270, 240, 231, 225, 225, 144, 49, 75, 0, 38, + /* 720 */ 225, 240, 225, 225, 225, 250, 270, 256, 288, 289, + /* 730 */ 290, 291, 292, 293, 163, 295, 79, 256, 298, 82, + /* 740 */ 98, 72, 302, 303, 304, 292, 12, 13, 14, 15, + /* 750 */ 16, 225, 110, 313, 225, 225, 250, 225, 270, 270, + /* 760 */ 307, 308, 309, 225, 311, 270, 50, 270, 270, 270, + /* 770 */ 52, 53, 54, 55, 56, 250, 58, 59, 60, 61, + /* 780 */ 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, + /* 790 */ 148, 14, 265, 266, 197, 198, 270, 20, 250, 270, + /* 800 */ 270, 140, 270, 225, 159, 160, 225, 225, 270, 72, + /* 810 */ 76, 169, 170, 76, 172, 173, 174, 175, 176, 177, + /* 820 */ 178, 179, 180, 181, 182, 183, 184, 185, 186, 75, + /* 830 */ 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, + /* 840 */ 99, 87, 101, 102, 103, 104, 105, 106, 270, 18, + /* 850 */ 225, 270, 270, 79, 23, 0, 82, 79, 0, 21, + /* 860 */ 82, 0, 38, 225, 72, 72, 35, 278, 76, 76, + /* 870 */ 1, 2, 34, 38, 249, 184, 185, 22, 72, 48, + /* 880 */ 22, 256, 76, 22, 226, 169, 72, 249, 219, 72, + /* 890 */ 76, 266, 72, 76, 256, 270, 76, 72, 238, 72, + /* 900 */ 275, 76, 72, 76, 266, 72, 76, 334, 270, 76, + /* 910 */ 325, 259, 282, 288, 289, 290, 291, 292, 293, 72, + /* 920 */ 295, 72, 319, 76, 228, 76, 288, 289, 290, 291, + /* 930 */ 292, 293, 312, 295, 110, 287, 298, 249, 72, 314, + /* 940 */ 302, 303, 76, 112, 225, 110, 328, 38, 315, 36, + /* 950 */ 33, 72, 327, 36, 72, 76, 331, 20, 76, 42, + /* 960 */ 231, 44, 45, 46, 47, 283, 57, 235, 249, 38, + /* 970 */ 276, 231, 141, 142, 143, 256, 145, 231, 146, 264, + /* 980 */ 121, 150, 134, 262, 262, 266, 231, 20, 280, 270, + /* 990 */ 233, 74, 266, 162, 77, 164, 225, 166, 167, 168, + /* 1000 */ 20, 274, 233, 256, 20, 267, 233, 288, 289, 290, + /* 1010 */ 291, 292, 293, 233, 295, 231, 233, 298, 20, 19, + /* 1020 */ 249, 302, 303, 227, 249, 249, 231, 256, 249, 227, + /* 1030 */ 199, 249, 57, 33, 249, 249, 36, 266, 249, 249, + /* 1040 */ 249, 270, 42, 249, 44, 45, 46, 47, 249, 230, + /* 1050 */ 280, 225, 135, 155, 137, 274, 139, 270, 230, 288, + /* 1060 */ 289, 290, 291, 292, 293, 294, 295, 296, 297, 266, + /* 1070 */ 230, 256, 267, 156, 74, 249, 279, 77, 20, 207, + /* 1080 */ 230, 271, 256, 324, 206, 287, 270, 324, 214, 271, + /* 1090 */ 213, 270, 266, 270, 202, 201, 270, 198, 256, 20, + /* 1100 */ 225, 221, 121, 335, 218, 216, 305, 107, 286, 321, + /* 1110 */ 75, 270, 301, 270, 288, 289, 290, 291, 292, 293, + /* 1120 */ 320, 295, 318, 271, 249, 317, 137, 271, 245, 256, + /* 1130 */ 270, 256, 268, 230, 267, 230, 136, 330, 329, 139, + /* 1140 */ 256, 266, 75, 252, 230, 270, 239, 231, 227, 281, + /* 1150 */ 232, 243, 225, 277, 154, 243, 156, 223, 332, 333, + /* 1160 */ 231, 0, 0, 288, 289, 290, 291, 292, 293, 64, + /* 1170 */ 295, 0, 38, 298, 165, 38, 249, 38, 303, 38, + /* 1180 */ 165, 0, 225, 256, 38, 256, 38, 165, 0, 38, + /* 1190 */ 0, 0, 38, 266, 75, 152, 151, 270, 110, 148, + /* 1200 */ 273, 0, 0, 53, 275, 144, 249, 0, 0, 87, + /* 1210 */ 0, 0, 0, 256, 121, 288, 289, 290, 291, 292, + /* 1220 */ 293, 292, 295, 266, 0, 0, 0, 270, 0, 0, + /* 1230 */ 0, 0, 0, 225, 0, 0, 307, 308, 309, 0, + /* 1240 */ 311, 0, 0, 314, 0, 288, 289, 290, 291, 292, + /* 1250 */ 293, 0, 295, 0, 0, 0, 327, 249, 0, 0, + /* 1260 */ 331, 0, 22, 225, 256, 0, 0, 0, 0, 0, + /* 1270 */ 0, 0, 0, 0, 266, 82, 43, 0, 270, 0, + /* 1280 */ 0, 273, 36, 38, 36, 38, 43, 249, 51, 0, + /* 1290 */ 333, 225, 43, 43, 256, 38, 288, 289, 290, 291, + /* 1300 */ 292, 293, 0, 295, 266, 43, 0, 0, 270, 0, + /* 1310 */ 36, 0, 22, 0, 38, 249, 38, 36, 0, 22, + /* 1320 */ 84, 38, 256, 38, 72, 38, 288, 289, 290, 291, + /* 1330 */ 292, 293, 266, 295, 38, 297, 270, 38, 38, 273, + /* 1340 */ 225, 38, 12, 13, 0, 0, 22, 39, 22, 22, + /* 1350 */ 38, 0, 22, 22, 288, 289, 290, 291, 292, 293, + /* 1360 */ 72, 295, 0, 22, 249, 38, 20, 0, 38, 0, + /* 1370 */ 38, 256, 153, 0, 22, 137, 0, 0, 225, 75, + /* 1380 */ 43, 266, 72, 203, 57, 270, 72, 57, 273, 225, + /* 1390 */ 203, 76, 72, 76, 75, 197, 140, 75, 135, 203, + /* 1400 */ 140, 76, 249, 288, 289, 290, 291, 292, 293, 256, + /* 1410 */ 295, 76, 72, 249, 75, 87, 87, 75, 38, 266, + /* 1420 */ 256, 72, 140, 270, 76, 98, 72, 72, 98, 76, + /* 1430 */ 266, 38, 38, 38, 270, 38, 38, 110, 2, 225, + /* 1440 */ 110, 288, 289, 290, 291, 292, 293, 87, 295, 75, + /* 1450 */ 72, 76, 288, 289, 290, 291, 292, 293, 76, 295, + /* 1460 */ 169, 75, 75, 249, 76, 75, 87, 22, 87, 171, + /* 1470 */ 256, 75, 138, 0, 75, 148, 76, 76, 148, 75, + /* 1480 */ 266, 75, 43, 75, 270, 135, 75, 85, 22, 225, + /* 1490 */ 87, 76, 38, 38, 87, 86, 169, 170, 75, 169, + /* 1500 */ 225, 76, 288, 289, 290, 291, 292, 293, 38, 295, + /* 1510 */ 75, 22, 38, 249, 76, 75, 38, 76, 75, 38, + /* 1520 */ 256, 100, 76, 75, 249, 88, 100, 100, 100, 38, + /* 1530 */ 266, 256, 75, 75, 270, 75, 38, 22, 51, 110, + /* 1540 */ 50, 266, 38, 57, 73, 270, 38, 72, 225, 38, + /* 1550 */ 38, 38, 288, 289, 290, 291, 292, 293, 38, 295, + /* 1560 */ 38, 225, 4, 288, 289, 290, 291, 292, 293, 38, + /* 1570 */ 295, 22, 249, 57, 38, 38, 38, 19, 38, 256, + /* 1580 */ 38, 38, 38, 38, 0, 249, 38, 36, 0, 266, + /* 1590 */ 43, 33, 256, 270, 36, 38, 36, 43, 0, 41, + /* 1600 */ 38, 36, 266, 0, 43, 47, 270, 38, 36, 43, + /* 1610 */ 0, 288, 289, 290, 291, 292, 293, 38, 295, 37, + /* 1620 */ 0, 0, 22, 21, 288, 289, 290, 291, 292, 293, + /* 1630 */ 225, 295, 74, 22, 22, 77, 21, 20, 336, 225, + /* 1640 */ 336, 336, 336, 336, 336, 336, 336, 336, 336, 336, + /* 1650 */ 336, 336, 336, 336, 249, 336, 336, 336, 336, 336, + /* 1660 */ 336, 256, 336, 249, 336, 336, 336, 336, 336, 336, + /* 1670 */ 256, 266, 336, 336, 336, 270, 336, 336, 336, 336, + /* 1680 */ 266, 336, 336, 336, 270, 336, 336, 336, 336, 225, + /* 1690 */ 336, 336, 336, 288, 289, 290, 291, 292, 293, 336, + /* 1700 */ 295, 336, 288, 289, 290, 291, 292, 293, 336, 295, + /* 1710 */ 336, 336, 336, 249, 336, 336, 336, 336, 336, 336, + /* 1720 */ 256, 336, 336, 336, 336, 336, 336, 336, 336, 336, + /* 1730 */ 266, 336, 336, 336, 270, 336, 336, 336, 336, 225, + /* 1740 */ 336, 336, 336, 336, 336, 336, 336, 336, 336, 336, + /* 1750 */ 225, 336, 288, 289, 290, 291, 292, 293, 336, 295, + /* 1760 */ 336, 336, 336, 249, 336, 336, 336, 336, 336, 336, + /* 1770 */ 256, 336, 336, 336, 249, 336, 336, 336, 336, 336, + /* 1780 */ 266, 256, 336, 336, 270, 336, 336, 336, 336, 336, + /* 1790 */ 336, 266, 336, 336, 336, 270, 336, 336, 225, 336, + /* 1800 */ 336, 336, 288, 289, 290, 291, 292, 293, 336, 295, + /* 1810 */ 336, 225, 336, 288, 289, 290, 291, 292, 293, 336, + /* 1820 */ 295, 336, 249, 336, 336, 336, 336, 336, 336, 256, + /* 1830 */ 336, 336, 336, 336, 336, 249, 336, 336, 336, 266, + /* 1840 */ 336, 336, 256, 270, 336, 336, 336, 336, 336, 336, + /* 1850 */ 336, 336, 266, 336, 336, 336, 270, 336, 336, 336, + /* 1860 */ 336, 288, 289, 290, 291, 292, 293, 336, 295, 336, + /* 1870 */ 336, 336, 336, 336, 288, 289, 290, 291, 292, 293, + /* 1880 */ 225, 295, 336, 336, 336, 336, 336, 336, 336, 225, + /* 1890 */ 336, 336, 336, 336, 336, 336, 336, 336, 336, 336, + /* 1900 */ 336, 336, 336, 336, 249, 336, 336, 336, 336, 336, + /* 1910 */ 336, 256, 336, 249, 336, 336, 336, 336, 336, 336, + /* 1920 */ 256, 266, 336, 336, 336, 270, 336, 336, 336, 336, + /* 1930 */ 266, 336, 336, 336, 270, 336, 336, 336, 336, 336, + /* 1940 */ 336, 336, 336, 288, 289, 290, 291, 292, 293, 336, + /* 1950 */ 295, 336, 288, 289, 290, 291, 292, 293, 336, 295, }; -#define YY_SHIFT_COUNT (567) +#define YY_SHIFT_COUNT (571) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1644) +#define YY_SHIFT_MAX (1621) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 822, 0, 39, 90, 90, 90, 90, 218, 90, 90, - /* 10 */ 264, 392, 438, 392, 392, 392, 392, 392, 392, 392, - /* 20 */ 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, - /* 30 */ 392, 392, 392, 392, 392, 392, 122, 21, 21, 21, - /* 40 */ 121, 1096, 1096, 45, 38, 38, 5, 1096, 38, 38, - /* 50 */ 38, 38, 38, 38, 27, 38, 290, 362, 5, 391, - /* 60 */ 290, 38, 38, 290, 38, 290, 391, 290, 290, 38, - /* 70 */ 458, 513, 231, 574, 574, 269, 435, 297, 435, 435, - /* 80 */ 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, - /* 90 */ 435, 435, 435, 435, 435, 435, 435, 259, 219, 86, - /* 100 */ 191, 191, 445, 445, 445, 116, 191, 191, 483, 391, - /* 110 */ 290, 290, 290, 452, 544, 425, 425, 425, 425, 425, - /* 120 */ 425, 425, 637, 129, 137, 636, 159, 492, 49, 52, - /* 130 */ 322, 672, 499, 654, 499, 736, 642, 732, 774, 920, - /* 140 */ 918, 928, 814, 920, 920, 848, 863, 863, 920, 985, - /* 150 */ 27, 391, 990, 27, 483, 995, 27, 27, 920, 27, - /* 160 */ 1007, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 170 */ 290, 290, 920, 1007, 988, 985, 458, 917, 391, 990, - /* 180 */ 458, 483, 995, 458, 1055, 878, 880, 988, 878, 880, - /* 190 */ 988, 988, 898, 901, 914, 916, 923, 483, 1097, 1003, - /* 200 */ 906, 911, 912, 1054, 290, 880, 988, 988, 880, 988, - /* 210 */ 1011, 483, 995, 458, 452, 458, 483, 1081, 544, 920, - /* 220 */ 458, 1007, 1863, 1863, 1863, 1863, 1863, 1863, 287, 987, - /* 230 */ 1339, 754, 490, 836, 16, 118, 714, 321, 769, 484, - /* 240 */ 484, 484, 484, 484, 484, 484, 484, 236, 215, 473, - /* 250 */ 526, 717, 717, 717, 717, 776, 656, 338, 725, 726, - /* 260 */ 735, 748, 502, 558, 777, 817, 651, 778, 787, 788, - /* 270 */ 845, 678, 411, 161, 793, 379, 798, 806, 823, 842, - /* 280 */ 853, 856, 874, 847, 855, 876, 877, 903, 908, 910, - /* 290 */ 927, 826, 430, 1190, 1194, 1131, 1196, 1159, 1035, 1175, - /* 300 */ 1178, 1179, 1057, 1218, 1181, 1183, 1060, 1224, 1187, 1226, - /* 310 */ 1189, 1228, 1155, 1080, 1082, 1124, 1088, 1235, 1240, 1191, - /* 320 */ 1101, 1245, 1246, 1152, 1247, 1248, 1249, 1250, 1251, 1252, - /* 330 */ 1253, 1260, 1262, 1264, 1266, 1267, 1268, 1269, 1270, 1271, - /* 340 */ 1153, 1274, 1275, 1276, 1277, 1278, 1280, 1259, 1283, 1284, - /* 350 */ 1285, 1287, 1288, 1289, 1291, 1292, 1293, 1254, 1294, 1244, - /* 360 */ 1296, 1298, 1261, 1265, 1257, 1302, 1279, 1282, 1272, 1312, - /* 370 */ 1281, 1286, 1273, 1323, 1290, 1297, 1308, 1325, 1326, 1329, - /* 380 */ 1330, 1258, 1255, 1299, 1263, 1303, 1310, 1340, 1314, 1321, - /* 390 */ 1324, 1335, 1263, 1303, 1337, 1338, 1350, 1355, 1378, 1358, - /* 400 */ 1342, 1382, 1361, 1346, 1386, 1365, 1388, 1368, 1371, 1394, - /* 410 */ 1295, 1357, 1396, 1256, 1375, 1300, 1301, 1398, 1400, 1304, - /* 420 */ 1401, 1331, 1360, 1317, 1333, 1348, 1208, 1347, 1349, 1359, - /* 430 */ 1336, 1343, 1363, 1366, 1364, 1362, 1369, 1374, 1220, 1372, - /* 440 */ 1376, 1370, 1305, 1381, 1373, 1379, 1384, 1306, 1453, 1420, - /* 450 */ 1422, 1430, 1441, 1442, 1443, 1480, 1316, 1413, 1411, 1414, - /* 460 */ 1415, 1419, 1421, 1403, 1423, 1424, 1408, 1473, 1332, 1425, - /* 470 */ 1428, 1429, 1434, 1444, 1383, 1446, 1505, 1472, 1389, 1447, - /* 480 */ 1433, 1437, 1448, 1510, 1459, 1450, 1461, 1499, 1501, 1466, - /* 490 */ 1467, 1503, 1469, 1470, 1506, 1474, 1471, 1509, 1475, 1476, - /* 500 */ 1512, 1479, 1456, 1457, 1458, 1460, 1530, 1477, 1484, 1522, - /* 510 */ 1454, 1491, 1492, 1529, 1263, 1303, 1548, 1521, 1523, 1517, - /* 520 */ 1504, 1507, 1539, 1542, 1543, 1544, 1546, 1547, 1549, 1564, - /* 530 */ 1532, 1263, 1550, 1303, 1552, 1553, 1572, 1573, 1574, 1576, - /* 540 */ 1577, 1607, 1579, 1582, 1578, 1619, 1584, 1587, 1581, 1620, - /* 550 */ 1588, 1589, 1585, 1627, 1591, 1594, 1596, 1637, 1603, 1605, - /* 560 */ 1643, 1644, 1623, 1625, 1626, 1628, 1630, 1629, + /* 0 */ 831, 0, 39, 214, 214, 214, 214, 253, 214, 214, + /* 10 */ 428, 467, 642, 467, 467, 467, 467, 467, 467, 467, + /* 20 */ 467, 467, 467, 467, 467, 467, 467, 467, 467, 467, + /* 30 */ 467, 467, 467, 467, 467, 467, 71, 383, 383, 383, + /* 40 */ 139, 1330, 1330, 110, 42, 42, 29, 1330, 348, 42, + /* 50 */ 42, 42, 42, 42, 42, 52, 42, 98, 140, 29, + /* 60 */ 142, 98, 42, 42, 98, 42, 98, 142, 98, 98, + /* 70 */ 42, 193, 217, 179, 393, 393, 389, 1327, 421, 187, + /* 80 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, + /* 90 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 665, + /* 100 */ 327, 109, 109, 360, 360, 360, 667, 109, 109, 312, + /* 110 */ 142, 98, 142, 98, 422, 345, 741, 741, 741, 741, + /* 120 */ 741, 741, 741, 1000, 444, 564, 480, 111, 396, 120, + /* 130 */ 27, 681, 546, 597, 342, 597, 687, 478, 264, 777, + /* 140 */ 937, 913, 931, 832, 937, 937, 859, 848, 848, 937, + /* 150 */ 967, 52, 142, 980, 52, 312, 984, 52, 52, 937, + /* 160 */ 52, 998, 98, 98, 98, 98, 98, 98, 98, 98, + /* 170 */ 98, 98, 98, 937, 998, 975, 967, 193, 898, 142, + /* 180 */ 193, 980, 193, 312, 984, 193, 1058, 872, 878, 975, + /* 190 */ 872, 878, 975, 975, 874, 877, 892, 894, 899, 312, + /* 200 */ 1079, 981, 880, 886, 889, 1035, 98, 878, 975, 975, + /* 210 */ 878, 975, 989, 312, 984, 193, 422, 193, 312, 1067, + /* 220 */ 345, 937, 193, 998, 1960, 1960, 1960, 1960, 1960, 1960, + /* 230 */ 718, 917, 329, 1558, 47, 483, 503, 272, 280, 661, + /* 240 */ 734, 11, 11, 11, 11, 11, 11, 11, 11, 94, + /* 250 */ 505, 196, 409, 283, 283, 283, 283, 595, 571, 373, + /* 260 */ 547, 657, 774, 778, 855, 858, 861, 838, 645, 737, + /* 270 */ 792, 793, 869, 691, 332, 669, 806, 716, 814, 754, + /* 280 */ 817, 820, 825, 827, 830, 824, 835, 833, 847, 849, + /* 290 */ 866, 879, 882, 125, 909, 1161, 1162, 1105, 1171, 1134, + /* 300 */ 1009, 1137, 1139, 1141, 1015, 1181, 1146, 1148, 1022, 1188, + /* 310 */ 1151, 1190, 1154, 1191, 1119, 1043, 1045, 1088, 1051, 1201, + /* 320 */ 1202, 1150, 1061, 1207, 1208, 1122, 1210, 1211, 1212, 1224, + /* 330 */ 1225, 1226, 1228, 1229, 1230, 1231, 1232, 1234, 1235, 1239, + /* 340 */ 1241, 1242, 1244, 1093, 1251, 1253, 1254, 1255, 1258, 1259, + /* 350 */ 1240, 1261, 1265, 1266, 1267, 1268, 1269, 1270, 1271, 1272, + /* 360 */ 1233, 1273, 1237, 1277, 1279, 1245, 1246, 1243, 1280, 1247, + /* 370 */ 1248, 1249, 1289, 1257, 1274, 1250, 1302, 1278, 1281, 1262, + /* 380 */ 1306, 1307, 1309, 1311, 1236, 1193, 1276, 1252, 1288, 1290, + /* 390 */ 1313, 1283, 1285, 1287, 1296, 1299, 1252, 1288, 1300, 1303, + /* 400 */ 1318, 1297, 1344, 1324, 1308, 1345, 1326, 1312, 1351, 1331, + /* 410 */ 1362, 1341, 1346, 1367, 1256, 1332, 1369, 1219, 1352, 1260, + /* 420 */ 1238, 1373, 1376, 1282, 1377, 1304, 1337, 1263, 1310, 1314, + /* 430 */ 1180, 1315, 1320, 1317, 1319, 1322, 1339, 1325, 1340, 1328, + /* 440 */ 1342, 1349, 1187, 1335, 1348, 1329, 1198, 1354, 1360, 1353, + /* 450 */ 1355, 1196, 1380, 1393, 1394, 1395, 1397, 1398, 1436, 1291, + /* 460 */ 1378, 1375, 1374, 1382, 1386, 1388, 1379, 1387, 1390, 1381, + /* 470 */ 1445, 1298, 1396, 1400, 1401, 1399, 1404, 1334, 1406, 1473, + /* 480 */ 1439, 1350, 1408, 1402, 1403, 1407, 1466, 1411, 1409, 1415, + /* 490 */ 1454, 1455, 1423, 1425, 1470, 1435, 1438, 1474, 1440, 1441, + /* 500 */ 1478, 1443, 1446, 1481, 1448, 1421, 1426, 1427, 1428, 1489, + /* 510 */ 1437, 1457, 1491, 1429, 1458, 1460, 1498, 1252, 1288, 1515, + /* 520 */ 1487, 1490, 1504, 1486, 1471, 1475, 1508, 1511, 1512, 1513, + /* 530 */ 1520, 1522, 1531, 1549, 1516, 1252, 1536, 1288, 1537, 1538, + /* 540 */ 1540, 1542, 1543, 1544, 1545, 1584, 1548, 1551, 1547, 1588, + /* 550 */ 1557, 1560, 1554, 1598, 1562, 1565, 1561, 1603, 1569, 1572, + /* 560 */ 1566, 1610, 1579, 1582, 1620, 1621, 1600, 1602, 1611, 1612, + /* 570 */ 1615, 1617, }; -#define YY_REDUCE_COUNT (227) -#define YY_REDUCE_MIN (-311) -#define YY_REDUCE_MAX (1569) +#define YY_REDUCE_COUNT (229) +#define YY_REDUCE_MIN (-273) +#define YY_REDUCE_MAX (1664) static const short yy_reduce_ofst[] = { - /* 0 */ -211, -208, 400, 451, 532, 588, 618, 648, 703, 753, - /* 10 */ 801, 812, 883, 921, 913, -220, 968, 1018, 1056, 1067, - /* 20 */ 1125, 1138, 1176, 1185, 1223, 1238, 1307, 1315, 1345, 1393, - /* 30 */ 1431, 1440, 1478, 1493, 1531, 1569, 179, -182, -2, 174, - /* 40 */ -186, -247, -228, 58, -221, -139, 181, -257, -154, -5, - /* 50 */ -4, -3, -1, 224, 128, 307, -109, -102, -311, -101, - /* 60 */ 69, 353, 417, 214, 424, 216, 278, 346, 420, 431, - /* 70 */ 228, -222, -292, -292, -292, -117, -223, -95, -204, -174, - /* 80 */ 62, 271, 282, 286, 288, 326, 404, 440, 454, 457, - /* 90 */ 480, 481, 486, 489, 512, 546, 569, 139, -33, -189, - /* 100 */ 46, 173, 415, 462, 494, -130, 193, 432, 324, -254, - /* 110 */ 550, 460, 487, 352, 54, -246, 385, 413, 469, 545, - /* 120 */ 567, 576, 555, 615, 612, 534, 560, 610, 607, 579, - /* 130 */ 665, 630, 593, 593, 593, 675, 599, 619, 675, 724, - /* 140 */ 674, 739, 685, 740, 746, 718, 737, 741, 779, 728, - /* 150 */ 780, 745, 742, 781, 759, 751, 791, 794, 789, 795, - /* 160 */ 803, 783, 790, 800, 804, 805, 809, 810, 811, 815, - /* 170 */ 820, 821, 807, 813, 782, 786, 833, 796, 808, 799, - /* 180 */ 846, 824, 816, 849, 797, 757, 828, 818, 763, 835, - /* 190 */ 839, 843, 802, 827, 819, 825, 593, 859, 832, 829, - /* 200 */ 830, 831, 834, 840, 675, 860, 865, 870, 862, 881, - /* 210 */ 882, 896, 887, 926, 915, 929, 902, 930, 925, 935, - /* 220 */ 937, 941, 907, 899, 942, 945, 957, 967, + /* 0 */ -212, -210, -159, 368, 13, 265, 440, 625, 638, 719, + /* 10 */ 771, 826, 875, 927, -221, 957, 1008, 1038, 1066, 1115, + /* 20 */ 1153, 1164, 1214, 1264, 1275, 1323, 1336, 1405, 1414, 1464, + /* 30 */ 1514, 1525, 1573, 1586, 1655, 1664, 929, -183, -104, 453, + /* 40 */ -125, -253, -251, -211, -226, 266, 50, -241, -273, -118, + /* 50 */ -89, 222, 267, 294, 306, -93, 438, -59, -248, 51, + /* 60 */ -263, 206, 448, 471, -6, 481, 210, -171, 281, 271, + /* 70 */ 452, 16, -224, -200, -200, -200, -208, 154, 59, -198, + /* 80 */ 221, 263, 304, 398, 456, 488, 489, 495, 497, 498, + /* 90 */ 499, 526, 529, 530, 532, 538, 578, 581, 582, 183, + /* 100 */ -178, 275, 316, -10, 4, 289, 10, 397, 462, 276, + /* 110 */ 188, 433, 527, 310, -12, -153, -239, 323, 445, 475, + /* 120 */ 506, 525, 548, 589, 658, 660, 573, 585, 652, 630, + /* 130 */ 603, 696, 648, 620, 620, 620, 688, 618, 633, 688, + /* 140 */ 729, 682, 732, 694, 740, 746, 715, 721, 722, 755, + /* 150 */ 708, 757, 726, 727, 769, 747, 738, 773, 780, 784, + /* 160 */ 783, 796, 775, 776, 779, 782, 785, 786, 789, 790, + /* 170 */ 791, 794, 799, 795, 802, 787, 770, 819, 797, 803, + /* 180 */ 828, 781, 840, 815, 805, 850, 798, 759, 810, 816, + /* 190 */ 763, 818, 821, 823, 788, 800, 804, 808, 620, 842, + /* 200 */ 822, 801, 768, 807, 809, 811, 688, 852, 841, 843, + /* 210 */ 856, 860, 864, 873, 867, 903, 883, 905, 884, 891, + /* 220 */ 907, 916, 914, 921, 876, 868, 908, 912, 918, 934, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 10 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 20 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 30 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 40 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 50 */ 1276, 1276, 1276, 1276, 1335, 1276, 1276, 1276, 1276, 1276, - /* 60 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 70 */ 1333, 1475, 1276, 1631, 1276, 1276, 1276, 1276, 1276, 1276, - /* 80 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 90 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1335, - /* 100 */ 1276, 1276, 1642, 1642, 1642, 1333, 1276, 1276, 1276, 1276, - /* 110 */ 1276, 1276, 1276, 1428, 1276, 1276, 1276, 1276, 1276, 1276, - /* 120 */ 1276, 1276, 1509, 1276, 1276, 1706, 1276, 1381, 1515, 1666, - /* 130 */ 1276, 1658, 1634, 1648, 1635, 1276, 1691, 1651, 1276, 1276, - /* 140 */ 1276, 1276, 1501, 1276, 1276, 1480, 1477, 1477, 1276, 1276, - /* 150 */ 1335, 1276, 1276, 1335, 1276, 1276, 1335, 1335, 1276, 1335, - /* 160 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 170 */ 1276, 1276, 1276, 1276, 1276, 1276, 1333, 1511, 1276, 1276, - /* 180 */ 1333, 1276, 1276, 1333, 1276, 1673, 1671, 1276, 1673, 1671, - /* 190 */ 1276, 1276, 1685, 1681, 1664, 1662, 1648, 1276, 1276, 1276, - /* 200 */ 1709, 1697, 1693, 1276, 1276, 1671, 1276, 1276, 1671, 1276, - /* 210 */ 1488, 1276, 1276, 1333, 1276, 1333, 1276, 1397, 1276, 1276, - /* 220 */ 1333, 1276, 1503, 1517, 1431, 1431, 1336, 1281, 1276, 1276, - /* 230 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1578, - /* 240 */ 1684, 1683, 1607, 1606, 1605, 1603, 1577, 1276, 1276, 1276, - /* 250 */ 1276, 1571, 1572, 1570, 1569, 1276, 1276, 1276, 1276, 1276, - /* 260 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 270 */ 1632, 1276, 1694, 1698, 1276, 1276, 1276, 1555, 1276, 1276, - /* 280 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 290 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 300 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 310 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 320 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 330 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 340 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 350 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 360 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 370 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 380 */ 1276, 1276, 1276, 1276, 1444, 1443, 1276, 1276, 1276, 1276, - /* 390 */ 1276, 1276, 1362, 1361, 1276, 1276, 1276, 1276, 1276, 1276, - /* 400 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 410 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 420 */ 1276, 1276, 1276, 1276, 1655, 1665, 1276, 1276, 1276, 1276, - /* 430 */ 1276, 1276, 1276, 1276, 1276, 1555, 1276, 1682, 1276, 1641, - /* 440 */ 1637, 1276, 1276, 1633, 1276, 1276, 1692, 1276, 1276, 1276, - /* 450 */ 1276, 1276, 1276, 1276, 1276, 1627, 1276, 1600, 1276, 1276, - /* 460 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1565, 1276, - /* 470 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 480 */ 1276, 1554, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1425, - /* 490 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 500 */ 1276, 1276, 1410, 1408, 1407, 1406, 1276, 1403, 1276, 1276, - /* 510 */ 1276, 1276, 1276, 1276, 1434, 1433, 1276, 1276, 1276, 1276, - /* 520 */ 1276, 1356, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 530 */ 1276, 1347, 1276, 1346, 1276, 1276, 1276, 1276, 1276, 1276, - /* 540 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 550 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, - /* 560 */ 1276, 1276, 1276, 1276, 1276, 1276, 1276, 1276, + /* 0 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 10 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 20 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 30 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 40 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 50 */ 1287, 1287, 1287, 1287, 1287, 1346, 1287, 1287, 1287, 1287, + /* 60 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 70 */ 1287, 1344, 1489, 1287, 1645, 1287, 1287, 1287, 1287, 1287, + /* 80 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 90 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 100 */ 1346, 1287, 1287, 1656, 1656, 1656, 1344, 1287, 1287, 1287, + /* 110 */ 1287, 1287, 1287, 1287, 1441, 1287, 1287, 1287, 1287, 1287, + /* 120 */ 1287, 1287, 1287, 1523, 1287, 1287, 1721, 1287, 1394, 1529, + /* 130 */ 1680, 1287, 1672, 1648, 1662, 1649, 1287, 1706, 1665, 1287, + /* 140 */ 1287, 1287, 1287, 1515, 1287, 1287, 1494, 1491, 1491, 1287, + /* 150 */ 1287, 1346, 1287, 1287, 1346, 1287, 1287, 1346, 1346, 1287, + /* 160 */ 1346, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 170 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1344, 1525, 1287, + /* 180 */ 1344, 1287, 1344, 1287, 1287, 1344, 1287, 1687, 1685, 1287, + /* 190 */ 1687, 1685, 1287, 1287, 1699, 1695, 1678, 1676, 1662, 1287, + /* 200 */ 1287, 1287, 1724, 1712, 1708, 1287, 1287, 1685, 1287, 1287, + /* 210 */ 1685, 1287, 1502, 1287, 1287, 1344, 1287, 1344, 1287, 1410, + /* 220 */ 1287, 1287, 1344, 1287, 1517, 1531, 1444, 1444, 1347, 1292, + /* 230 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 240 */ 1287, 1592, 1698, 1697, 1621, 1620, 1619, 1617, 1591, 1287, + /* 250 */ 1287, 1287, 1287, 1585, 1586, 1584, 1583, 1287, 1287, 1287, + /* 260 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 270 */ 1287, 1287, 1646, 1287, 1709, 1713, 1287, 1287, 1287, 1569, + /* 280 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 290 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 300 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 310 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 320 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 330 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 340 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 350 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 360 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 370 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 380 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1457, 1456, 1287, + /* 390 */ 1287, 1287, 1287, 1287, 1287, 1287, 1374, 1373, 1287, 1287, + /* 400 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 410 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 420 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1669, 1679, + /* 430 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1569, + /* 440 */ 1287, 1696, 1287, 1655, 1651, 1287, 1287, 1647, 1287, 1287, + /* 450 */ 1707, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1641, 1287, + /* 460 */ 1614, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 470 */ 1287, 1579, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 480 */ 1287, 1287, 1287, 1287, 1568, 1287, 1287, 1287, 1287, 1287, + /* 490 */ 1287, 1287, 1438, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 500 */ 1287, 1287, 1287, 1287, 1287, 1423, 1421, 1420, 1419, 1287, + /* 510 */ 1416, 1287, 1287, 1287, 1287, 1287, 1287, 1447, 1446, 1287, + /* 520 */ 1287, 1287, 1287, 1287, 1287, 1367, 1287, 1287, 1287, 1287, + /* 530 */ 1287, 1287, 1287, 1287, 1287, 1358, 1287, 1357, 1287, 1287, + /* 540 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 550 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 560 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 570 */ 1287, 1287, }; /********** End of lemon-generated parsing tables *****************************/ @@ -914,269 +934,271 @@ static const char *const yyTokenName[] = { /* 68 */ "SINGLE_STABLE", /* 69 */ "STREAM_MODE", /* 70 */ "RETENTIONS", - /* 71 */ "NK_COMMA", - /* 72 */ "NK_COLON", - /* 73 */ "TABLE", - /* 74 */ "NK_LP", - /* 75 */ "NK_RP", - /* 76 */ "STABLE", - /* 77 */ "ADD", - /* 78 */ "COLUMN", - /* 79 */ "MODIFY", - /* 80 */ "RENAME", - /* 81 */ "TAG", - /* 82 */ "SET", - /* 83 */ "NK_EQ", - /* 84 */ "USING", - /* 85 */ "TAGS", - /* 86 */ "NK_DOT", - /* 87 */ "COMMENT", - /* 88 */ "BOOL", - /* 89 */ "TINYINT", - /* 90 */ "SMALLINT", - /* 91 */ "INT", - /* 92 */ "INTEGER", - /* 93 */ "BIGINT", - /* 94 */ "FLOAT", - /* 95 */ "DOUBLE", - /* 96 */ "BINARY", - /* 97 */ "TIMESTAMP", - /* 98 */ "NCHAR", - /* 99 */ "UNSIGNED", - /* 100 */ "JSON", - /* 101 */ "VARCHAR", - /* 102 */ "MEDIUMBLOB", - /* 103 */ "BLOB", - /* 104 */ "VARBINARY", - /* 105 */ "DECIMAL", - /* 106 */ "SMA", - /* 107 */ "ROLLUP", - /* 108 */ "FILE_FACTOR", - /* 109 */ "NK_FLOAT", - /* 110 */ "DELAY", - /* 111 */ "SHOW", - /* 112 */ "DATABASES", - /* 113 */ "TABLES", - /* 114 */ "STABLES", - /* 115 */ "MNODES", - /* 116 */ "MODULES", - /* 117 */ "QNODES", - /* 118 */ "FUNCTIONS", - /* 119 */ "INDEXES", - /* 120 */ "FROM", - /* 121 */ "ACCOUNTS", - /* 122 */ "APPS", - /* 123 */ "CONNECTIONS", - /* 124 */ "LICENCE", - /* 125 */ "GRANTS", - /* 126 */ "QUERIES", - /* 127 */ "SCORES", - /* 128 */ "TOPICS", - /* 129 */ "VARIABLES", - /* 130 */ "BNODES", - /* 131 */ "SNODES", - /* 132 */ "LIKE", - /* 133 */ "INDEX", - /* 134 */ "FULLTEXT", - /* 135 */ "FUNCTION", - /* 136 */ "INTERVAL", - /* 137 */ "TOPIC", - /* 138 */ "AS", - /* 139 */ "DESC", - /* 140 */ "DESCRIBE", - /* 141 */ "RESET", - /* 142 */ "QUERY", - /* 143 */ "EXPLAIN", - /* 144 */ "ANALYZE", - /* 145 */ "VERBOSE", - /* 146 */ "NK_BOOL", - /* 147 */ "RATIO", - /* 148 */ "COMPACT", - /* 149 */ "VNODES", - /* 150 */ "IN", - /* 151 */ "OUTPUTTYPE", - /* 152 */ "AGGREGATE", - /* 153 */ "BUFSIZE", - /* 154 */ "STREAM", - /* 155 */ "INTO", - /* 156 */ "TRIGGER", - /* 157 */ "AT_ONCE", - /* 158 */ "WINDOW_CLOSE", - /* 159 */ "WATERMARK", - /* 160 */ "KILL", - /* 161 */ "CONNECTION", - /* 162 */ "MERGE", - /* 163 */ "VGROUP", - /* 164 */ "REDISTRIBUTE", - /* 165 */ "SPLIT", - /* 166 */ "SYNCDB", - /* 167 */ "NULL", - /* 168 */ "NK_QUESTION", - /* 169 */ "NK_ARROW", - /* 170 */ "ROWTS", - /* 171 */ "TBNAME", - /* 172 */ "QSTARTTS", - /* 173 */ "QENDTS", - /* 174 */ "WSTARTTS", - /* 175 */ "WENDTS", - /* 176 */ "WDURATION", - /* 177 */ "CAST", - /* 178 */ "NOW", - /* 179 */ "TODAY", - /* 180 */ "TIMEZONE", - /* 181 */ "COUNT", - /* 182 */ "FIRST", - /* 183 */ "LAST", - /* 184 */ "LAST_ROW", - /* 185 */ "BETWEEN", - /* 186 */ "IS", - /* 187 */ "NK_LT", - /* 188 */ "NK_GT", - /* 189 */ "NK_LE", - /* 190 */ "NK_GE", - /* 191 */ "NK_NE", - /* 192 */ "MATCH", - /* 193 */ "NMATCH", - /* 194 */ "CONTAINS", - /* 195 */ "JOIN", - /* 196 */ "INNER", - /* 197 */ "SELECT", - /* 198 */ "DISTINCT", - /* 199 */ "WHERE", - /* 200 */ "PARTITION", - /* 201 */ "BY", - /* 202 */ "SESSION", - /* 203 */ "STATE_WINDOW", - /* 204 */ "SLIDING", - /* 205 */ "FILL", - /* 206 */ "VALUE", - /* 207 */ "NONE", - /* 208 */ "PREV", - /* 209 */ "LINEAR", - /* 210 */ "NEXT", - /* 211 */ "GROUP", - /* 212 */ "HAVING", - /* 213 */ "ORDER", - /* 214 */ "SLIMIT", - /* 215 */ "SOFFSET", - /* 216 */ "LIMIT", - /* 217 */ "OFFSET", - /* 218 */ "ASC", - /* 219 */ "NULLS", - /* 220 */ "cmd", - /* 221 */ "account_options", - /* 222 */ "alter_account_options", - /* 223 */ "literal", - /* 224 */ "alter_account_option", - /* 225 */ "user_name", - /* 226 */ "dnode_endpoint", - /* 227 */ "dnode_host_name", - /* 228 */ "not_exists_opt", - /* 229 */ "db_name", - /* 230 */ "db_options", - /* 231 */ "exists_opt", - /* 232 */ "alter_db_options", - /* 233 */ "integer_list", - /* 234 */ "variable_list", - /* 235 */ "retention_list", - /* 236 */ "alter_db_option", - /* 237 */ "retention", - /* 238 */ "full_table_name", - /* 239 */ "column_def_list", - /* 240 */ "tags_def_opt", - /* 241 */ "table_options", - /* 242 */ "multi_create_clause", - /* 243 */ "tags_def", - /* 244 */ "multi_drop_clause", - /* 245 */ "alter_table_clause", - /* 246 */ "alter_table_options", - /* 247 */ "column_name", - /* 248 */ "type_name", - /* 249 */ "create_subtable_clause", - /* 250 */ "specific_tags_opt", - /* 251 */ "literal_list", - /* 252 */ "drop_table_clause", - /* 253 */ "col_name_list", - /* 254 */ "table_name", - /* 255 */ "column_def", - /* 256 */ "func_name_list", - /* 257 */ "alter_table_option", - /* 258 */ "col_name", - /* 259 */ "db_name_cond_opt", - /* 260 */ "like_pattern_opt", - /* 261 */ "table_name_cond", - /* 262 */ "from_db_opt", - /* 263 */ "func_name", - /* 264 */ "function_name", - /* 265 */ "index_name", - /* 266 */ "index_options", - /* 267 */ "func_list", - /* 268 */ "duration_literal", - /* 269 */ "sliding_opt", - /* 270 */ "func", - /* 271 */ "expression_list", - /* 272 */ "topic_name", - /* 273 */ "query_expression", - /* 274 */ "analyze_opt", - /* 275 */ "explain_options", - /* 276 */ "agg_func_opt", - /* 277 */ "bufsize_opt", - /* 278 */ "stream_name", - /* 279 */ "stream_options", - /* 280 */ "into_opt", - /* 281 */ "dnode_list", - /* 282 */ "signed", - /* 283 */ "signed_literal", - /* 284 */ "table_alias", - /* 285 */ "column_alias", - /* 286 */ "expression", - /* 287 */ "pseudo_column", - /* 288 */ "column_reference", - /* 289 */ "function_expression", - /* 290 */ "subquery", - /* 291 */ "star_func", - /* 292 */ "star_func_para_list", - /* 293 */ "noarg_func", - /* 294 */ "other_para_list", - /* 295 */ "star_func_para", - /* 296 */ "predicate", - /* 297 */ "compare_op", - /* 298 */ "in_op", - /* 299 */ "in_predicate_value", - /* 300 */ "boolean_value_expression", - /* 301 */ "boolean_primary", - /* 302 */ "common_expression", - /* 303 */ "from_clause", - /* 304 */ "table_reference_list", - /* 305 */ "table_reference", - /* 306 */ "table_primary", - /* 307 */ "joined_table", - /* 308 */ "alias_opt", - /* 309 */ "parenthesized_joined_table", - /* 310 */ "join_type", - /* 311 */ "search_condition", - /* 312 */ "query_specification", - /* 313 */ "set_quantifier_opt", - /* 314 */ "select_list", - /* 315 */ "where_clause_opt", - /* 316 */ "partition_by_clause_opt", - /* 317 */ "twindow_clause_opt", - /* 318 */ "group_by_clause_opt", - /* 319 */ "having_clause_opt", - /* 320 */ "select_sublist", - /* 321 */ "select_item", - /* 322 */ "fill_opt", - /* 323 */ "fill_mode", - /* 324 */ "group_by_list", - /* 325 */ "query_expression_body", - /* 326 */ "order_by_clause_opt", - /* 327 */ "slimit_clause_opt", - /* 328 */ "limit_clause_opt", - /* 329 */ "query_primary", - /* 330 */ "sort_specification_list", - /* 331 */ "sort_specification", - /* 332 */ "ordering_specification_opt", - /* 333 */ "null_ordering_opt", + /* 71 */ "STRICT", + /* 72 */ "NK_COMMA", + /* 73 */ "NK_COLON", + /* 74 */ "TABLE", + /* 75 */ "NK_LP", + /* 76 */ "NK_RP", + /* 77 */ "STABLE", + /* 78 */ "ADD", + /* 79 */ "COLUMN", + /* 80 */ "MODIFY", + /* 81 */ "RENAME", + /* 82 */ "TAG", + /* 83 */ "SET", + /* 84 */ "NK_EQ", + /* 85 */ "USING", + /* 86 */ "TAGS", + /* 87 */ "NK_DOT", + /* 88 */ "COMMENT", + /* 89 */ "BOOL", + /* 90 */ "TINYINT", + /* 91 */ "SMALLINT", + /* 92 */ "INT", + /* 93 */ "INTEGER", + /* 94 */ "BIGINT", + /* 95 */ "FLOAT", + /* 96 */ "DOUBLE", + /* 97 */ "BINARY", + /* 98 */ "TIMESTAMP", + /* 99 */ "NCHAR", + /* 100 */ "UNSIGNED", + /* 101 */ "JSON", + /* 102 */ "VARCHAR", + /* 103 */ "MEDIUMBLOB", + /* 104 */ "BLOB", + /* 105 */ "VARBINARY", + /* 106 */ "DECIMAL", + /* 107 */ "SMA", + /* 108 */ "ROLLUP", + /* 109 */ "FILE_FACTOR", + /* 110 */ "NK_FLOAT", + /* 111 */ "DELAY", + /* 112 */ "SHOW", + /* 113 */ "DATABASES", + /* 114 */ "TABLES", + /* 115 */ "STABLES", + /* 116 */ "MNODES", + /* 117 */ "MODULES", + /* 118 */ "QNODES", + /* 119 */ "FUNCTIONS", + /* 120 */ "INDEXES", + /* 121 */ "FROM", + /* 122 */ "ACCOUNTS", + /* 123 */ "APPS", + /* 124 */ "CONNECTIONS", + /* 125 */ "LICENCE", + /* 126 */ "GRANTS", + /* 127 */ "QUERIES", + /* 128 */ "SCORES", + /* 129 */ "TOPICS", + /* 130 */ "VARIABLES", + /* 131 */ "BNODES", + /* 132 */ "SNODES", + /* 133 */ "CLUSTER", + /* 134 */ "LIKE", + /* 135 */ "INDEX", + /* 136 */ "FULLTEXT", + /* 137 */ "FUNCTION", + /* 138 */ "INTERVAL", + /* 139 */ "TOPIC", + /* 140 */ "AS", + /* 141 */ "DESC", + /* 142 */ "DESCRIBE", + /* 143 */ "RESET", + /* 144 */ "QUERY", + /* 145 */ "EXPLAIN", + /* 146 */ "ANALYZE", + /* 147 */ "VERBOSE", + /* 148 */ "NK_BOOL", + /* 149 */ "RATIO", + /* 150 */ "COMPACT", + /* 151 */ "VNODES", + /* 152 */ "IN", + /* 153 */ "OUTPUTTYPE", + /* 154 */ "AGGREGATE", + /* 155 */ "BUFSIZE", + /* 156 */ "STREAM", + /* 157 */ "INTO", + /* 158 */ "TRIGGER", + /* 159 */ "AT_ONCE", + /* 160 */ "WINDOW_CLOSE", + /* 161 */ "WATERMARK", + /* 162 */ "KILL", + /* 163 */ "CONNECTION", + /* 164 */ "MERGE", + /* 165 */ "VGROUP", + /* 166 */ "REDISTRIBUTE", + /* 167 */ "SPLIT", + /* 168 */ "SYNCDB", + /* 169 */ "NULL", + /* 170 */ "NK_QUESTION", + /* 171 */ "NK_ARROW", + /* 172 */ "ROWTS", + /* 173 */ "TBNAME", + /* 174 */ "QSTARTTS", + /* 175 */ "QENDTS", + /* 176 */ "WSTARTTS", + /* 177 */ "WENDTS", + /* 178 */ "WDURATION", + /* 179 */ "CAST", + /* 180 */ "NOW", + /* 181 */ "TODAY", + /* 182 */ "TIMEZONE", + /* 183 */ "COUNT", + /* 184 */ "FIRST", + /* 185 */ "LAST", + /* 186 */ "LAST_ROW", + /* 187 */ "BETWEEN", + /* 188 */ "IS", + /* 189 */ "NK_LT", + /* 190 */ "NK_GT", + /* 191 */ "NK_LE", + /* 192 */ "NK_GE", + /* 193 */ "NK_NE", + /* 194 */ "MATCH", + /* 195 */ "NMATCH", + /* 196 */ "CONTAINS", + /* 197 */ "JOIN", + /* 198 */ "INNER", + /* 199 */ "SELECT", + /* 200 */ "DISTINCT", + /* 201 */ "WHERE", + /* 202 */ "PARTITION", + /* 203 */ "BY", + /* 204 */ "SESSION", + /* 205 */ "STATE_WINDOW", + /* 206 */ "SLIDING", + /* 207 */ "FILL", + /* 208 */ "VALUE", + /* 209 */ "NONE", + /* 210 */ "PREV", + /* 211 */ "LINEAR", + /* 212 */ "NEXT", + /* 213 */ "GROUP", + /* 214 */ "HAVING", + /* 215 */ "ORDER", + /* 216 */ "SLIMIT", + /* 217 */ "SOFFSET", + /* 218 */ "LIMIT", + /* 219 */ "OFFSET", + /* 220 */ "ASC", + /* 221 */ "NULLS", + /* 222 */ "cmd", + /* 223 */ "account_options", + /* 224 */ "alter_account_options", + /* 225 */ "literal", + /* 226 */ "alter_account_option", + /* 227 */ "user_name", + /* 228 */ "dnode_endpoint", + /* 229 */ "dnode_host_name", + /* 230 */ "not_exists_opt", + /* 231 */ "db_name", + /* 232 */ "db_options", + /* 233 */ "exists_opt", + /* 234 */ "alter_db_options", + /* 235 */ "integer_list", + /* 236 */ "variable_list", + /* 237 */ "retention_list", + /* 238 */ "alter_db_option", + /* 239 */ "retention", + /* 240 */ "full_table_name", + /* 241 */ "column_def_list", + /* 242 */ "tags_def_opt", + /* 243 */ "table_options", + /* 244 */ "multi_create_clause", + /* 245 */ "tags_def", + /* 246 */ "multi_drop_clause", + /* 247 */ "alter_table_clause", + /* 248 */ "alter_table_options", + /* 249 */ "column_name", + /* 250 */ "type_name", + /* 251 */ "create_subtable_clause", + /* 252 */ "specific_tags_opt", + /* 253 */ "literal_list", + /* 254 */ "drop_table_clause", + /* 255 */ "col_name_list", + /* 256 */ "table_name", + /* 257 */ "column_def", + /* 258 */ "func_name_list", + /* 259 */ "alter_table_option", + /* 260 */ "col_name", + /* 261 */ "db_name_cond_opt", + /* 262 */ "like_pattern_opt", + /* 263 */ "table_name_cond", + /* 264 */ "from_db_opt", + /* 265 */ "func_name", + /* 266 */ "function_name", + /* 267 */ "index_name", + /* 268 */ "index_options", + /* 269 */ "func_list", + /* 270 */ "duration_literal", + /* 271 */ "sliding_opt", + /* 272 */ "func", + /* 273 */ "expression_list", + /* 274 */ "topic_name", + /* 275 */ "query_expression", + /* 276 */ "analyze_opt", + /* 277 */ "explain_options", + /* 278 */ "agg_func_opt", + /* 279 */ "bufsize_opt", + /* 280 */ "stream_name", + /* 281 */ "stream_options", + /* 282 */ "into_opt", + /* 283 */ "dnode_list", + /* 284 */ "signed", + /* 285 */ "signed_literal", + /* 286 */ "table_alias", + /* 287 */ "column_alias", + /* 288 */ "expression", + /* 289 */ "pseudo_column", + /* 290 */ "column_reference", + /* 291 */ "function_expression", + /* 292 */ "subquery", + /* 293 */ "star_func", + /* 294 */ "star_func_para_list", + /* 295 */ "noarg_func", + /* 296 */ "other_para_list", + /* 297 */ "star_func_para", + /* 298 */ "predicate", + /* 299 */ "compare_op", + /* 300 */ "in_op", + /* 301 */ "in_predicate_value", + /* 302 */ "boolean_value_expression", + /* 303 */ "boolean_primary", + /* 304 */ "common_expression", + /* 305 */ "from_clause", + /* 306 */ "table_reference_list", + /* 307 */ "table_reference", + /* 308 */ "table_primary", + /* 309 */ "joined_table", + /* 310 */ "alias_opt", + /* 311 */ "parenthesized_joined_table", + /* 312 */ "join_type", + /* 313 */ "search_condition", + /* 314 */ "query_specification", + /* 315 */ "set_quantifier_opt", + /* 316 */ "select_list", + /* 317 */ "where_clause_opt", + /* 318 */ "partition_by_clause_opt", + /* 319 */ "twindow_clause_opt", + /* 320 */ "group_by_clause_opt", + /* 321 */ "having_clause_opt", + /* 322 */ "select_sublist", + /* 323 */ "select_item", + /* 324 */ "fill_opt", + /* 325 */ "fill_mode", + /* 326 */ "group_by_list", + /* 327 */ "query_expression_body", + /* 328 */ "order_by_clause_opt", + /* 329 */ "slimit_clause_opt", + /* 330 */ "limit_clause_opt", + /* 331 */ "query_primary", + /* 332 */ "sort_specification_list", + /* 333 */ "sort_specification", + /* 334 */ "ordering_specification_opt", + /* 335 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1262,361 +1284,365 @@ static const char *const yyRuleName[] = { /* 75 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", /* 76 */ "db_options ::= db_options STREAM_MODE NK_INTEGER", /* 77 */ "db_options ::= db_options RETENTIONS retention_list", - /* 78 */ "alter_db_options ::= alter_db_option", - /* 79 */ "alter_db_options ::= alter_db_options alter_db_option", - /* 80 */ "alter_db_option ::= BLOCKS NK_INTEGER", - /* 81 */ "alter_db_option ::= FSYNC NK_INTEGER", - /* 82 */ "alter_db_option ::= KEEP integer_list", - /* 83 */ "alter_db_option ::= KEEP variable_list", - /* 84 */ "alter_db_option ::= WAL NK_INTEGER", - /* 85 */ "alter_db_option ::= QUORUM NK_INTEGER", - /* 86 */ "alter_db_option ::= CACHELAST NK_INTEGER", - /* 87 */ "alter_db_option ::= REPLICA NK_INTEGER", - /* 88 */ "integer_list ::= NK_INTEGER", - /* 89 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", - /* 90 */ "variable_list ::= NK_VARIABLE", - /* 91 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", - /* 92 */ "retention_list ::= retention", - /* 93 */ "retention_list ::= retention_list NK_COMMA retention", - /* 94 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", - /* 95 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", - /* 96 */ "cmd ::= CREATE TABLE multi_create_clause", - /* 97 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", - /* 98 */ "cmd ::= DROP TABLE multi_drop_clause", - /* 99 */ "cmd ::= DROP STABLE exists_opt full_table_name", - /* 100 */ "cmd ::= ALTER TABLE alter_table_clause", - /* 101 */ "cmd ::= ALTER STABLE alter_table_clause", - /* 102 */ "alter_table_clause ::= full_table_name alter_table_options", - /* 103 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", - /* 104 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", - /* 105 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", - /* 106 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", - /* 107 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", - /* 108 */ "alter_table_clause ::= full_table_name DROP TAG column_name", - /* 109 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", - /* 110 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", - /* 111 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal", - /* 112 */ "multi_create_clause ::= create_subtable_clause", - /* 113 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", - /* 114 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP", - /* 115 */ "multi_drop_clause ::= drop_table_clause", - /* 116 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", - /* 117 */ "drop_table_clause ::= exists_opt full_table_name", - /* 118 */ "specific_tags_opt ::=", - /* 119 */ "specific_tags_opt ::= NK_LP col_name_list NK_RP", - /* 120 */ "full_table_name ::= table_name", - /* 121 */ "full_table_name ::= db_name NK_DOT table_name", - /* 122 */ "column_def_list ::= column_def", - /* 123 */ "column_def_list ::= column_def_list NK_COMMA column_def", - /* 124 */ "column_def ::= column_name type_name", - /* 125 */ "column_def ::= column_name type_name COMMENT NK_STRING", - /* 126 */ "type_name ::= BOOL", - /* 127 */ "type_name ::= TINYINT", - /* 128 */ "type_name ::= SMALLINT", - /* 129 */ "type_name ::= INT", - /* 130 */ "type_name ::= INTEGER", - /* 131 */ "type_name ::= BIGINT", - /* 132 */ "type_name ::= FLOAT", - /* 133 */ "type_name ::= DOUBLE", - /* 134 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", - /* 135 */ "type_name ::= TIMESTAMP", - /* 136 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", - /* 137 */ "type_name ::= TINYINT UNSIGNED", - /* 138 */ "type_name ::= SMALLINT UNSIGNED", - /* 139 */ "type_name ::= INT UNSIGNED", - /* 140 */ "type_name ::= BIGINT UNSIGNED", - /* 141 */ "type_name ::= JSON", - /* 142 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", - /* 143 */ "type_name ::= MEDIUMBLOB", - /* 144 */ "type_name ::= BLOB", - /* 145 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", - /* 146 */ "type_name ::= DECIMAL", - /* 147 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", - /* 148 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", - /* 149 */ "tags_def_opt ::=", - /* 150 */ "tags_def_opt ::= tags_def", - /* 151 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", - /* 152 */ "table_options ::=", - /* 153 */ "table_options ::= table_options COMMENT NK_STRING", - /* 154 */ "table_options ::= table_options KEEP integer_list", - /* 155 */ "table_options ::= table_options KEEP variable_list", - /* 156 */ "table_options ::= table_options TTL NK_INTEGER", - /* 157 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", - /* 158 */ "table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP", - /* 159 */ "table_options ::= table_options FILE_FACTOR NK_FLOAT", - /* 160 */ "table_options ::= table_options DELAY NK_INTEGER", - /* 161 */ "alter_table_options ::= alter_table_option", - /* 162 */ "alter_table_options ::= alter_table_options alter_table_option", - /* 163 */ "alter_table_option ::= COMMENT NK_STRING", - /* 164 */ "alter_table_option ::= KEEP integer_list", - /* 165 */ "alter_table_option ::= KEEP variable_list", - /* 166 */ "alter_table_option ::= TTL NK_INTEGER", - /* 167 */ "col_name_list ::= col_name", - /* 168 */ "col_name_list ::= col_name_list NK_COMMA col_name", - /* 169 */ "col_name ::= column_name", - /* 170 */ "cmd ::= SHOW DNODES", - /* 171 */ "cmd ::= SHOW USERS", - /* 172 */ "cmd ::= SHOW DATABASES", - /* 173 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", - /* 174 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", - /* 175 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", - /* 176 */ "cmd ::= SHOW MNODES", - /* 177 */ "cmd ::= SHOW MODULES", - /* 178 */ "cmd ::= SHOW QNODES", - /* 179 */ "cmd ::= SHOW FUNCTIONS", - /* 180 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", - /* 181 */ "cmd ::= SHOW STREAMS", - /* 182 */ "cmd ::= SHOW ACCOUNTS", - /* 183 */ "cmd ::= SHOW APPS", - /* 184 */ "cmd ::= SHOW CONNECTIONS", - /* 185 */ "cmd ::= SHOW LICENCE", - /* 186 */ "cmd ::= SHOW GRANTS", - /* 187 */ "cmd ::= SHOW CREATE DATABASE db_name", - /* 188 */ "cmd ::= SHOW CREATE TABLE full_table_name", - /* 189 */ "cmd ::= SHOW CREATE STABLE full_table_name", - /* 190 */ "cmd ::= SHOW QUERIES", - /* 191 */ "cmd ::= SHOW SCORES", - /* 192 */ "cmd ::= SHOW TOPICS", - /* 193 */ "cmd ::= SHOW VARIABLES", - /* 194 */ "cmd ::= SHOW BNODES", - /* 195 */ "cmd ::= SHOW SNODES", - /* 196 */ "db_name_cond_opt ::=", - /* 197 */ "db_name_cond_opt ::= db_name NK_DOT", - /* 198 */ "like_pattern_opt ::=", - /* 199 */ "like_pattern_opt ::= LIKE NK_STRING", - /* 200 */ "table_name_cond ::= table_name", - /* 201 */ "from_db_opt ::=", - /* 202 */ "from_db_opt ::= FROM db_name", - /* 203 */ "func_name_list ::= func_name", - /* 204 */ "func_name_list ::= func_name_list NK_COMMA col_name", - /* 205 */ "func_name ::= function_name", - /* 206 */ "cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options", - /* 207 */ "cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP", - /* 208 */ "cmd ::= DROP INDEX exists_opt index_name ON table_name", - /* 209 */ "index_options ::=", - /* 210 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt", - /* 211 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt", - /* 212 */ "func_list ::= func", - /* 213 */ "func_list ::= func_list NK_COMMA func", - /* 214 */ "func ::= function_name NK_LP expression_list NK_RP", - /* 215 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression", - /* 216 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name", - /* 217 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 218 */ "cmd ::= DESC full_table_name", - /* 219 */ "cmd ::= DESCRIBE full_table_name", - /* 220 */ "cmd ::= RESET QUERY CACHE", - /* 221 */ "cmd ::= EXPLAIN analyze_opt explain_options query_expression", - /* 222 */ "analyze_opt ::=", - /* 223 */ "analyze_opt ::= ANALYZE", - /* 224 */ "explain_options ::=", - /* 225 */ "explain_options ::= explain_options VERBOSE NK_BOOL", - /* 226 */ "explain_options ::= explain_options RATIO NK_FLOAT", - /* 227 */ "cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP", - /* 228 */ "cmd ::= CREATE agg_func_opt FUNCTION function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", - /* 229 */ "cmd ::= DROP FUNCTION function_name", - /* 230 */ "agg_func_opt ::=", - /* 231 */ "agg_func_opt ::= AGGREGATE", - /* 232 */ "bufsize_opt ::=", - /* 233 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", - /* 234 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression", - /* 235 */ "cmd ::= DROP STREAM exists_opt stream_name", - /* 236 */ "into_opt ::=", - /* 237 */ "into_opt ::= INTO full_table_name", - /* 238 */ "stream_options ::=", - /* 239 */ "stream_options ::= stream_options TRIGGER AT_ONCE", - /* 240 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", - /* 241 */ "stream_options ::= stream_options WATERMARK duration_literal", - /* 242 */ "cmd ::= KILL CONNECTION NK_INTEGER", - /* 243 */ "cmd ::= KILL QUERY NK_INTEGER", - /* 244 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", - /* 245 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", - /* 246 */ "cmd ::= SPLIT VGROUP NK_INTEGER", - /* 247 */ "dnode_list ::= DNODE NK_INTEGER", - /* 248 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", - /* 249 */ "cmd ::= SYNCDB db_name REPLICA", - /* 250 */ "cmd ::= query_expression", - /* 251 */ "literal ::= NK_INTEGER", - /* 252 */ "literal ::= NK_FLOAT", - /* 253 */ "literal ::= NK_STRING", - /* 254 */ "literal ::= NK_BOOL", - /* 255 */ "literal ::= TIMESTAMP NK_STRING", - /* 256 */ "literal ::= duration_literal", - /* 257 */ "literal ::= NULL", - /* 258 */ "literal ::= NK_QUESTION", - /* 259 */ "duration_literal ::= NK_VARIABLE", - /* 260 */ "signed ::= NK_INTEGER", - /* 261 */ "signed ::= NK_PLUS NK_INTEGER", - /* 262 */ "signed ::= NK_MINUS NK_INTEGER", - /* 263 */ "signed ::= NK_FLOAT", - /* 264 */ "signed ::= NK_PLUS NK_FLOAT", - /* 265 */ "signed ::= NK_MINUS NK_FLOAT", - /* 266 */ "signed_literal ::= signed", - /* 267 */ "signed_literal ::= NK_STRING", - /* 268 */ "signed_literal ::= NK_BOOL", - /* 269 */ "signed_literal ::= TIMESTAMP NK_STRING", - /* 270 */ "signed_literal ::= duration_literal", - /* 271 */ "signed_literal ::= NULL", - /* 272 */ "literal_list ::= signed_literal", - /* 273 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 274 */ "db_name ::= NK_ID", - /* 275 */ "table_name ::= NK_ID", - /* 276 */ "column_name ::= NK_ID", - /* 277 */ "function_name ::= NK_ID", - /* 278 */ "table_alias ::= NK_ID", - /* 279 */ "column_alias ::= NK_ID", - /* 280 */ "user_name ::= NK_ID", - /* 281 */ "index_name ::= NK_ID", - /* 282 */ "topic_name ::= NK_ID", - /* 283 */ "stream_name ::= NK_ID", - /* 284 */ "expression ::= literal", - /* 285 */ "expression ::= pseudo_column", - /* 286 */ "expression ::= column_reference", - /* 287 */ "expression ::= function_expression", - /* 288 */ "expression ::= subquery", - /* 289 */ "expression ::= NK_LP expression NK_RP", - /* 290 */ "expression ::= NK_PLUS expression", - /* 291 */ "expression ::= NK_MINUS expression", - /* 292 */ "expression ::= expression NK_PLUS expression", - /* 293 */ "expression ::= expression NK_MINUS expression", - /* 294 */ "expression ::= expression NK_STAR expression", - /* 295 */ "expression ::= expression NK_SLASH expression", - /* 296 */ "expression ::= expression NK_REM expression", - /* 297 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 298 */ "expression_list ::= expression", - /* 299 */ "expression_list ::= expression_list NK_COMMA expression", - /* 300 */ "column_reference ::= column_name", - /* 301 */ "column_reference ::= table_name NK_DOT column_name", - /* 302 */ "pseudo_column ::= ROWTS", - /* 303 */ "pseudo_column ::= TBNAME", - /* 304 */ "pseudo_column ::= QSTARTTS", - /* 305 */ "pseudo_column ::= QENDTS", - /* 306 */ "pseudo_column ::= WSTARTTS", - /* 307 */ "pseudo_column ::= WENDTS", - /* 308 */ "pseudo_column ::= WDURATION", - /* 309 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 310 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 311 */ "function_expression ::= CAST NK_LP expression AS type_name NK_RP", - /* 312 */ "function_expression ::= noarg_func NK_LP NK_RP", - /* 313 */ "noarg_func ::= NOW", - /* 314 */ "noarg_func ::= TODAY", - /* 315 */ "noarg_func ::= TIMEZONE", - /* 316 */ "star_func ::= COUNT", - /* 317 */ "star_func ::= FIRST", - /* 318 */ "star_func ::= LAST", - /* 319 */ "star_func ::= LAST_ROW", - /* 320 */ "star_func_para_list ::= NK_STAR", - /* 321 */ "star_func_para_list ::= other_para_list", - /* 322 */ "other_para_list ::= star_func_para", - /* 323 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 324 */ "star_func_para ::= expression", - /* 325 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 326 */ "predicate ::= expression compare_op expression", - /* 327 */ "predicate ::= expression BETWEEN expression AND expression", - /* 328 */ "predicate ::= expression NOT BETWEEN expression AND expression", - /* 329 */ "predicate ::= expression IS NULL", - /* 330 */ "predicate ::= expression IS NOT NULL", - /* 331 */ "predicate ::= expression in_op in_predicate_value", - /* 332 */ "compare_op ::= NK_LT", - /* 333 */ "compare_op ::= NK_GT", - /* 334 */ "compare_op ::= NK_LE", - /* 335 */ "compare_op ::= NK_GE", - /* 336 */ "compare_op ::= NK_NE", - /* 337 */ "compare_op ::= NK_EQ", - /* 338 */ "compare_op ::= LIKE", - /* 339 */ "compare_op ::= NOT LIKE", - /* 340 */ "compare_op ::= MATCH", - /* 341 */ "compare_op ::= NMATCH", - /* 342 */ "compare_op ::= CONTAINS", - /* 343 */ "in_op ::= IN", - /* 344 */ "in_op ::= NOT IN", - /* 345 */ "in_predicate_value ::= NK_LP expression_list NK_RP", - /* 346 */ "boolean_value_expression ::= boolean_primary", - /* 347 */ "boolean_value_expression ::= NOT boolean_primary", - /* 348 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 349 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 350 */ "boolean_primary ::= predicate", - /* 351 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 352 */ "common_expression ::= expression", - /* 353 */ "common_expression ::= boolean_value_expression", - /* 354 */ "from_clause ::= FROM table_reference_list", - /* 355 */ "table_reference_list ::= table_reference", - /* 356 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 357 */ "table_reference ::= table_primary", - /* 358 */ "table_reference ::= joined_table", - /* 359 */ "table_primary ::= table_name alias_opt", - /* 360 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 361 */ "table_primary ::= subquery alias_opt", - /* 362 */ "table_primary ::= parenthesized_joined_table", - /* 363 */ "alias_opt ::=", - /* 364 */ "alias_opt ::= table_alias", - /* 365 */ "alias_opt ::= AS table_alias", - /* 366 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 367 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 368 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 369 */ "join_type ::=", - /* 370 */ "join_type ::= INNER", - /* 371 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 372 */ "set_quantifier_opt ::=", - /* 373 */ "set_quantifier_opt ::= DISTINCT", - /* 374 */ "set_quantifier_opt ::= ALL", - /* 375 */ "select_list ::= NK_STAR", - /* 376 */ "select_list ::= select_sublist", - /* 377 */ "select_sublist ::= select_item", - /* 378 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 379 */ "select_item ::= common_expression", - /* 380 */ "select_item ::= common_expression column_alias", - /* 381 */ "select_item ::= common_expression AS column_alias", - /* 382 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 383 */ "where_clause_opt ::=", - /* 384 */ "where_clause_opt ::= WHERE search_condition", - /* 385 */ "partition_by_clause_opt ::=", - /* 386 */ "partition_by_clause_opt ::= PARTITION BY expression_list", - /* 387 */ "twindow_clause_opt ::=", - /* 388 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 389 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", - /* 390 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 391 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 392 */ "sliding_opt ::=", - /* 393 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 394 */ "fill_opt ::=", - /* 395 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 396 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 397 */ "fill_mode ::= NONE", - /* 398 */ "fill_mode ::= PREV", - /* 399 */ "fill_mode ::= NULL", - /* 400 */ "fill_mode ::= LINEAR", - /* 401 */ "fill_mode ::= NEXT", - /* 402 */ "group_by_clause_opt ::=", - /* 403 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 404 */ "group_by_list ::= expression", - /* 405 */ "group_by_list ::= group_by_list NK_COMMA expression", - /* 406 */ "having_clause_opt ::=", - /* 407 */ "having_clause_opt ::= HAVING search_condition", - /* 408 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 409 */ "query_expression_body ::= query_primary", - /* 410 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 411 */ "query_primary ::= query_specification", - /* 412 */ "order_by_clause_opt ::=", - /* 413 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 414 */ "slimit_clause_opt ::=", - /* 415 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 416 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 417 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 418 */ "limit_clause_opt ::=", - /* 419 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 420 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 421 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 422 */ "subquery ::= NK_LP query_expression NK_RP", - /* 423 */ "search_condition ::= common_expression", - /* 424 */ "sort_specification_list ::= sort_specification", - /* 425 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 426 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 427 */ "ordering_specification_opt ::=", - /* 428 */ "ordering_specification_opt ::= ASC", - /* 429 */ "ordering_specification_opt ::= DESC", - /* 430 */ "null_ordering_opt ::=", - /* 431 */ "null_ordering_opt ::= NULLS FIRST", - /* 432 */ "null_ordering_opt ::= NULLS LAST", + /* 78 */ "db_options ::= db_options STRICT NK_INTEGER", + /* 79 */ "alter_db_options ::= alter_db_option", + /* 80 */ "alter_db_options ::= alter_db_options alter_db_option", + /* 81 */ "alter_db_option ::= BLOCKS NK_INTEGER", + /* 82 */ "alter_db_option ::= FSYNC NK_INTEGER", + /* 83 */ "alter_db_option ::= KEEP integer_list", + /* 84 */ "alter_db_option ::= KEEP variable_list", + /* 85 */ "alter_db_option ::= WAL NK_INTEGER", + /* 86 */ "alter_db_option ::= QUORUM NK_INTEGER", + /* 87 */ "alter_db_option ::= CACHELAST NK_INTEGER", + /* 88 */ "alter_db_option ::= REPLICA NK_INTEGER", + /* 89 */ "alter_db_option ::= STRICT NK_INTEGER", + /* 90 */ "integer_list ::= NK_INTEGER", + /* 91 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", + /* 92 */ "variable_list ::= NK_VARIABLE", + /* 93 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", + /* 94 */ "retention_list ::= retention", + /* 95 */ "retention_list ::= retention_list NK_COMMA retention", + /* 96 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", + /* 97 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", + /* 98 */ "cmd ::= CREATE TABLE multi_create_clause", + /* 99 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", + /* 100 */ "cmd ::= DROP TABLE multi_drop_clause", + /* 101 */ "cmd ::= DROP STABLE exists_opt full_table_name", + /* 102 */ "cmd ::= ALTER TABLE alter_table_clause", + /* 103 */ "cmd ::= ALTER STABLE alter_table_clause", + /* 104 */ "alter_table_clause ::= full_table_name alter_table_options", + /* 105 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", + /* 106 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", + /* 107 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", + /* 108 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", + /* 109 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", + /* 110 */ "alter_table_clause ::= full_table_name DROP TAG column_name", + /* 111 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", + /* 112 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", + /* 113 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal", + /* 114 */ "multi_create_clause ::= create_subtable_clause", + /* 115 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", + /* 116 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP", + /* 117 */ "multi_drop_clause ::= drop_table_clause", + /* 118 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", + /* 119 */ "drop_table_clause ::= exists_opt full_table_name", + /* 120 */ "specific_tags_opt ::=", + /* 121 */ "specific_tags_opt ::= NK_LP col_name_list NK_RP", + /* 122 */ "full_table_name ::= table_name", + /* 123 */ "full_table_name ::= db_name NK_DOT table_name", + /* 124 */ "column_def_list ::= column_def", + /* 125 */ "column_def_list ::= column_def_list NK_COMMA column_def", + /* 126 */ "column_def ::= column_name type_name", + /* 127 */ "column_def ::= column_name type_name COMMENT NK_STRING", + /* 128 */ "type_name ::= BOOL", + /* 129 */ "type_name ::= TINYINT", + /* 130 */ "type_name ::= SMALLINT", + /* 131 */ "type_name ::= INT", + /* 132 */ "type_name ::= INTEGER", + /* 133 */ "type_name ::= BIGINT", + /* 134 */ "type_name ::= FLOAT", + /* 135 */ "type_name ::= DOUBLE", + /* 136 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", + /* 137 */ "type_name ::= TIMESTAMP", + /* 138 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", + /* 139 */ "type_name ::= TINYINT UNSIGNED", + /* 140 */ "type_name ::= SMALLINT UNSIGNED", + /* 141 */ "type_name ::= INT UNSIGNED", + /* 142 */ "type_name ::= BIGINT UNSIGNED", + /* 143 */ "type_name ::= JSON", + /* 144 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", + /* 145 */ "type_name ::= MEDIUMBLOB", + /* 146 */ "type_name ::= BLOB", + /* 147 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", + /* 148 */ "type_name ::= DECIMAL", + /* 149 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", + /* 150 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", + /* 151 */ "tags_def_opt ::=", + /* 152 */ "tags_def_opt ::= tags_def", + /* 153 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", + /* 154 */ "table_options ::=", + /* 155 */ "table_options ::= table_options COMMENT NK_STRING", + /* 156 */ "table_options ::= table_options KEEP integer_list", + /* 157 */ "table_options ::= table_options KEEP variable_list", + /* 158 */ "table_options ::= table_options TTL NK_INTEGER", + /* 159 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", + /* 160 */ "table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP", + /* 161 */ "table_options ::= table_options FILE_FACTOR NK_FLOAT", + /* 162 */ "table_options ::= table_options DELAY NK_INTEGER", + /* 163 */ "alter_table_options ::= alter_table_option", + /* 164 */ "alter_table_options ::= alter_table_options alter_table_option", + /* 165 */ "alter_table_option ::= COMMENT NK_STRING", + /* 166 */ "alter_table_option ::= KEEP integer_list", + /* 167 */ "alter_table_option ::= KEEP variable_list", + /* 168 */ "alter_table_option ::= TTL NK_INTEGER", + /* 169 */ "col_name_list ::= col_name", + /* 170 */ "col_name_list ::= col_name_list NK_COMMA col_name", + /* 171 */ "col_name ::= column_name", + /* 172 */ "cmd ::= SHOW DNODES", + /* 173 */ "cmd ::= SHOW USERS", + /* 174 */ "cmd ::= SHOW DATABASES", + /* 175 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", + /* 176 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", + /* 177 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", + /* 178 */ "cmd ::= SHOW MNODES", + /* 179 */ "cmd ::= SHOW MODULES", + /* 180 */ "cmd ::= SHOW QNODES", + /* 181 */ "cmd ::= SHOW FUNCTIONS", + /* 182 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", + /* 183 */ "cmd ::= SHOW STREAMS", + /* 184 */ "cmd ::= SHOW ACCOUNTS", + /* 185 */ "cmd ::= SHOW APPS", + /* 186 */ "cmd ::= SHOW CONNECTIONS", + /* 187 */ "cmd ::= SHOW LICENCE", + /* 188 */ "cmd ::= SHOW GRANTS", + /* 189 */ "cmd ::= SHOW CREATE DATABASE db_name", + /* 190 */ "cmd ::= SHOW CREATE TABLE full_table_name", + /* 191 */ "cmd ::= SHOW CREATE STABLE full_table_name", + /* 192 */ "cmd ::= SHOW QUERIES", + /* 193 */ "cmd ::= SHOW SCORES", + /* 194 */ "cmd ::= SHOW TOPICS", + /* 195 */ "cmd ::= SHOW VARIABLES", + /* 196 */ "cmd ::= SHOW BNODES", + /* 197 */ "cmd ::= SHOW SNODES", + /* 198 */ "cmd ::= SHOW CLUSTER", + /* 199 */ "db_name_cond_opt ::=", + /* 200 */ "db_name_cond_opt ::= db_name NK_DOT", + /* 201 */ "like_pattern_opt ::=", + /* 202 */ "like_pattern_opt ::= LIKE NK_STRING", + /* 203 */ "table_name_cond ::= table_name", + /* 204 */ "from_db_opt ::=", + /* 205 */ "from_db_opt ::= FROM db_name", + /* 206 */ "func_name_list ::= func_name", + /* 207 */ "func_name_list ::= func_name_list NK_COMMA func_name", + /* 208 */ "func_name ::= function_name", + /* 209 */ "cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options", + /* 210 */ "cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP", + /* 211 */ "cmd ::= DROP INDEX exists_opt index_name ON table_name", + /* 212 */ "index_options ::=", + /* 213 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt", + /* 214 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt", + /* 215 */ "func_list ::= func", + /* 216 */ "func_list ::= func_list NK_COMMA func", + /* 217 */ "func ::= function_name NK_LP expression_list NK_RP", + /* 218 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression", + /* 219 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name", + /* 220 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 221 */ "cmd ::= DESC full_table_name", + /* 222 */ "cmd ::= DESCRIBE full_table_name", + /* 223 */ "cmd ::= RESET QUERY CACHE", + /* 224 */ "cmd ::= EXPLAIN analyze_opt explain_options query_expression", + /* 225 */ "analyze_opt ::=", + /* 226 */ "analyze_opt ::= ANALYZE", + /* 227 */ "explain_options ::=", + /* 228 */ "explain_options ::= explain_options VERBOSE NK_BOOL", + /* 229 */ "explain_options ::= explain_options RATIO NK_FLOAT", + /* 230 */ "cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP", + /* 231 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", + /* 232 */ "cmd ::= DROP FUNCTION function_name", + /* 233 */ "agg_func_opt ::=", + /* 234 */ "agg_func_opt ::= AGGREGATE", + /* 235 */ "bufsize_opt ::=", + /* 236 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", + /* 237 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression", + /* 238 */ "cmd ::= DROP STREAM exists_opt stream_name", + /* 239 */ "into_opt ::=", + /* 240 */ "into_opt ::= INTO full_table_name", + /* 241 */ "stream_options ::=", + /* 242 */ "stream_options ::= stream_options TRIGGER AT_ONCE", + /* 243 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", + /* 244 */ "stream_options ::= stream_options WATERMARK duration_literal", + /* 245 */ "cmd ::= KILL CONNECTION NK_INTEGER", + /* 246 */ "cmd ::= KILL QUERY NK_INTEGER", + /* 247 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", + /* 248 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", + /* 249 */ "cmd ::= SPLIT VGROUP NK_INTEGER", + /* 250 */ "dnode_list ::= DNODE NK_INTEGER", + /* 251 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", + /* 252 */ "cmd ::= SYNCDB db_name REPLICA", + /* 253 */ "cmd ::= query_expression", + /* 254 */ "literal ::= NK_INTEGER", + /* 255 */ "literal ::= NK_FLOAT", + /* 256 */ "literal ::= NK_STRING", + /* 257 */ "literal ::= NK_BOOL", + /* 258 */ "literal ::= TIMESTAMP NK_STRING", + /* 259 */ "literal ::= duration_literal", + /* 260 */ "literal ::= NULL", + /* 261 */ "literal ::= NK_QUESTION", + /* 262 */ "duration_literal ::= NK_VARIABLE", + /* 263 */ "signed ::= NK_INTEGER", + /* 264 */ "signed ::= NK_PLUS NK_INTEGER", + /* 265 */ "signed ::= NK_MINUS NK_INTEGER", + /* 266 */ "signed ::= NK_FLOAT", + /* 267 */ "signed ::= NK_PLUS NK_FLOAT", + /* 268 */ "signed ::= NK_MINUS NK_FLOAT", + /* 269 */ "signed_literal ::= signed", + /* 270 */ "signed_literal ::= NK_STRING", + /* 271 */ "signed_literal ::= NK_BOOL", + /* 272 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 273 */ "signed_literal ::= duration_literal", + /* 274 */ "signed_literal ::= NULL", + /* 275 */ "literal_list ::= signed_literal", + /* 276 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 277 */ "db_name ::= NK_ID", + /* 278 */ "table_name ::= NK_ID", + /* 279 */ "column_name ::= NK_ID", + /* 280 */ "function_name ::= NK_ID", + /* 281 */ "table_alias ::= NK_ID", + /* 282 */ "column_alias ::= NK_ID", + /* 283 */ "user_name ::= NK_ID", + /* 284 */ "index_name ::= NK_ID", + /* 285 */ "topic_name ::= NK_ID", + /* 286 */ "stream_name ::= NK_ID", + /* 287 */ "expression ::= literal", + /* 288 */ "expression ::= pseudo_column", + /* 289 */ "expression ::= column_reference", + /* 290 */ "expression ::= function_expression", + /* 291 */ "expression ::= subquery", + /* 292 */ "expression ::= NK_LP expression NK_RP", + /* 293 */ "expression ::= NK_PLUS expression", + /* 294 */ "expression ::= NK_MINUS expression", + /* 295 */ "expression ::= expression NK_PLUS expression", + /* 296 */ "expression ::= expression NK_MINUS expression", + /* 297 */ "expression ::= expression NK_STAR expression", + /* 298 */ "expression ::= expression NK_SLASH expression", + /* 299 */ "expression ::= expression NK_REM expression", + /* 300 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 301 */ "expression_list ::= expression", + /* 302 */ "expression_list ::= expression_list NK_COMMA expression", + /* 303 */ "column_reference ::= column_name", + /* 304 */ "column_reference ::= table_name NK_DOT column_name", + /* 305 */ "pseudo_column ::= ROWTS", + /* 306 */ "pseudo_column ::= TBNAME", + /* 307 */ "pseudo_column ::= QSTARTTS", + /* 308 */ "pseudo_column ::= QENDTS", + /* 309 */ "pseudo_column ::= WSTARTTS", + /* 310 */ "pseudo_column ::= WENDTS", + /* 311 */ "pseudo_column ::= WDURATION", + /* 312 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 313 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 314 */ "function_expression ::= CAST NK_LP expression AS type_name NK_RP", + /* 315 */ "function_expression ::= noarg_func NK_LP NK_RP", + /* 316 */ "noarg_func ::= NOW", + /* 317 */ "noarg_func ::= TODAY", + /* 318 */ "noarg_func ::= TIMEZONE", + /* 319 */ "star_func ::= COUNT", + /* 320 */ "star_func ::= FIRST", + /* 321 */ "star_func ::= LAST", + /* 322 */ "star_func ::= LAST_ROW", + /* 323 */ "star_func_para_list ::= NK_STAR", + /* 324 */ "star_func_para_list ::= other_para_list", + /* 325 */ "other_para_list ::= star_func_para", + /* 326 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 327 */ "star_func_para ::= expression", + /* 328 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 329 */ "predicate ::= expression compare_op expression", + /* 330 */ "predicate ::= expression BETWEEN expression AND expression", + /* 331 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 332 */ "predicate ::= expression IS NULL", + /* 333 */ "predicate ::= expression IS NOT NULL", + /* 334 */ "predicate ::= expression in_op in_predicate_value", + /* 335 */ "compare_op ::= NK_LT", + /* 336 */ "compare_op ::= NK_GT", + /* 337 */ "compare_op ::= NK_LE", + /* 338 */ "compare_op ::= NK_GE", + /* 339 */ "compare_op ::= NK_NE", + /* 340 */ "compare_op ::= NK_EQ", + /* 341 */ "compare_op ::= LIKE", + /* 342 */ "compare_op ::= NOT LIKE", + /* 343 */ "compare_op ::= MATCH", + /* 344 */ "compare_op ::= NMATCH", + /* 345 */ "compare_op ::= CONTAINS", + /* 346 */ "in_op ::= IN", + /* 347 */ "in_op ::= NOT IN", + /* 348 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 349 */ "boolean_value_expression ::= boolean_primary", + /* 350 */ "boolean_value_expression ::= NOT boolean_primary", + /* 351 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 352 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 353 */ "boolean_primary ::= predicate", + /* 354 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 355 */ "common_expression ::= expression", + /* 356 */ "common_expression ::= boolean_value_expression", + /* 357 */ "from_clause ::= FROM table_reference_list", + /* 358 */ "table_reference_list ::= table_reference", + /* 359 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 360 */ "table_reference ::= table_primary", + /* 361 */ "table_reference ::= joined_table", + /* 362 */ "table_primary ::= table_name alias_opt", + /* 363 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 364 */ "table_primary ::= subquery alias_opt", + /* 365 */ "table_primary ::= parenthesized_joined_table", + /* 366 */ "alias_opt ::=", + /* 367 */ "alias_opt ::= table_alias", + /* 368 */ "alias_opt ::= AS table_alias", + /* 369 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 370 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 371 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 372 */ "join_type ::=", + /* 373 */ "join_type ::= INNER", + /* 374 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 375 */ "set_quantifier_opt ::=", + /* 376 */ "set_quantifier_opt ::= DISTINCT", + /* 377 */ "set_quantifier_opt ::= ALL", + /* 378 */ "select_list ::= NK_STAR", + /* 379 */ "select_list ::= select_sublist", + /* 380 */ "select_sublist ::= select_item", + /* 381 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 382 */ "select_item ::= common_expression", + /* 383 */ "select_item ::= common_expression column_alias", + /* 384 */ "select_item ::= common_expression AS column_alias", + /* 385 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 386 */ "where_clause_opt ::=", + /* 387 */ "where_clause_opt ::= WHERE search_condition", + /* 388 */ "partition_by_clause_opt ::=", + /* 389 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 390 */ "twindow_clause_opt ::=", + /* 391 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 392 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", + /* 393 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 394 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 395 */ "sliding_opt ::=", + /* 396 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 397 */ "fill_opt ::=", + /* 398 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 399 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 400 */ "fill_mode ::= NONE", + /* 401 */ "fill_mode ::= PREV", + /* 402 */ "fill_mode ::= NULL", + /* 403 */ "fill_mode ::= LINEAR", + /* 404 */ "fill_mode ::= NEXT", + /* 405 */ "group_by_clause_opt ::=", + /* 406 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 407 */ "group_by_list ::= expression", + /* 408 */ "group_by_list ::= group_by_list NK_COMMA expression", + /* 409 */ "having_clause_opt ::=", + /* 410 */ "having_clause_opt ::= HAVING search_condition", + /* 411 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 412 */ "query_expression_body ::= query_primary", + /* 413 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 414 */ "query_expression_body ::= query_expression_body UNION query_expression_body", + /* 415 */ "query_primary ::= query_specification", + /* 416 */ "order_by_clause_opt ::=", + /* 417 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 418 */ "slimit_clause_opt ::=", + /* 419 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 420 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 421 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 422 */ "limit_clause_opt ::=", + /* 423 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 424 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 425 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 426 */ "subquery ::= NK_LP query_expression NK_RP", + /* 427 */ "search_condition ::= common_expression", + /* 428 */ "sort_specification_list ::= sort_specification", + /* 429 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 430 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 431 */ "ordering_specification_opt ::=", + /* 432 */ "ordering_specification_opt ::= ASC", + /* 433 */ "ordering_specification_opt ::= DESC", + /* 434 */ "null_ordering_opt ::=", + /* 435 */ "null_ordering_opt ::= NULLS FIRST", + /* 436 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -1743,164 +1769,164 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 220: /* cmd */ - case 223: /* literal */ - case 230: /* db_options */ - case 232: /* alter_db_options */ - case 237: /* retention */ - case 238: /* full_table_name */ - case 241: /* table_options */ - case 245: /* alter_table_clause */ - case 246: /* alter_table_options */ - case 249: /* create_subtable_clause */ - case 252: /* drop_table_clause */ - case 255: /* column_def */ - case 258: /* col_name */ - case 259: /* db_name_cond_opt */ - case 260: /* like_pattern_opt */ - case 261: /* table_name_cond */ - case 262: /* from_db_opt */ - case 263: /* func_name */ - case 266: /* index_options */ - case 268: /* duration_literal */ - case 269: /* sliding_opt */ - case 270: /* func */ - case 273: /* query_expression */ - case 275: /* explain_options */ - case 279: /* stream_options */ - case 280: /* into_opt */ - case 282: /* signed */ - case 283: /* signed_literal */ - case 286: /* expression */ - case 287: /* pseudo_column */ - case 288: /* column_reference */ - case 289: /* function_expression */ - case 290: /* subquery */ - case 295: /* star_func_para */ - case 296: /* predicate */ - case 299: /* in_predicate_value */ - case 300: /* boolean_value_expression */ - case 301: /* boolean_primary */ - case 302: /* common_expression */ - case 303: /* from_clause */ - case 304: /* table_reference_list */ - case 305: /* table_reference */ - case 306: /* table_primary */ - case 307: /* joined_table */ - case 309: /* parenthesized_joined_table */ - case 311: /* search_condition */ - case 312: /* query_specification */ - case 315: /* where_clause_opt */ - case 317: /* twindow_clause_opt */ - case 319: /* having_clause_opt */ - case 321: /* select_item */ - case 322: /* fill_opt */ - case 325: /* query_expression_body */ - case 327: /* slimit_clause_opt */ - case 328: /* limit_clause_opt */ - case 329: /* query_primary */ - case 331: /* sort_specification */ + case 222: /* cmd */ + case 225: /* literal */ + case 232: /* db_options */ + case 234: /* alter_db_options */ + case 239: /* retention */ + case 240: /* full_table_name */ + case 243: /* table_options */ + case 247: /* alter_table_clause */ + case 248: /* alter_table_options */ + case 251: /* create_subtable_clause */ + case 254: /* drop_table_clause */ + case 257: /* column_def */ + case 260: /* col_name */ + case 261: /* db_name_cond_opt */ + case 262: /* like_pattern_opt */ + case 263: /* table_name_cond */ + case 264: /* from_db_opt */ + case 265: /* func_name */ + case 268: /* index_options */ + case 270: /* duration_literal */ + case 271: /* sliding_opt */ + case 272: /* func */ + case 275: /* query_expression */ + case 277: /* explain_options */ + case 281: /* stream_options */ + case 282: /* into_opt */ + case 284: /* signed */ + case 285: /* signed_literal */ + case 288: /* expression */ + case 289: /* pseudo_column */ + case 290: /* column_reference */ + case 291: /* function_expression */ + case 292: /* subquery */ + case 297: /* star_func_para */ + case 298: /* predicate */ + case 301: /* in_predicate_value */ + case 302: /* boolean_value_expression */ + case 303: /* boolean_primary */ + case 304: /* common_expression */ + case 305: /* from_clause */ + case 306: /* table_reference_list */ + case 307: /* table_reference */ + case 308: /* table_primary */ + case 309: /* joined_table */ + case 311: /* parenthesized_joined_table */ + case 313: /* search_condition */ + case 314: /* query_specification */ + case 317: /* where_clause_opt */ + case 319: /* twindow_clause_opt */ + case 321: /* having_clause_opt */ + case 323: /* select_item */ + case 324: /* fill_opt */ + case 327: /* query_expression_body */ + case 329: /* slimit_clause_opt */ + case 330: /* limit_clause_opt */ + case 331: /* query_primary */ + case 333: /* sort_specification */ { - nodesDestroyNode((yypminor->yy456)); + nodesDestroyNode((yypminor->yy392)); } break; - case 221: /* account_options */ - case 222: /* alter_account_options */ - case 224: /* alter_account_option */ - case 277: /* bufsize_opt */ + case 223: /* account_options */ + case 224: /* alter_account_options */ + case 226: /* alter_account_option */ + case 279: /* bufsize_opt */ { } break; - case 225: /* user_name */ - case 226: /* dnode_endpoint */ - case 227: /* dnode_host_name */ - case 229: /* db_name */ - case 247: /* column_name */ - case 254: /* table_name */ - case 264: /* function_name */ - case 265: /* index_name */ - case 272: /* topic_name */ - case 278: /* stream_name */ - case 284: /* table_alias */ - case 285: /* column_alias */ - case 291: /* star_func */ - case 293: /* noarg_func */ - case 308: /* alias_opt */ + case 227: /* user_name */ + case 228: /* dnode_endpoint */ + case 229: /* dnode_host_name */ + case 231: /* db_name */ + case 249: /* column_name */ + case 256: /* table_name */ + case 266: /* function_name */ + case 267: /* index_name */ + case 274: /* topic_name */ + case 280: /* stream_name */ + case 286: /* table_alias */ + case 287: /* column_alias */ + case 293: /* star_func */ + case 295: /* noarg_func */ + case 310: /* alias_opt */ { } break; - case 228: /* not_exists_opt */ - case 231: /* exists_opt */ - case 274: /* analyze_opt */ - case 276: /* agg_func_opt */ - case 313: /* set_quantifier_opt */ + case 230: /* not_exists_opt */ + case 233: /* exists_opt */ + case 276: /* analyze_opt */ + case 278: /* agg_func_opt */ + case 315: /* set_quantifier_opt */ { } break; - case 233: /* integer_list */ - case 234: /* variable_list */ - case 235: /* retention_list */ - case 239: /* column_def_list */ - case 240: /* tags_def_opt */ - case 242: /* multi_create_clause */ - case 243: /* tags_def */ - case 244: /* multi_drop_clause */ - case 250: /* specific_tags_opt */ - case 251: /* literal_list */ - case 253: /* col_name_list */ - case 256: /* func_name_list */ - case 267: /* func_list */ - case 271: /* expression_list */ - case 281: /* dnode_list */ - case 292: /* star_func_para_list */ - case 294: /* other_para_list */ - case 314: /* select_list */ - case 316: /* partition_by_clause_opt */ - case 318: /* group_by_clause_opt */ - case 320: /* select_sublist */ - case 324: /* group_by_list */ - case 326: /* order_by_clause_opt */ - case 330: /* sort_specification_list */ + case 235: /* integer_list */ + case 236: /* variable_list */ + case 237: /* retention_list */ + case 241: /* column_def_list */ + case 242: /* tags_def_opt */ + case 244: /* multi_create_clause */ + case 245: /* tags_def */ + case 246: /* multi_drop_clause */ + case 252: /* specific_tags_opt */ + case 253: /* literal_list */ + case 255: /* col_name_list */ + case 258: /* func_name_list */ + case 269: /* func_list */ + case 273: /* expression_list */ + case 283: /* dnode_list */ + case 294: /* star_func_para_list */ + case 296: /* other_para_list */ + case 316: /* select_list */ + case 318: /* partition_by_clause_opt */ + case 320: /* group_by_clause_opt */ + case 322: /* select_sublist */ + case 326: /* group_by_list */ + case 328: /* order_by_clause_opt */ + case 332: /* sort_specification_list */ { - nodesDestroyList((yypminor->yy632)); + nodesDestroyList((yypminor->yy376)); } break; - case 236: /* alter_db_option */ - case 257: /* alter_table_option */ + case 238: /* alter_db_option */ + case 259: /* alter_table_option */ { } break; - case 248: /* type_name */ + case 250: /* type_name */ { } break; - case 297: /* compare_op */ - case 298: /* in_op */ + case 299: /* compare_op */ + case 300: /* in_op */ { } break; - case 310: /* join_type */ + case 312: /* join_type */ { } break; - case 323: /* fill_mode */ + case 325: /* fill_mode */ { } break; - case 332: /* ordering_specification_opt */ + case 334: /* ordering_specification_opt */ { } break; - case 333: /* null_ordering_opt */ + case 335: /* null_ordering_opt */ { } @@ -2199,439 +2225,443 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 220, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ - { 220, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ - { 221, 0 }, /* (2) account_options ::= */ - { 221, -3 }, /* (3) account_options ::= account_options PPS literal */ - { 221, -3 }, /* (4) account_options ::= account_options TSERIES literal */ - { 221, -3 }, /* (5) account_options ::= account_options STORAGE literal */ - { 221, -3 }, /* (6) account_options ::= account_options STREAMS literal */ - { 221, -3 }, /* (7) account_options ::= account_options QTIME literal */ - { 221, -3 }, /* (8) account_options ::= account_options DBS literal */ - { 221, -3 }, /* (9) account_options ::= account_options USERS literal */ - { 221, -3 }, /* (10) account_options ::= account_options CONNS literal */ - { 221, -3 }, /* (11) account_options ::= account_options STATE literal */ - { 222, -1 }, /* (12) alter_account_options ::= alter_account_option */ - { 222, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ - { 224, -2 }, /* (14) alter_account_option ::= PASS literal */ - { 224, -2 }, /* (15) alter_account_option ::= PPS literal */ - { 224, -2 }, /* (16) alter_account_option ::= TSERIES literal */ - { 224, -2 }, /* (17) alter_account_option ::= STORAGE literal */ - { 224, -2 }, /* (18) alter_account_option ::= STREAMS literal */ - { 224, -2 }, /* (19) alter_account_option ::= QTIME literal */ - { 224, -2 }, /* (20) alter_account_option ::= DBS literal */ - { 224, -2 }, /* (21) alter_account_option ::= USERS literal */ - { 224, -2 }, /* (22) alter_account_option ::= CONNS literal */ - { 224, -2 }, /* (23) alter_account_option ::= STATE literal */ - { 220, -5 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING */ - { 220, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ - { 220, -5 }, /* (26) cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ - { 220, -3 }, /* (27) cmd ::= DROP USER user_name */ - { 220, -3 }, /* (28) cmd ::= CREATE DNODE dnode_endpoint */ - { 220, -5 }, /* (29) cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ - { 220, -3 }, /* (30) cmd ::= DROP DNODE NK_INTEGER */ - { 220, -3 }, /* (31) cmd ::= DROP DNODE dnode_endpoint */ - { 220, -4 }, /* (32) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ - { 220, -5 }, /* (33) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ - { 220, -4 }, /* (34) cmd ::= ALTER ALL DNODES NK_STRING */ - { 220, -5 }, /* (35) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ - { 226, -1 }, /* (36) dnode_endpoint ::= NK_STRING */ - { 227, -1 }, /* (37) dnode_host_name ::= NK_ID */ - { 227, -1 }, /* (38) dnode_host_name ::= NK_IPTOKEN */ - { 220, -3 }, /* (39) cmd ::= ALTER LOCAL NK_STRING */ - { 220, -4 }, /* (40) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ - { 220, -5 }, /* (41) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ - { 220, -5 }, /* (42) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ - { 220, -5 }, /* (43) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ - { 220, -5 }, /* (44) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ - { 220, -5 }, /* (45) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ - { 220, -5 }, /* (46) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ - { 220, -5 }, /* (47) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ - { 220, -5 }, /* (48) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ - { 220, -5 }, /* (49) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - { 220, -4 }, /* (50) cmd ::= DROP DATABASE exists_opt db_name */ - { 220, -2 }, /* (51) cmd ::= USE db_name */ - { 220, -4 }, /* (52) cmd ::= ALTER DATABASE db_name alter_db_options */ - { 228, -3 }, /* (53) not_exists_opt ::= IF NOT EXISTS */ - { 228, 0 }, /* (54) not_exists_opt ::= */ - { 231, -2 }, /* (55) exists_opt ::= IF EXISTS */ - { 231, 0 }, /* (56) exists_opt ::= */ - { 230, 0 }, /* (57) db_options ::= */ - { 230, -3 }, /* (58) db_options ::= db_options BLOCKS NK_INTEGER */ - { 230, -3 }, /* (59) db_options ::= db_options CACHE NK_INTEGER */ - { 230, -3 }, /* (60) db_options ::= db_options CACHELAST NK_INTEGER */ - { 230, -3 }, /* (61) db_options ::= db_options COMP NK_INTEGER */ - { 230, -3 }, /* (62) db_options ::= db_options DAYS NK_INTEGER */ - { 230, -3 }, /* (63) db_options ::= db_options DAYS NK_VARIABLE */ - { 230, -3 }, /* (64) db_options ::= db_options FSYNC NK_INTEGER */ - { 230, -3 }, /* (65) db_options ::= db_options MAXROWS NK_INTEGER */ - { 230, -3 }, /* (66) db_options ::= db_options MINROWS NK_INTEGER */ - { 230, -3 }, /* (67) db_options ::= db_options KEEP integer_list */ - { 230, -3 }, /* (68) db_options ::= db_options KEEP variable_list */ - { 230, -3 }, /* (69) db_options ::= db_options PRECISION NK_STRING */ - { 230, -3 }, /* (70) db_options ::= db_options QUORUM NK_INTEGER */ - { 230, -3 }, /* (71) db_options ::= db_options REPLICA NK_INTEGER */ - { 230, -3 }, /* (72) db_options ::= db_options TTL NK_INTEGER */ - { 230, -3 }, /* (73) db_options ::= db_options WAL NK_INTEGER */ - { 230, -3 }, /* (74) db_options ::= db_options VGROUPS NK_INTEGER */ - { 230, -3 }, /* (75) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - { 230, -3 }, /* (76) db_options ::= db_options STREAM_MODE NK_INTEGER */ - { 230, -3 }, /* (77) db_options ::= db_options RETENTIONS retention_list */ - { 232, -1 }, /* (78) alter_db_options ::= alter_db_option */ - { 232, -2 }, /* (79) alter_db_options ::= alter_db_options alter_db_option */ - { 236, -2 }, /* (80) alter_db_option ::= BLOCKS NK_INTEGER */ - { 236, -2 }, /* (81) alter_db_option ::= FSYNC NK_INTEGER */ - { 236, -2 }, /* (82) alter_db_option ::= KEEP integer_list */ - { 236, -2 }, /* (83) alter_db_option ::= KEEP variable_list */ - { 236, -2 }, /* (84) alter_db_option ::= WAL NK_INTEGER */ - { 236, -2 }, /* (85) alter_db_option ::= QUORUM NK_INTEGER */ - { 236, -2 }, /* (86) alter_db_option ::= CACHELAST NK_INTEGER */ - { 236, -2 }, /* (87) alter_db_option ::= REPLICA NK_INTEGER */ - { 233, -1 }, /* (88) integer_list ::= NK_INTEGER */ - { 233, -3 }, /* (89) integer_list ::= integer_list NK_COMMA NK_INTEGER */ - { 234, -1 }, /* (90) variable_list ::= NK_VARIABLE */ - { 234, -3 }, /* (91) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ - { 235, -1 }, /* (92) retention_list ::= retention */ - { 235, -3 }, /* (93) retention_list ::= retention_list NK_COMMA retention */ - { 237, -3 }, /* (94) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ - { 220, -9 }, /* (95) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - { 220, -3 }, /* (96) cmd ::= CREATE TABLE multi_create_clause */ - { 220, -9 }, /* (97) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - { 220, -3 }, /* (98) cmd ::= DROP TABLE multi_drop_clause */ - { 220, -4 }, /* (99) cmd ::= DROP STABLE exists_opt full_table_name */ - { 220, -3 }, /* (100) cmd ::= ALTER TABLE alter_table_clause */ - { 220, -3 }, /* (101) cmd ::= ALTER STABLE alter_table_clause */ - { 245, -2 }, /* (102) alter_table_clause ::= full_table_name alter_table_options */ - { 245, -5 }, /* (103) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ - { 245, -4 }, /* (104) alter_table_clause ::= full_table_name DROP COLUMN column_name */ - { 245, -5 }, /* (105) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ - { 245, -5 }, /* (106) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ - { 245, -5 }, /* (107) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ - { 245, -4 }, /* (108) alter_table_clause ::= full_table_name DROP TAG column_name */ - { 245, -5 }, /* (109) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ - { 245, -5 }, /* (110) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ - { 245, -6 }, /* (111) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ - { 242, -1 }, /* (112) multi_create_clause ::= create_subtable_clause */ - { 242, -2 }, /* (113) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 249, -9 }, /* (114) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ - { 244, -1 }, /* (115) multi_drop_clause ::= drop_table_clause */ - { 244, -2 }, /* (116) multi_drop_clause ::= multi_drop_clause drop_table_clause */ - { 252, -2 }, /* (117) drop_table_clause ::= exists_opt full_table_name */ - { 250, 0 }, /* (118) specific_tags_opt ::= */ - { 250, -3 }, /* (119) specific_tags_opt ::= NK_LP col_name_list NK_RP */ - { 238, -1 }, /* (120) full_table_name ::= table_name */ - { 238, -3 }, /* (121) full_table_name ::= db_name NK_DOT table_name */ - { 239, -1 }, /* (122) column_def_list ::= column_def */ - { 239, -3 }, /* (123) column_def_list ::= column_def_list NK_COMMA column_def */ - { 255, -2 }, /* (124) column_def ::= column_name type_name */ - { 255, -4 }, /* (125) column_def ::= column_name type_name COMMENT NK_STRING */ - { 248, -1 }, /* (126) type_name ::= BOOL */ - { 248, -1 }, /* (127) type_name ::= TINYINT */ - { 248, -1 }, /* (128) type_name ::= SMALLINT */ - { 248, -1 }, /* (129) type_name ::= INT */ - { 248, -1 }, /* (130) type_name ::= INTEGER */ - { 248, -1 }, /* (131) type_name ::= BIGINT */ - { 248, -1 }, /* (132) type_name ::= FLOAT */ - { 248, -1 }, /* (133) type_name ::= DOUBLE */ - { 248, -4 }, /* (134) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - { 248, -1 }, /* (135) type_name ::= TIMESTAMP */ - { 248, -4 }, /* (136) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - { 248, -2 }, /* (137) type_name ::= TINYINT UNSIGNED */ - { 248, -2 }, /* (138) type_name ::= SMALLINT UNSIGNED */ - { 248, -2 }, /* (139) type_name ::= INT UNSIGNED */ - { 248, -2 }, /* (140) type_name ::= BIGINT UNSIGNED */ - { 248, -1 }, /* (141) type_name ::= JSON */ - { 248, -4 }, /* (142) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - { 248, -1 }, /* (143) type_name ::= MEDIUMBLOB */ - { 248, -1 }, /* (144) type_name ::= BLOB */ - { 248, -4 }, /* (145) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - { 248, -1 }, /* (146) type_name ::= DECIMAL */ - { 248, -4 }, /* (147) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - { 248, -6 }, /* (148) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - { 240, 0 }, /* (149) tags_def_opt ::= */ - { 240, -1 }, /* (150) tags_def_opt ::= tags_def */ - { 243, -4 }, /* (151) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - { 241, 0 }, /* (152) table_options ::= */ - { 241, -3 }, /* (153) table_options ::= table_options COMMENT NK_STRING */ - { 241, -3 }, /* (154) table_options ::= table_options KEEP integer_list */ - { 241, -3 }, /* (155) table_options ::= table_options KEEP variable_list */ - { 241, -3 }, /* (156) table_options ::= table_options TTL NK_INTEGER */ - { 241, -5 }, /* (157) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - { 241, -5 }, /* (158) table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ - { 241, -3 }, /* (159) table_options ::= table_options FILE_FACTOR NK_FLOAT */ - { 241, -3 }, /* (160) table_options ::= table_options DELAY NK_INTEGER */ - { 246, -1 }, /* (161) alter_table_options ::= alter_table_option */ - { 246, -2 }, /* (162) alter_table_options ::= alter_table_options alter_table_option */ - { 257, -2 }, /* (163) alter_table_option ::= COMMENT NK_STRING */ - { 257, -2 }, /* (164) alter_table_option ::= KEEP integer_list */ - { 257, -2 }, /* (165) alter_table_option ::= KEEP variable_list */ - { 257, -2 }, /* (166) alter_table_option ::= TTL NK_INTEGER */ - { 253, -1 }, /* (167) col_name_list ::= col_name */ - { 253, -3 }, /* (168) col_name_list ::= col_name_list NK_COMMA col_name */ - { 258, -1 }, /* (169) col_name ::= column_name */ - { 220, -2 }, /* (170) cmd ::= SHOW DNODES */ - { 220, -2 }, /* (171) cmd ::= SHOW USERS */ - { 220, -2 }, /* (172) cmd ::= SHOW DATABASES */ - { 220, -4 }, /* (173) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ - { 220, -4 }, /* (174) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ - { 220, -3 }, /* (175) cmd ::= SHOW db_name_cond_opt VGROUPS */ - { 220, -2 }, /* (176) cmd ::= SHOW MNODES */ - { 220, -2 }, /* (177) cmd ::= SHOW MODULES */ - { 220, -2 }, /* (178) cmd ::= SHOW QNODES */ - { 220, -2 }, /* (179) cmd ::= SHOW FUNCTIONS */ - { 220, -5 }, /* (180) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - { 220, -2 }, /* (181) cmd ::= SHOW STREAMS */ - { 220, -2 }, /* (182) cmd ::= SHOW ACCOUNTS */ - { 220, -2 }, /* (183) cmd ::= SHOW APPS */ - { 220, -2 }, /* (184) cmd ::= SHOW CONNECTIONS */ - { 220, -2 }, /* (185) cmd ::= SHOW LICENCE */ - { 220, -2 }, /* (186) cmd ::= SHOW GRANTS */ - { 220, -4 }, /* (187) cmd ::= SHOW CREATE DATABASE db_name */ - { 220, -4 }, /* (188) cmd ::= SHOW CREATE TABLE full_table_name */ - { 220, -4 }, /* (189) cmd ::= SHOW CREATE STABLE full_table_name */ - { 220, -2 }, /* (190) cmd ::= SHOW QUERIES */ - { 220, -2 }, /* (191) cmd ::= SHOW SCORES */ - { 220, -2 }, /* (192) cmd ::= SHOW TOPICS */ - { 220, -2 }, /* (193) cmd ::= SHOW VARIABLES */ - { 220, -2 }, /* (194) cmd ::= SHOW BNODES */ - { 220, -2 }, /* (195) cmd ::= SHOW SNODES */ - { 259, 0 }, /* (196) db_name_cond_opt ::= */ - { 259, -2 }, /* (197) db_name_cond_opt ::= db_name NK_DOT */ - { 260, 0 }, /* (198) like_pattern_opt ::= */ - { 260, -2 }, /* (199) like_pattern_opt ::= LIKE NK_STRING */ - { 261, -1 }, /* (200) table_name_cond ::= table_name */ - { 262, 0 }, /* (201) from_db_opt ::= */ - { 262, -2 }, /* (202) from_db_opt ::= FROM db_name */ - { 256, -1 }, /* (203) func_name_list ::= func_name */ - { 256, -3 }, /* (204) func_name_list ::= func_name_list NK_COMMA col_name */ - { 263, -1 }, /* (205) func_name ::= function_name */ - { 220, -8 }, /* (206) cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ - { 220, -10 }, /* (207) cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ - { 220, -6 }, /* (208) cmd ::= DROP INDEX exists_opt index_name ON table_name */ - { 266, 0 }, /* (209) index_options ::= */ - { 266, -9 }, /* (210) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ - { 266, -11 }, /* (211) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ - { 267, -1 }, /* (212) func_list ::= func */ - { 267, -3 }, /* (213) func_list ::= func_list NK_COMMA func */ - { 270, -4 }, /* (214) func ::= function_name NK_LP expression_list NK_RP */ - { 220, -6 }, /* (215) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ - { 220, -6 }, /* (216) cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ - { 220, -4 }, /* (217) cmd ::= DROP TOPIC exists_opt topic_name */ - { 220, -2 }, /* (218) cmd ::= DESC full_table_name */ - { 220, -2 }, /* (219) cmd ::= DESCRIBE full_table_name */ - { 220, -3 }, /* (220) cmd ::= RESET QUERY CACHE */ - { 220, -4 }, /* (221) cmd ::= EXPLAIN analyze_opt explain_options query_expression */ - { 274, 0 }, /* (222) analyze_opt ::= */ - { 274, -1 }, /* (223) analyze_opt ::= ANALYZE */ - { 275, 0 }, /* (224) explain_options ::= */ - { 275, -3 }, /* (225) explain_options ::= explain_options VERBOSE NK_BOOL */ - { 275, -3 }, /* (226) explain_options ::= explain_options RATIO NK_FLOAT */ - { 220, -6 }, /* (227) cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ - { 220, -9 }, /* (228) cmd ::= CREATE agg_func_opt FUNCTION function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ - { 220, -3 }, /* (229) cmd ::= DROP FUNCTION function_name */ - { 276, 0 }, /* (230) agg_func_opt ::= */ - { 276, -1 }, /* (231) agg_func_opt ::= AGGREGATE */ - { 277, 0 }, /* (232) bufsize_opt ::= */ - { 277, -2 }, /* (233) bufsize_opt ::= BUFSIZE NK_INTEGER */ - { 220, -8 }, /* (234) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ - { 220, -4 }, /* (235) cmd ::= DROP STREAM exists_opt stream_name */ - { 280, 0 }, /* (236) into_opt ::= */ - { 280, -2 }, /* (237) into_opt ::= INTO full_table_name */ - { 279, 0 }, /* (238) stream_options ::= */ - { 279, -3 }, /* (239) stream_options ::= stream_options TRIGGER AT_ONCE */ - { 279, -3 }, /* (240) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - { 279, -3 }, /* (241) stream_options ::= stream_options WATERMARK duration_literal */ - { 220, -3 }, /* (242) cmd ::= KILL CONNECTION NK_INTEGER */ - { 220, -3 }, /* (243) cmd ::= KILL QUERY NK_INTEGER */ - { 220, -4 }, /* (244) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ - { 220, -4 }, /* (245) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ - { 220, -3 }, /* (246) cmd ::= SPLIT VGROUP NK_INTEGER */ - { 281, -2 }, /* (247) dnode_list ::= DNODE NK_INTEGER */ - { 281, -3 }, /* (248) dnode_list ::= dnode_list DNODE NK_INTEGER */ - { 220, -3 }, /* (249) cmd ::= SYNCDB db_name REPLICA */ - { 220, -1 }, /* (250) cmd ::= query_expression */ - { 223, -1 }, /* (251) literal ::= NK_INTEGER */ - { 223, -1 }, /* (252) literal ::= NK_FLOAT */ - { 223, -1 }, /* (253) literal ::= NK_STRING */ - { 223, -1 }, /* (254) literal ::= NK_BOOL */ - { 223, -2 }, /* (255) literal ::= TIMESTAMP NK_STRING */ - { 223, -1 }, /* (256) literal ::= duration_literal */ - { 223, -1 }, /* (257) literal ::= NULL */ - { 223, -1 }, /* (258) literal ::= NK_QUESTION */ - { 268, -1 }, /* (259) duration_literal ::= NK_VARIABLE */ - { 282, -1 }, /* (260) signed ::= NK_INTEGER */ - { 282, -2 }, /* (261) signed ::= NK_PLUS NK_INTEGER */ - { 282, -2 }, /* (262) signed ::= NK_MINUS NK_INTEGER */ - { 282, -1 }, /* (263) signed ::= NK_FLOAT */ - { 282, -2 }, /* (264) signed ::= NK_PLUS NK_FLOAT */ - { 282, -2 }, /* (265) signed ::= NK_MINUS NK_FLOAT */ - { 283, -1 }, /* (266) signed_literal ::= signed */ - { 283, -1 }, /* (267) signed_literal ::= NK_STRING */ - { 283, -1 }, /* (268) signed_literal ::= NK_BOOL */ - { 283, -2 }, /* (269) signed_literal ::= TIMESTAMP NK_STRING */ - { 283, -1 }, /* (270) signed_literal ::= duration_literal */ - { 283, -1 }, /* (271) signed_literal ::= NULL */ - { 251, -1 }, /* (272) literal_list ::= signed_literal */ - { 251, -3 }, /* (273) literal_list ::= literal_list NK_COMMA signed_literal */ - { 229, -1 }, /* (274) db_name ::= NK_ID */ - { 254, -1 }, /* (275) table_name ::= NK_ID */ - { 247, -1 }, /* (276) column_name ::= NK_ID */ - { 264, -1 }, /* (277) function_name ::= NK_ID */ - { 284, -1 }, /* (278) table_alias ::= NK_ID */ - { 285, -1 }, /* (279) column_alias ::= NK_ID */ - { 225, -1 }, /* (280) user_name ::= NK_ID */ - { 265, -1 }, /* (281) index_name ::= NK_ID */ - { 272, -1 }, /* (282) topic_name ::= NK_ID */ - { 278, -1 }, /* (283) stream_name ::= NK_ID */ - { 286, -1 }, /* (284) expression ::= literal */ - { 286, -1 }, /* (285) expression ::= pseudo_column */ - { 286, -1 }, /* (286) expression ::= column_reference */ - { 286, -1 }, /* (287) expression ::= function_expression */ - { 286, -1 }, /* (288) expression ::= subquery */ - { 286, -3 }, /* (289) expression ::= NK_LP expression NK_RP */ - { 286, -2 }, /* (290) expression ::= NK_PLUS expression */ - { 286, -2 }, /* (291) expression ::= NK_MINUS expression */ - { 286, -3 }, /* (292) expression ::= expression NK_PLUS expression */ - { 286, -3 }, /* (293) expression ::= expression NK_MINUS expression */ - { 286, -3 }, /* (294) expression ::= expression NK_STAR expression */ - { 286, -3 }, /* (295) expression ::= expression NK_SLASH expression */ - { 286, -3 }, /* (296) expression ::= expression NK_REM expression */ - { 286, -3 }, /* (297) expression ::= column_reference NK_ARROW NK_STRING */ - { 271, -1 }, /* (298) expression_list ::= expression */ - { 271, -3 }, /* (299) expression_list ::= expression_list NK_COMMA expression */ - { 288, -1 }, /* (300) column_reference ::= column_name */ - { 288, -3 }, /* (301) column_reference ::= table_name NK_DOT column_name */ - { 287, -1 }, /* (302) pseudo_column ::= ROWTS */ - { 287, -1 }, /* (303) pseudo_column ::= TBNAME */ - { 287, -1 }, /* (304) pseudo_column ::= QSTARTTS */ - { 287, -1 }, /* (305) pseudo_column ::= QENDTS */ - { 287, -1 }, /* (306) pseudo_column ::= WSTARTTS */ - { 287, -1 }, /* (307) pseudo_column ::= WENDTS */ - { 287, -1 }, /* (308) pseudo_column ::= WDURATION */ - { 289, -4 }, /* (309) function_expression ::= function_name NK_LP expression_list NK_RP */ - { 289, -4 }, /* (310) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - { 289, -6 }, /* (311) function_expression ::= CAST NK_LP expression AS type_name NK_RP */ - { 289, -3 }, /* (312) function_expression ::= noarg_func NK_LP NK_RP */ - { 293, -1 }, /* (313) noarg_func ::= NOW */ - { 293, -1 }, /* (314) noarg_func ::= TODAY */ - { 293, -1 }, /* (315) noarg_func ::= TIMEZONE */ - { 291, -1 }, /* (316) star_func ::= COUNT */ - { 291, -1 }, /* (317) star_func ::= FIRST */ - { 291, -1 }, /* (318) star_func ::= LAST */ - { 291, -1 }, /* (319) star_func ::= LAST_ROW */ - { 292, -1 }, /* (320) star_func_para_list ::= NK_STAR */ - { 292, -1 }, /* (321) star_func_para_list ::= other_para_list */ - { 294, -1 }, /* (322) other_para_list ::= star_func_para */ - { 294, -3 }, /* (323) other_para_list ::= other_para_list NK_COMMA star_func_para */ - { 295, -1 }, /* (324) star_func_para ::= expression */ - { 295, -3 }, /* (325) star_func_para ::= table_name NK_DOT NK_STAR */ - { 296, -3 }, /* (326) predicate ::= expression compare_op expression */ - { 296, -5 }, /* (327) predicate ::= expression BETWEEN expression AND expression */ - { 296, -6 }, /* (328) predicate ::= expression NOT BETWEEN expression AND expression */ - { 296, -3 }, /* (329) predicate ::= expression IS NULL */ - { 296, -4 }, /* (330) predicate ::= expression IS NOT NULL */ - { 296, -3 }, /* (331) predicate ::= expression in_op in_predicate_value */ - { 297, -1 }, /* (332) compare_op ::= NK_LT */ - { 297, -1 }, /* (333) compare_op ::= NK_GT */ - { 297, -1 }, /* (334) compare_op ::= NK_LE */ - { 297, -1 }, /* (335) compare_op ::= NK_GE */ - { 297, -1 }, /* (336) compare_op ::= NK_NE */ - { 297, -1 }, /* (337) compare_op ::= NK_EQ */ - { 297, -1 }, /* (338) compare_op ::= LIKE */ - { 297, -2 }, /* (339) compare_op ::= NOT LIKE */ - { 297, -1 }, /* (340) compare_op ::= MATCH */ - { 297, -1 }, /* (341) compare_op ::= NMATCH */ - { 297, -1 }, /* (342) compare_op ::= CONTAINS */ - { 298, -1 }, /* (343) in_op ::= IN */ - { 298, -2 }, /* (344) in_op ::= NOT IN */ - { 299, -3 }, /* (345) in_predicate_value ::= NK_LP expression_list NK_RP */ - { 300, -1 }, /* (346) boolean_value_expression ::= boolean_primary */ - { 300, -2 }, /* (347) boolean_value_expression ::= NOT boolean_primary */ - { 300, -3 }, /* (348) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 300, -3 }, /* (349) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 301, -1 }, /* (350) boolean_primary ::= predicate */ - { 301, -3 }, /* (351) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 302, -1 }, /* (352) common_expression ::= expression */ - { 302, -1 }, /* (353) common_expression ::= boolean_value_expression */ - { 303, -2 }, /* (354) from_clause ::= FROM table_reference_list */ - { 304, -1 }, /* (355) table_reference_list ::= table_reference */ - { 304, -3 }, /* (356) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 305, -1 }, /* (357) table_reference ::= table_primary */ - { 305, -1 }, /* (358) table_reference ::= joined_table */ - { 306, -2 }, /* (359) table_primary ::= table_name alias_opt */ - { 306, -4 }, /* (360) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 306, -2 }, /* (361) table_primary ::= subquery alias_opt */ - { 306, -1 }, /* (362) table_primary ::= parenthesized_joined_table */ - { 308, 0 }, /* (363) alias_opt ::= */ - { 308, -1 }, /* (364) alias_opt ::= table_alias */ - { 308, -2 }, /* (365) alias_opt ::= AS table_alias */ - { 309, -3 }, /* (366) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 309, -3 }, /* (367) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 307, -6 }, /* (368) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 310, 0 }, /* (369) join_type ::= */ - { 310, -1 }, /* (370) join_type ::= INNER */ - { 312, -9 }, /* (371) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 313, 0 }, /* (372) set_quantifier_opt ::= */ - { 313, -1 }, /* (373) set_quantifier_opt ::= DISTINCT */ - { 313, -1 }, /* (374) set_quantifier_opt ::= ALL */ - { 314, -1 }, /* (375) select_list ::= NK_STAR */ - { 314, -1 }, /* (376) select_list ::= select_sublist */ - { 320, -1 }, /* (377) select_sublist ::= select_item */ - { 320, -3 }, /* (378) select_sublist ::= select_sublist NK_COMMA select_item */ - { 321, -1 }, /* (379) select_item ::= common_expression */ - { 321, -2 }, /* (380) select_item ::= common_expression column_alias */ - { 321, -3 }, /* (381) select_item ::= common_expression AS column_alias */ - { 321, -3 }, /* (382) select_item ::= table_name NK_DOT NK_STAR */ - { 315, 0 }, /* (383) where_clause_opt ::= */ - { 315, -2 }, /* (384) where_clause_opt ::= WHERE search_condition */ - { 316, 0 }, /* (385) partition_by_clause_opt ::= */ - { 316, -3 }, /* (386) partition_by_clause_opt ::= PARTITION BY expression_list */ - { 317, 0 }, /* (387) twindow_clause_opt ::= */ - { 317, -6 }, /* (388) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - { 317, -4 }, /* (389) twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ - { 317, -6 }, /* (390) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 317, -8 }, /* (391) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 269, 0 }, /* (392) sliding_opt ::= */ - { 269, -4 }, /* (393) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 322, 0 }, /* (394) fill_opt ::= */ - { 322, -4 }, /* (395) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 322, -6 }, /* (396) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 323, -1 }, /* (397) fill_mode ::= NONE */ - { 323, -1 }, /* (398) fill_mode ::= PREV */ - { 323, -1 }, /* (399) fill_mode ::= NULL */ - { 323, -1 }, /* (400) fill_mode ::= LINEAR */ - { 323, -1 }, /* (401) fill_mode ::= NEXT */ - { 318, 0 }, /* (402) group_by_clause_opt ::= */ - { 318, -3 }, /* (403) group_by_clause_opt ::= GROUP BY group_by_list */ - { 324, -1 }, /* (404) group_by_list ::= expression */ - { 324, -3 }, /* (405) group_by_list ::= group_by_list NK_COMMA expression */ - { 319, 0 }, /* (406) having_clause_opt ::= */ - { 319, -2 }, /* (407) having_clause_opt ::= HAVING search_condition */ - { 273, -4 }, /* (408) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 325, -1 }, /* (409) query_expression_body ::= query_primary */ - { 325, -4 }, /* (410) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 329, -1 }, /* (411) query_primary ::= query_specification */ - { 326, 0 }, /* (412) order_by_clause_opt ::= */ - { 326, -3 }, /* (413) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 327, 0 }, /* (414) slimit_clause_opt ::= */ - { 327, -2 }, /* (415) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 327, -4 }, /* (416) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 327, -4 }, /* (417) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 328, 0 }, /* (418) limit_clause_opt ::= */ - { 328, -2 }, /* (419) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 328, -4 }, /* (420) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 328, -4 }, /* (421) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 290, -3 }, /* (422) subquery ::= NK_LP query_expression NK_RP */ - { 311, -1 }, /* (423) search_condition ::= common_expression */ - { 330, -1 }, /* (424) sort_specification_list ::= sort_specification */ - { 330, -3 }, /* (425) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 331, -3 }, /* (426) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 332, 0 }, /* (427) ordering_specification_opt ::= */ - { 332, -1 }, /* (428) ordering_specification_opt ::= ASC */ - { 332, -1 }, /* (429) ordering_specification_opt ::= DESC */ - { 333, 0 }, /* (430) null_ordering_opt ::= */ - { 333, -2 }, /* (431) null_ordering_opt ::= NULLS FIRST */ - { 333, -2 }, /* (432) null_ordering_opt ::= NULLS LAST */ + { 222, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ + { 222, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ + { 223, 0 }, /* (2) account_options ::= */ + { 223, -3 }, /* (3) account_options ::= account_options PPS literal */ + { 223, -3 }, /* (4) account_options ::= account_options TSERIES literal */ + { 223, -3 }, /* (5) account_options ::= account_options STORAGE literal */ + { 223, -3 }, /* (6) account_options ::= account_options STREAMS literal */ + { 223, -3 }, /* (7) account_options ::= account_options QTIME literal */ + { 223, -3 }, /* (8) account_options ::= account_options DBS literal */ + { 223, -3 }, /* (9) account_options ::= account_options USERS literal */ + { 223, -3 }, /* (10) account_options ::= account_options CONNS literal */ + { 223, -3 }, /* (11) account_options ::= account_options STATE literal */ + { 224, -1 }, /* (12) alter_account_options ::= alter_account_option */ + { 224, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ + { 226, -2 }, /* (14) alter_account_option ::= PASS literal */ + { 226, -2 }, /* (15) alter_account_option ::= PPS literal */ + { 226, -2 }, /* (16) alter_account_option ::= TSERIES literal */ + { 226, -2 }, /* (17) alter_account_option ::= STORAGE literal */ + { 226, -2 }, /* (18) alter_account_option ::= STREAMS literal */ + { 226, -2 }, /* (19) alter_account_option ::= QTIME literal */ + { 226, -2 }, /* (20) alter_account_option ::= DBS literal */ + { 226, -2 }, /* (21) alter_account_option ::= USERS literal */ + { 226, -2 }, /* (22) alter_account_option ::= CONNS literal */ + { 226, -2 }, /* (23) alter_account_option ::= STATE literal */ + { 222, -5 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING */ + { 222, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ + { 222, -5 }, /* (26) cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ + { 222, -3 }, /* (27) cmd ::= DROP USER user_name */ + { 222, -3 }, /* (28) cmd ::= CREATE DNODE dnode_endpoint */ + { 222, -5 }, /* (29) cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ + { 222, -3 }, /* (30) cmd ::= DROP DNODE NK_INTEGER */ + { 222, -3 }, /* (31) cmd ::= DROP DNODE dnode_endpoint */ + { 222, -4 }, /* (32) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + { 222, -5 }, /* (33) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + { 222, -4 }, /* (34) cmd ::= ALTER ALL DNODES NK_STRING */ + { 222, -5 }, /* (35) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + { 228, -1 }, /* (36) dnode_endpoint ::= NK_STRING */ + { 229, -1 }, /* (37) dnode_host_name ::= NK_ID */ + { 229, -1 }, /* (38) dnode_host_name ::= NK_IPTOKEN */ + { 222, -3 }, /* (39) cmd ::= ALTER LOCAL NK_STRING */ + { 222, -4 }, /* (40) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + { 222, -5 }, /* (41) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + { 222, -5 }, /* (42) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + { 222, -5 }, /* (43) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + { 222, -5 }, /* (44) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + { 222, -5 }, /* (45) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + { 222, -5 }, /* (46) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + { 222, -5 }, /* (47) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + { 222, -5 }, /* (48) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + { 222, -5 }, /* (49) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + { 222, -4 }, /* (50) cmd ::= DROP DATABASE exists_opt db_name */ + { 222, -2 }, /* (51) cmd ::= USE db_name */ + { 222, -4 }, /* (52) cmd ::= ALTER DATABASE db_name alter_db_options */ + { 230, -3 }, /* (53) not_exists_opt ::= IF NOT EXISTS */ + { 230, 0 }, /* (54) not_exists_opt ::= */ + { 233, -2 }, /* (55) exists_opt ::= IF EXISTS */ + { 233, 0 }, /* (56) exists_opt ::= */ + { 232, 0 }, /* (57) db_options ::= */ + { 232, -3 }, /* (58) db_options ::= db_options BLOCKS NK_INTEGER */ + { 232, -3 }, /* (59) db_options ::= db_options CACHE NK_INTEGER */ + { 232, -3 }, /* (60) db_options ::= db_options CACHELAST NK_INTEGER */ + { 232, -3 }, /* (61) db_options ::= db_options COMP NK_INTEGER */ + { 232, -3 }, /* (62) db_options ::= db_options DAYS NK_INTEGER */ + { 232, -3 }, /* (63) db_options ::= db_options DAYS NK_VARIABLE */ + { 232, -3 }, /* (64) db_options ::= db_options FSYNC NK_INTEGER */ + { 232, -3 }, /* (65) db_options ::= db_options MAXROWS NK_INTEGER */ + { 232, -3 }, /* (66) db_options ::= db_options MINROWS NK_INTEGER */ + { 232, -3 }, /* (67) db_options ::= db_options KEEP integer_list */ + { 232, -3 }, /* (68) db_options ::= db_options KEEP variable_list */ + { 232, -3 }, /* (69) db_options ::= db_options PRECISION NK_STRING */ + { 232, -3 }, /* (70) db_options ::= db_options QUORUM NK_INTEGER */ + { 232, -3 }, /* (71) db_options ::= db_options REPLICA NK_INTEGER */ + { 232, -3 }, /* (72) db_options ::= db_options TTL NK_INTEGER */ + { 232, -3 }, /* (73) db_options ::= db_options WAL NK_INTEGER */ + { 232, -3 }, /* (74) db_options ::= db_options VGROUPS NK_INTEGER */ + { 232, -3 }, /* (75) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + { 232, -3 }, /* (76) db_options ::= db_options STREAM_MODE NK_INTEGER */ + { 232, -3 }, /* (77) db_options ::= db_options RETENTIONS retention_list */ + { 232, -3 }, /* (78) db_options ::= db_options STRICT NK_INTEGER */ + { 234, -1 }, /* (79) alter_db_options ::= alter_db_option */ + { 234, -2 }, /* (80) alter_db_options ::= alter_db_options alter_db_option */ + { 238, -2 }, /* (81) alter_db_option ::= BLOCKS NK_INTEGER */ + { 238, -2 }, /* (82) alter_db_option ::= FSYNC NK_INTEGER */ + { 238, -2 }, /* (83) alter_db_option ::= KEEP integer_list */ + { 238, -2 }, /* (84) alter_db_option ::= KEEP variable_list */ + { 238, -2 }, /* (85) alter_db_option ::= WAL NK_INTEGER */ + { 238, -2 }, /* (86) alter_db_option ::= QUORUM NK_INTEGER */ + { 238, -2 }, /* (87) alter_db_option ::= CACHELAST NK_INTEGER */ + { 238, -2 }, /* (88) alter_db_option ::= REPLICA NK_INTEGER */ + { 238, -2 }, /* (89) alter_db_option ::= STRICT NK_INTEGER */ + { 235, -1 }, /* (90) integer_list ::= NK_INTEGER */ + { 235, -3 }, /* (91) integer_list ::= integer_list NK_COMMA NK_INTEGER */ + { 236, -1 }, /* (92) variable_list ::= NK_VARIABLE */ + { 236, -3 }, /* (93) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ + { 237, -1 }, /* (94) retention_list ::= retention */ + { 237, -3 }, /* (95) retention_list ::= retention_list NK_COMMA retention */ + { 239, -3 }, /* (96) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ + { 222, -9 }, /* (97) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + { 222, -3 }, /* (98) cmd ::= CREATE TABLE multi_create_clause */ + { 222, -9 }, /* (99) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + { 222, -3 }, /* (100) cmd ::= DROP TABLE multi_drop_clause */ + { 222, -4 }, /* (101) cmd ::= DROP STABLE exists_opt full_table_name */ + { 222, -3 }, /* (102) cmd ::= ALTER TABLE alter_table_clause */ + { 222, -3 }, /* (103) cmd ::= ALTER STABLE alter_table_clause */ + { 247, -2 }, /* (104) alter_table_clause ::= full_table_name alter_table_options */ + { 247, -5 }, /* (105) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + { 247, -4 }, /* (106) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + { 247, -5 }, /* (107) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + { 247, -5 }, /* (108) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + { 247, -5 }, /* (109) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + { 247, -4 }, /* (110) alter_table_clause ::= full_table_name DROP TAG column_name */ + { 247, -5 }, /* (111) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + { 247, -5 }, /* (112) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + { 247, -6 }, /* (113) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ + { 244, -1 }, /* (114) multi_create_clause ::= create_subtable_clause */ + { 244, -2 }, /* (115) multi_create_clause ::= multi_create_clause create_subtable_clause */ + { 251, -9 }, /* (116) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ + { 246, -1 }, /* (117) multi_drop_clause ::= drop_table_clause */ + { 246, -2 }, /* (118) multi_drop_clause ::= multi_drop_clause drop_table_clause */ + { 254, -2 }, /* (119) drop_table_clause ::= exists_opt full_table_name */ + { 252, 0 }, /* (120) specific_tags_opt ::= */ + { 252, -3 }, /* (121) specific_tags_opt ::= NK_LP col_name_list NK_RP */ + { 240, -1 }, /* (122) full_table_name ::= table_name */ + { 240, -3 }, /* (123) full_table_name ::= db_name NK_DOT table_name */ + { 241, -1 }, /* (124) column_def_list ::= column_def */ + { 241, -3 }, /* (125) column_def_list ::= column_def_list NK_COMMA column_def */ + { 257, -2 }, /* (126) column_def ::= column_name type_name */ + { 257, -4 }, /* (127) column_def ::= column_name type_name COMMENT NK_STRING */ + { 250, -1 }, /* (128) type_name ::= BOOL */ + { 250, -1 }, /* (129) type_name ::= TINYINT */ + { 250, -1 }, /* (130) type_name ::= SMALLINT */ + { 250, -1 }, /* (131) type_name ::= INT */ + { 250, -1 }, /* (132) type_name ::= INTEGER */ + { 250, -1 }, /* (133) type_name ::= BIGINT */ + { 250, -1 }, /* (134) type_name ::= FLOAT */ + { 250, -1 }, /* (135) type_name ::= DOUBLE */ + { 250, -4 }, /* (136) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + { 250, -1 }, /* (137) type_name ::= TIMESTAMP */ + { 250, -4 }, /* (138) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + { 250, -2 }, /* (139) type_name ::= TINYINT UNSIGNED */ + { 250, -2 }, /* (140) type_name ::= SMALLINT UNSIGNED */ + { 250, -2 }, /* (141) type_name ::= INT UNSIGNED */ + { 250, -2 }, /* (142) type_name ::= BIGINT UNSIGNED */ + { 250, -1 }, /* (143) type_name ::= JSON */ + { 250, -4 }, /* (144) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + { 250, -1 }, /* (145) type_name ::= MEDIUMBLOB */ + { 250, -1 }, /* (146) type_name ::= BLOB */ + { 250, -4 }, /* (147) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + { 250, -1 }, /* (148) type_name ::= DECIMAL */ + { 250, -4 }, /* (149) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + { 250, -6 }, /* (150) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + { 242, 0 }, /* (151) tags_def_opt ::= */ + { 242, -1 }, /* (152) tags_def_opt ::= tags_def */ + { 245, -4 }, /* (153) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + { 243, 0 }, /* (154) table_options ::= */ + { 243, -3 }, /* (155) table_options ::= table_options COMMENT NK_STRING */ + { 243, -3 }, /* (156) table_options ::= table_options KEEP integer_list */ + { 243, -3 }, /* (157) table_options ::= table_options KEEP variable_list */ + { 243, -3 }, /* (158) table_options ::= table_options TTL NK_INTEGER */ + { 243, -5 }, /* (159) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + { 243, -5 }, /* (160) table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ + { 243, -3 }, /* (161) table_options ::= table_options FILE_FACTOR NK_FLOAT */ + { 243, -3 }, /* (162) table_options ::= table_options DELAY NK_INTEGER */ + { 248, -1 }, /* (163) alter_table_options ::= alter_table_option */ + { 248, -2 }, /* (164) alter_table_options ::= alter_table_options alter_table_option */ + { 259, -2 }, /* (165) alter_table_option ::= COMMENT NK_STRING */ + { 259, -2 }, /* (166) alter_table_option ::= KEEP integer_list */ + { 259, -2 }, /* (167) alter_table_option ::= KEEP variable_list */ + { 259, -2 }, /* (168) alter_table_option ::= TTL NK_INTEGER */ + { 255, -1 }, /* (169) col_name_list ::= col_name */ + { 255, -3 }, /* (170) col_name_list ::= col_name_list NK_COMMA col_name */ + { 260, -1 }, /* (171) col_name ::= column_name */ + { 222, -2 }, /* (172) cmd ::= SHOW DNODES */ + { 222, -2 }, /* (173) cmd ::= SHOW USERS */ + { 222, -2 }, /* (174) cmd ::= SHOW DATABASES */ + { 222, -4 }, /* (175) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + { 222, -4 }, /* (176) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + { 222, -3 }, /* (177) cmd ::= SHOW db_name_cond_opt VGROUPS */ + { 222, -2 }, /* (178) cmd ::= SHOW MNODES */ + { 222, -2 }, /* (179) cmd ::= SHOW MODULES */ + { 222, -2 }, /* (180) cmd ::= SHOW QNODES */ + { 222, -2 }, /* (181) cmd ::= SHOW FUNCTIONS */ + { 222, -5 }, /* (182) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + { 222, -2 }, /* (183) cmd ::= SHOW STREAMS */ + { 222, -2 }, /* (184) cmd ::= SHOW ACCOUNTS */ + { 222, -2 }, /* (185) cmd ::= SHOW APPS */ + { 222, -2 }, /* (186) cmd ::= SHOW CONNECTIONS */ + { 222, -2 }, /* (187) cmd ::= SHOW LICENCE */ + { 222, -2 }, /* (188) cmd ::= SHOW GRANTS */ + { 222, -4 }, /* (189) cmd ::= SHOW CREATE DATABASE db_name */ + { 222, -4 }, /* (190) cmd ::= SHOW CREATE TABLE full_table_name */ + { 222, -4 }, /* (191) cmd ::= SHOW CREATE STABLE full_table_name */ + { 222, -2 }, /* (192) cmd ::= SHOW QUERIES */ + { 222, -2 }, /* (193) cmd ::= SHOW SCORES */ + { 222, -2 }, /* (194) cmd ::= SHOW TOPICS */ + { 222, -2 }, /* (195) cmd ::= SHOW VARIABLES */ + { 222, -2 }, /* (196) cmd ::= SHOW BNODES */ + { 222, -2 }, /* (197) cmd ::= SHOW SNODES */ + { 222, -2 }, /* (198) cmd ::= SHOW CLUSTER */ + { 261, 0 }, /* (199) db_name_cond_opt ::= */ + { 261, -2 }, /* (200) db_name_cond_opt ::= db_name NK_DOT */ + { 262, 0 }, /* (201) like_pattern_opt ::= */ + { 262, -2 }, /* (202) like_pattern_opt ::= LIKE NK_STRING */ + { 263, -1 }, /* (203) table_name_cond ::= table_name */ + { 264, 0 }, /* (204) from_db_opt ::= */ + { 264, -2 }, /* (205) from_db_opt ::= FROM db_name */ + { 258, -1 }, /* (206) func_name_list ::= func_name */ + { 258, -3 }, /* (207) func_name_list ::= func_name_list NK_COMMA func_name */ + { 265, -1 }, /* (208) func_name ::= function_name */ + { 222, -8 }, /* (209) cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ + { 222, -10 }, /* (210) cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ + { 222, -6 }, /* (211) cmd ::= DROP INDEX exists_opt index_name ON table_name */ + { 268, 0 }, /* (212) index_options ::= */ + { 268, -9 }, /* (213) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ + { 268, -11 }, /* (214) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ + { 269, -1 }, /* (215) func_list ::= func */ + { 269, -3 }, /* (216) func_list ::= func_list NK_COMMA func */ + { 272, -4 }, /* (217) func ::= function_name NK_LP expression_list NK_RP */ + { 222, -6 }, /* (218) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ + { 222, -6 }, /* (219) cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ + { 222, -4 }, /* (220) cmd ::= DROP TOPIC exists_opt topic_name */ + { 222, -2 }, /* (221) cmd ::= DESC full_table_name */ + { 222, -2 }, /* (222) cmd ::= DESCRIBE full_table_name */ + { 222, -3 }, /* (223) cmd ::= RESET QUERY CACHE */ + { 222, -4 }, /* (224) cmd ::= EXPLAIN analyze_opt explain_options query_expression */ + { 276, 0 }, /* (225) analyze_opt ::= */ + { 276, -1 }, /* (226) analyze_opt ::= ANALYZE */ + { 277, 0 }, /* (227) explain_options ::= */ + { 277, -3 }, /* (228) explain_options ::= explain_options VERBOSE NK_BOOL */ + { 277, -3 }, /* (229) explain_options ::= explain_options RATIO NK_FLOAT */ + { 222, -6 }, /* (230) cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ + { 222, -10 }, /* (231) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ + { 222, -3 }, /* (232) cmd ::= DROP FUNCTION function_name */ + { 278, 0 }, /* (233) agg_func_opt ::= */ + { 278, -1 }, /* (234) agg_func_opt ::= AGGREGATE */ + { 279, 0 }, /* (235) bufsize_opt ::= */ + { 279, -2 }, /* (236) bufsize_opt ::= BUFSIZE NK_INTEGER */ + { 222, -8 }, /* (237) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ + { 222, -4 }, /* (238) cmd ::= DROP STREAM exists_opt stream_name */ + { 282, 0 }, /* (239) into_opt ::= */ + { 282, -2 }, /* (240) into_opt ::= INTO full_table_name */ + { 281, 0 }, /* (241) stream_options ::= */ + { 281, -3 }, /* (242) stream_options ::= stream_options TRIGGER AT_ONCE */ + { 281, -3 }, /* (243) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + { 281, -3 }, /* (244) stream_options ::= stream_options WATERMARK duration_literal */ + { 222, -3 }, /* (245) cmd ::= KILL CONNECTION NK_INTEGER */ + { 222, -3 }, /* (246) cmd ::= KILL QUERY NK_INTEGER */ + { 222, -4 }, /* (247) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + { 222, -4 }, /* (248) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + { 222, -3 }, /* (249) cmd ::= SPLIT VGROUP NK_INTEGER */ + { 283, -2 }, /* (250) dnode_list ::= DNODE NK_INTEGER */ + { 283, -3 }, /* (251) dnode_list ::= dnode_list DNODE NK_INTEGER */ + { 222, -3 }, /* (252) cmd ::= SYNCDB db_name REPLICA */ + { 222, -1 }, /* (253) cmd ::= query_expression */ + { 225, -1 }, /* (254) literal ::= NK_INTEGER */ + { 225, -1 }, /* (255) literal ::= NK_FLOAT */ + { 225, -1 }, /* (256) literal ::= NK_STRING */ + { 225, -1 }, /* (257) literal ::= NK_BOOL */ + { 225, -2 }, /* (258) literal ::= TIMESTAMP NK_STRING */ + { 225, -1 }, /* (259) literal ::= duration_literal */ + { 225, -1 }, /* (260) literal ::= NULL */ + { 225, -1 }, /* (261) literal ::= NK_QUESTION */ + { 270, -1 }, /* (262) duration_literal ::= NK_VARIABLE */ + { 284, -1 }, /* (263) signed ::= NK_INTEGER */ + { 284, -2 }, /* (264) signed ::= NK_PLUS NK_INTEGER */ + { 284, -2 }, /* (265) signed ::= NK_MINUS NK_INTEGER */ + { 284, -1 }, /* (266) signed ::= NK_FLOAT */ + { 284, -2 }, /* (267) signed ::= NK_PLUS NK_FLOAT */ + { 284, -2 }, /* (268) signed ::= NK_MINUS NK_FLOAT */ + { 285, -1 }, /* (269) signed_literal ::= signed */ + { 285, -1 }, /* (270) signed_literal ::= NK_STRING */ + { 285, -1 }, /* (271) signed_literal ::= NK_BOOL */ + { 285, -2 }, /* (272) signed_literal ::= TIMESTAMP NK_STRING */ + { 285, -1 }, /* (273) signed_literal ::= duration_literal */ + { 285, -1 }, /* (274) signed_literal ::= NULL */ + { 253, -1 }, /* (275) literal_list ::= signed_literal */ + { 253, -3 }, /* (276) literal_list ::= literal_list NK_COMMA signed_literal */ + { 231, -1 }, /* (277) db_name ::= NK_ID */ + { 256, -1 }, /* (278) table_name ::= NK_ID */ + { 249, -1 }, /* (279) column_name ::= NK_ID */ + { 266, -1 }, /* (280) function_name ::= NK_ID */ + { 286, -1 }, /* (281) table_alias ::= NK_ID */ + { 287, -1 }, /* (282) column_alias ::= NK_ID */ + { 227, -1 }, /* (283) user_name ::= NK_ID */ + { 267, -1 }, /* (284) index_name ::= NK_ID */ + { 274, -1 }, /* (285) topic_name ::= NK_ID */ + { 280, -1 }, /* (286) stream_name ::= NK_ID */ + { 288, -1 }, /* (287) expression ::= literal */ + { 288, -1 }, /* (288) expression ::= pseudo_column */ + { 288, -1 }, /* (289) expression ::= column_reference */ + { 288, -1 }, /* (290) expression ::= function_expression */ + { 288, -1 }, /* (291) expression ::= subquery */ + { 288, -3 }, /* (292) expression ::= NK_LP expression NK_RP */ + { 288, -2 }, /* (293) expression ::= NK_PLUS expression */ + { 288, -2 }, /* (294) expression ::= NK_MINUS expression */ + { 288, -3 }, /* (295) expression ::= expression NK_PLUS expression */ + { 288, -3 }, /* (296) expression ::= expression NK_MINUS expression */ + { 288, -3 }, /* (297) expression ::= expression NK_STAR expression */ + { 288, -3 }, /* (298) expression ::= expression NK_SLASH expression */ + { 288, -3 }, /* (299) expression ::= expression NK_REM expression */ + { 288, -3 }, /* (300) expression ::= column_reference NK_ARROW NK_STRING */ + { 273, -1 }, /* (301) expression_list ::= expression */ + { 273, -3 }, /* (302) expression_list ::= expression_list NK_COMMA expression */ + { 290, -1 }, /* (303) column_reference ::= column_name */ + { 290, -3 }, /* (304) column_reference ::= table_name NK_DOT column_name */ + { 289, -1 }, /* (305) pseudo_column ::= ROWTS */ + { 289, -1 }, /* (306) pseudo_column ::= TBNAME */ + { 289, -1 }, /* (307) pseudo_column ::= QSTARTTS */ + { 289, -1 }, /* (308) pseudo_column ::= QENDTS */ + { 289, -1 }, /* (309) pseudo_column ::= WSTARTTS */ + { 289, -1 }, /* (310) pseudo_column ::= WENDTS */ + { 289, -1 }, /* (311) pseudo_column ::= WDURATION */ + { 291, -4 }, /* (312) function_expression ::= function_name NK_LP expression_list NK_RP */ + { 291, -4 }, /* (313) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + { 291, -6 }, /* (314) function_expression ::= CAST NK_LP expression AS type_name NK_RP */ + { 291, -3 }, /* (315) function_expression ::= noarg_func NK_LP NK_RP */ + { 295, -1 }, /* (316) noarg_func ::= NOW */ + { 295, -1 }, /* (317) noarg_func ::= TODAY */ + { 295, -1 }, /* (318) noarg_func ::= TIMEZONE */ + { 293, -1 }, /* (319) star_func ::= COUNT */ + { 293, -1 }, /* (320) star_func ::= FIRST */ + { 293, -1 }, /* (321) star_func ::= LAST */ + { 293, -1 }, /* (322) star_func ::= LAST_ROW */ + { 294, -1 }, /* (323) star_func_para_list ::= NK_STAR */ + { 294, -1 }, /* (324) star_func_para_list ::= other_para_list */ + { 296, -1 }, /* (325) other_para_list ::= star_func_para */ + { 296, -3 }, /* (326) other_para_list ::= other_para_list NK_COMMA star_func_para */ + { 297, -1 }, /* (327) star_func_para ::= expression */ + { 297, -3 }, /* (328) star_func_para ::= table_name NK_DOT NK_STAR */ + { 298, -3 }, /* (329) predicate ::= expression compare_op expression */ + { 298, -5 }, /* (330) predicate ::= expression BETWEEN expression AND expression */ + { 298, -6 }, /* (331) predicate ::= expression NOT BETWEEN expression AND expression */ + { 298, -3 }, /* (332) predicate ::= expression IS NULL */ + { 298, -4 }, /* (333) predicate ::= expression IS NOT NULL */ + { 298, -3 }, /* (334) predicate ::= expression in_op in_predicate_value */ + { 299, -1 }, /* (335) compare_op ::= NK_LT */ + { 299, -1 }, /* (336) compare_op ::= NK_GT */ + { 299, -1 }, /* (337) compare_op ::= NK_LE */ + { 299, -1 }, /* (338) compare_op ::= NK_GE */ + { 299, -1 }, /* (339) compare_op ::= NK_NE */ + { 299, -1 }, /* (340) compare_op ::= NK_EQ */ + { 299, -1 }, /* (341) compare_op ::= LIKE */ + { 299, -2 }, /* (342) compare_op ::= NOT LIKE */ + { 299, -1 }, /* (343) compare_op ::= MATCH */ + { 299, -1 }, /* (344) compare_op ::= NMATCH */ + { 299, -1 }, /* (345) compare_op ::= CONTAINS */ + { 300, -1 }, /* (346) in_op ::= IN */ + { 300, -2 }, /* (347) in_op ::= NOT IN */ + { 301, -3 }, /* (348) in_predicate_value ::= NK_LP expression_list NK_RP */ + { 302, -1 }, /* (349) boolean_value_expression ::= boolean_primary */ + { 302, -2 }, /* (350) boolean_value_expression ::= NOT boolean_primary */ + { 302, -3 }, /* (351) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 302, -3 }, /* (352) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 303, -1 }, /* (353) boolean_primary ::= predicate */ + { 303, -3 }, /* (354) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 304, -1 }, /* (355) common_expression ::= expression */ + { 304, -1 }, /* (356) common_expression ::= boolean_value_expression */ + { 305, -2 }, /* (357) from_clause ::= FROM table_reference_list */ + { 306, -1 }, /* (358) table_reference_list ::= table_reference */ + { 306, -3 }, /* (359) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 307, -1 }, /* (360) table_reference ::= table_primary */ + { 307, -1 }, /* (361) table_reference ::= joined_table */ + { 308, -2 }, /* (362) table_primary ::= table_name alias_opt */ + { 308, -4 }, /* (363) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 308, -2 }, /* (364) table_primary ::= subquery alias_opt */ + { 308, -1 }, /* (365) table_primary ::= parenthesized_joined_table */ + { 310, 0 }, /* (366) alias_opt ::= */ + { 310, -1 }, /* (367) alias_opt ::= table_alias */ + { 310, -2 }, /* (368) alias_opt ::= AS table_alias */ + { 311, -3 }, /* (369) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 311, -3 }, /* (370) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 309, -6 }, /* (371) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 312, 0 }, /* (372) join_type ::= */ + { 312, -1 }, /* (373) join_type ::= INNER */ + { 314, -9 }, /* (374) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 315, 0 }, /* (375) set_quantifier_opt ::= */ + { 315, -1 }, /* (376) set_quantifier_opt ::= DISTINCT */ + { 315, -1 }, /* (377) set_quantifier_opt ::= ALL */ + { 316, -1 }, /* (378) select_list ::= NK_STAR */ + { 316, -1 }, /* (379) select_list ::= select_sublist */ + { 322, -1 }, /* (380) select_sublist ::= select_item */ + { 322, -3 }, /* (381) select_sublist ::= select_sublist NK_COMMA select_item */ + { 323, -1 }, /* (382) select_item ::= common_expression */ + { 323, -2 }, /* (383) select_item ::= common_expression column_alias */ + { 323, -3 }, /* (384) select_item ::= common_expression AS column_alias */ + { 323, -3 }, /* (385) select_item ::= table_name NK_DOT NK_STAR */ + { 317, 0 }, /* (386) where_clause_opt ::= */ + { 317, -2 }, /* (387) where_clause_opt ::= WHERE search_condition */ + { 318, 0 }, /* (388) partition_by_clause_opt ::= */ + { 318, -3 }, /* (389) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 319, 0 }, /* (390) twindow_clause_opt ::= */ + { 319, -6 }, /* (391) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + { 319, -4 }, /* (392) twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ + { 319, -6 }, /* (393) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 319, -8 }, /* (394) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 271, 0 }, /* (395) sliding_opt ::= */ + { 271, -4 }, /* (396) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 324, 0 }, /* (397) fill_opt ::= */ + { 324, -4 }, /* (398) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 324, -6 }, /* (399) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 325, -1 }, /* (400) fill_mode ::= NONE */ + { 325, -1 }, /* (401) fill_mode ::= PREV */ + { 325, -1 }, /* (402) fill_mode ::= NULL */ + { 325, -1 }, /* (403) fill_mode ::= LINEAR */ + { 325, -1 }, /* (404) fill_mode ::= NEXT */ + { 320, 0 }, /* (405) group_by_clause_opt ::= */ + { 320, -3 }, /* (406) group_by_clause_opt ::= GROUP BY group_by_list */ + { 326, -1 }, /* (407) group_by_list ::= expression */ + { 326, -3 }, /* (408) group_by_list ::= group_by_list NK_COMMA expression */ + { 321, 0 }, /* (409) having_clause_opt ::= */ + { 321, -2 }, /* (410) having_clause_opt ::= HAVING search_condition */ + { 275, -4 }, /* (411) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 327, -1 }, /* (412) query_expression_body ::= query_primary */ + { 327, -4 }, /* (413) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 327, -3 }, /* (414) query_expression_body ::= query_expression_body UNION query_expression_body */ + { 331, -1 }, /* (415) query_primary ::= query_specification */ + { 328, 0 }, /* (416) order_by_clause_opt ::= */ + { 328, -3 }, /* (417) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 329, 0 }, /* (418) slimit_clause_opt ::= */ + { 329, -2 }, /* (419) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 329, -4 }, /* (420) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 329, -4 }, /* (421) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 330, 0 }, /* (422) limit_clause_opt ::= */ + { 330, -2 }, /* (423) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 330, -4 }, /* (424) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 330, -4 }, /* (425) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 292, -3 }, /* (426) subquery ::= NK_LP query_expression NK_RP */ + { 313, -1 }, /* (427) search_condition ::= common_expression */ + { 332, -1 }, /* (428) sort_specification_list ::= sort_specification */ + { 332, -3 }, /* (429) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 333, -3 }, /* (430) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 334, 0 }, /* (431) ordering_specification_opt ::= */ + { 334, -1 }, /* (432) ordering_specification_opt ::= ASC */ + { 334, -1 }, /* (433) ordering_specification_opt ::= DESC */ + { 335, 0 }, /* (434) null_ordering_opt ::= */ + { 335, -2 }, /* (435) null_ordering_opt ::= NULLS FIRST */ + { 335, -2 }, /* (436) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2720,11 +2750,11 @@ static YYACTIONTYPE yy_reduce( YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,221,&yymsp[0].minor); + yy_destructor(yypParser,223,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,222,&yymsp[0].minor); + yy_destructor(yypParser,224,&yymsp[0].minor); break; case 2: /* account_options ::= */ { } @@ -2738,20 +2768,20 @@ static YYACTIONTYPE yy_reduce( case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); -{ yy_destructor(yypParser,221,&yymsp[-2].minor); +{ yy_destructor(yypParser,223,&yymsp[-2].minor); { } - yy_destructor(yypParser,223,&yymsp[0].minor); + yy_destructor(yypParser,225,&yymsp[0].minor); } break; case 12: /* alter_account_options ::= alter_account_option */ -{ yy_destructor(yypParser,224,&yymsp[0].minor); +{ yy_destructor(yypParser,226,&yymsp[0].minor); { } } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ -{ yy_destructor(yypParser,222,&yymsp[-1].minor); +{ yy_destructor(yypParser,224,&yymsp[-1].minor); { } - yy_destructor(yypParser,224,&yymsp[0].minor); + yy_destructor(yypParser,226,&yymsp[0].minor); } break; case 14: /* alter_account_option ::= PASS literal */ @@ -2765,31 +2795,31 @@ static YYACTIONTYPE yy_reduce( case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); { } - yy_destructor(yypParser,223,&yymsp[0].minor); + yy_destructor(yypParser,225,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy537, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy0); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy537, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy449, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy537, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy449, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy537); } +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy449); } break; case 28: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy537, NULL); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy449, NULL); } break; case 29: /* cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy537, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy0); } break; case 30: /* cmd ::= DROP DNODE NK_INTEGER */ { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy0); } break; case 31: /* cmd ::= DROP DNODE dnode_endpoint */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy537); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy449); } break; case 32: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } @@ -2806,25 +2836,25 @@ static YYACTIONTYPE yy_reduce( case 36: /* dnode_endpoint ::= NK_STRING */ case 37: /* dnode_host_name ::= NK_ID */ yytestcase(yyruleno==37); case 38: /* dnode_host_name ::= NK_IPTOKEN */ yytestcase(yyruleno==38); - case 274: /* db_name ::= NK_ID */ yytestcase(yyruleno==274); - case 275: /* table_name ::= NK_ID */ yytestcase(yyruleno==275); - case 276: /* column_name ::= NK_ID */ yytestcase(yyruleno==276); - case 277: /* function_name ::= NK_ID */ yytestcase(yyruleno==277); - case 278: /* table_alias ::= NK_ID */ yytestcase(yyruleno==278); - case 279: /* column_alias ::= NK_ID */ yytestcase(yyruleno==279); - case 280: /* user_name ::= NK_ID */ yytestcase(yyruleno==280); - case 281: /* index_name ::= NK_ID */ yytestcase(yyruleno==281); - case 282: /* topic_name ::= NK_ID */ yytestcase(yyruleno==282); - case 283: /* stream_name ::= NK_ID */ yytestcase(yyruleno==283); - case 313: /* noarg_func ::= NOW */ yytestcase(yyruleno==313); - case 314: /* noarg_func ::= TODAY */ yytestcase(yyruleno==314); - case 315: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==315); - case 316: /* star_func ::= COUNT */ yytestcase(yyruleno==316); - case 317: /* star_func ::= FIRST */ yytestcase(yyruleno==317); - case 318: /* star_func ::= LAST */ yytestcase(yyruleno==318); - case 319: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==319); -{ yylhsminor.yy537 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy537 = yylhsminor.yy537; + case 277: /* db_name ::= NK_ID */ yytestcase(yyruleno==277); + case 278: /* table_name ::= NK_ID */ yytestcase(yyruleno==278); + case 279: /* column_name ::= NK_ID */ yytestcase(yyruleno==279); + case 280: /* function_name ::= NK_ID */ yytestcase(yyruleno==280); + case 281: /* table_alias ::= NK_ID */ yytestcase(yyruleno==281); + case 282: /* column_alias ::= NK_ID */ yytestcase(yyruleno==282); + case 283: /* user_name ::= NK_ID */ yytestcase(yyruleno==283); + case 284: /* index_name ::= NK_ID */ yytestcase(yyruleno==284); + case 285: /* topic_name ::= NK_ID */ yytestcase(yyruleno==285); + case 286: /* stream_name ::= NK_ID */ yytestcase(yyruleno==286); + case 316: /* noarg_func ::= NOW */ yytestcase(yyruleno==316); + case 317: /* noarg_func ::= TODAY */ yytestcase(yyruleno==317); + case 318: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==318); + case 319: /* star_func ::= COUNT */ yytestcase(yyruleno==319); + case 320: /* star_func ::= FIRST */ yytestcase(yyruleno==320); + case 321: /* star_func ::= LAST */ yytestcase(yyruleno==321); + case 322: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==322); +{ yylhsminor.yy449 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy449 = yylhsminor.yy449; break; case 39: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } @@ -2857,1125 +2887,1139 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } break; case 49: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy649, &yymsp[-1].minor.yy537, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy89, &yymsp[-1].minor.yy449, yymsp[0].minor.yy392); } break; case 50: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy649, &yymsp[0].minor.yy537); } +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy89, &yymsp[0].minor.yy449); } break; case 51: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy537); } +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy449); } break; case 52: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy537, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy449, yymsp[0].minor.yy392); } break; case 53: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy649 = true; } +{ yymsp[-2].minor.yy89 = true; } break; case 54: /* not_exists_opt ::= */ case 56: /* exists_opt ::= */ yytestcase(yyruleno==56); - case 222: /* analyze_opt ::= */ yytestcase(yyruleno==222); - case 230: /* agg_func_opt ::= */ yytestcase(yyruleno==230); - case 372: /* set_quantifier_opt ::= */ yytestcase(yyruleno==372); -{ yymsp[1].minor.yy649 = false; } + case 225: /* analyze_opt ::= */ yytestcase(yyruleno==225); + case 233: /* agg_func_opt ::= */ yytestcase(yyruleno==233); + case 375: /* set_quantifier_opt ::= */ yytestcase(yyruleno==375); +{ yymsp[1].minor.yy89 = false; } break; case 55: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy649 = true; } +{ yymsp[-1].minor.yy89 = true; } break; case 57: /* db_options ::= */ -{ yymsp[1].minor.yy456 = createDatabaseOptions(pCxt); } +{ yymsp[1].minor.yy392 = createDatabaseOptions(pCxt); } break; case 58: /* db_options ::= db_options BLOCKS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pNumOfBlocks = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pNumOfBlocks = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 59: /* db_options ::= db_options CACHE NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pCacheBlockSize = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pCacheBlockSize = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 60: /* db_options ::= db_options CACHELAST NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pCachelast = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pCachelast = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 61: /* db_options ::= db_options COMP NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pCompressionLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pCompressionLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 62: /* db_options ::= db_options DAYS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pDaysPerFile = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pDaysPerFile = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 63: /* db_options ::= db_options DAYS NK_VARIABLE */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 64: /* db_options ::= db_options FSYNC NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pFsyncPeriod = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pFsyncPeriod = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 65: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pMaxRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pMaxRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 66: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pMinRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pMinRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 67: /* db_options ::= db_options KEEP integer_list */ case 68: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==68); -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pKeep = yymsp[0].minor.yy632; yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pKeep = yymsp[0].minor.yy376; yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 69: /* db_options ::= db_options PRECISION NK_STRING */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pPrecision = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pPrecision = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 70: /* db_options ::= db_options QUORUM NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pQuorum = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pQuorum = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 71: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pReplica = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pReplica = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 72: /* db_options ::= db_options TTL NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 73: /* db_options ::= db_options WAL NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pWalLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pWalLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 74: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pNumOfVgroups = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pNumOfVgroups = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 75: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pSingleStable = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pSingleStable = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 76: /* db_options ::= db_options STREAM_MODE NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pStreamMode = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pStreamMode = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; case 77: /* db_options ::= db_options RETENTIONS retention_list */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pRetentions = yymsp[0].minor.yy632; yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; - break; - case 78: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy456 = createDatabaseOptions(pCxt); yylhsminor.yy456 = setDatabaseAlterOption(pCxt, yylhsminor.yy456, &yymsp[0].minor.yy29); } - yymsp[0].minor.yy456 = yylhsminor.yy456; - break; - case 79: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy456 = setDatabaseAlterOption(pCxt, yymsp[-1].minor.yy456, &yymsp[0].minor.yy29); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; - break; - case 80: /* alter_db_option ::= BLOCKS NK_INTEGER */ -{ yymsp[-1].minor.yy29.type = DB_OPTION_BLOCKS; yymsp[-1].minor.yy29.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - break; - case 81: /* alter_db_option ::= FSYNC NK_INTEGER */ -{ yymsp[-1].minor.yy29.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy29.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - break; - case 82: /* alter_db_option ::= KEEP integer_list */ - case 83: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==83); -{ yymsp[-1].minor.yy29.type = DB_OPTION_KEEP; yymsp[-1].minor.yy29.pList = yymsp[0].minor.yy632; } - break; - case 84: /* alter_db_option ::= WAL NK_INTEGER */ -{ yymsp[-1].minor.yy29.type = DB_OPTION_WAL; yymsp[-1].minor.yy29.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - break; - case 85: /* alter_db_option ::= QUORUM NK_INTEGER */ -{ yymsp[-1].minor.yy29.type = DB_OPTION_QUORUM; yymsp[-1].minor.yy29.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - break; - case 86: /* alter_db_option ::= CACHELAST NK_INTEGER */ -{ yymsp[-1].minor.yy29.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy29.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - break; - case 87: /* alter_db_option ::= REPLICA NK_INTEGER */ -{ yymsp[-1].minor.yy29.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy29.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - break; - case 88: /* integer_list ::= NK_INTEGER */ -{ yylhsminor.yy632 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy632 = yylhsminor.yy632; - break; - case 89: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ - case 248: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==248); -{ yylhsminor.yy632 = addNodeToList(pCxt, yymsp[-2].minor.yy632, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy632 = yylhsminor.yy632; - break; - case 90: /* variable_list ::= NK_VARIABLE */ -{ yylhsminor.yy632 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy632 = yylhsminor.yy632; - break; - case 91: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ -{ yylhsminor.yy632 = addNodeToList(pCxt, yymsp[-2].minor.yy632, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy632 = yylhsminor.yy632; - break; - case 92: /* retention_list ::= retention */ - case 112: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==112); - case 115: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==115); - case 122: /* column_def_list ::= column_def */ yytestcase(yyruleno==122); - case 167: /* col_name_list ::= col_name */ yytestcase(yyruleno==167); - case 203: /* func_name_list ::= func_name */ yytestcase(yyruleno==203); - case 212: /* func_list ::= func */ yytestcase(yyruleno==212); - case 272: /* literal_list ::= signed_literal */ yytestcase(yyruleno==272); - case 322: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==322); - case 377: /* select_sublist ::= select_item */ yytestcase(yyruleno==377); - case 424: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==424); -{ yylhsminor.yy632 = createNodeList(pCxt, yymsp[0].minor.yy456); } - yymsp[0].minor.yy632 = yylhsminor.yy632; - break; - case 93: /* retention_list ::= retention_list NK_COMMA retention */ - case 123: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==123); - case 168: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==168); - case 204: /* func_name_list ::= func_name_list NK_COMMA col_name */ yytestcase(yyruleno==204); - case 213: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==213); - case 273: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==273); - case 323: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==323); - case 378: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==378); - case 425: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==425); -{ yylhsminor.yy632 = addNodeToList(pCxt, yymsp[-2].minor.yy632, yymsp[0].minor.yy456); } - yymsp[-2].minor.yy632 = yylhsminor.yy632; - break; - case 94: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -{ yylhsminor.yy456 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; - break; - case 95: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - case 97: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==97); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy649, yymsp[-5].minor.yy456, yymsp[-3].minor.yy632, yymsp[-1].minor.yy632, yymsp[0].minor.yy456); } - break; - case 96: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy632); } - break; - case 98: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy632); } - break; - case 99: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy649, yymsp[0].minor.yy456); } - break; - case 100: /* cmd ::= ALTER TABLE alter_table_clause */ - case 101: /* cmd ::= ALTER STABLE alter_table_clause */ yytestcase(yyruleno==101); - case 250: /* cmd ::= query_expression */ yytestcase(yyruleno==250); -{ pCxt->pRootNode = yymsp[0].minor.yy456; } - break; - case 102: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy456 = createAlterTableOption(pCxt, yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; - break; - case 103: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy456 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy537, yymsp[0].minor.yy388); } - yymsp[-4].minor.yy456 = yylhsminor.yy456; - break; - case 104: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy456 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy456, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy537); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; - break; - case 105: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy456 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy537, yymsp[0].minor.yy388); } - yymsp[-4].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pRetentions = yymsp[0].minor.yy376; yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; + break; + case 78: /* db_options ::= db_options STRICT NK_INTEGER */ +{ ((SDatabaseOptions*)yymsp[-2].minor.yy392)->pStrict = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; + break; + case 79: /* alter_db_options ::= alter_db_option */ +{ yylhsminor.yy392 = createDatabaseOptions(pCxt); yylhsminor.yy392 = setDatabaseAlterOption(pCxt, yylhsminor.yy392, &yymsp[0].minor.yy221); } + yymsp[0].minor.yy392 = yylhsminor.yy392; + break; + case 80: /* alter_db_options ::= alter_db_options alter_db_option */ +{ yylhsminor.yy392 = setDatabaseAlterOption(pCxt, yymsp[-1].minor.yy392, &yymsp[0].minor.yy221); } + yymsp[-1].minor.yy392 = yylhsminor.yy392; + break; + case 81: /* alter_db_option ::= BLOCKS NK_INTEGER */ +{ yymsp[-1].minor.yy221.type = DB_OPTION_BLOCKS; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + break; + case 82: /* alter_db_option ::= FSYNC NK_INTEGER */ +{ yymsp[-1].minor.yy221.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + break; + case 83: /* alter_db_option ::= KEEP integer_list */ + case 84: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==84); +{ yymsp[-1].minor.yy221.type = DB_OPTION_KEEP; yymsp[-1].minor.yy221.pList = yymsp[0].minor.yy376; } + break; + case 85: /* alter_db_option ::= WAL NK_INTEGER */ +{ yymsp[-1].minor.yy221.type = DB_OPTION_WAL; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + break; + case 86: /* alter_db_option ::= QUORUM NK_INTEGER */ +{ yymsp[-1].minor.yy221.type = DB_OPTION_QUORUM; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + break; + case 87: /* alter_db_option ::= CACHELAST NK_INTEGER */ +{ yymsp[-1].minor.yy221.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + break; + case 88: /* alter_db_option ::= REPLICA NK_INTEGER */ +{ yymsp[-1].minor.yy221.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + break; + case 89: /* alter_db_option ::= STRICT NK_INTEGER */ +{ yymsp[-1].minor.yy221.type = DB_OPTION_STRICT; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + break; + case 90: /* integer_list ::= NK_INTEGER */ +{ yylhsminor.yy376 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy376 = yylhsminor.yy376; + break; + case 91: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ + case 251: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==251); +{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy376 = yylhsminor.yy376; + break; + case 92: /* variable_list ::= NK_VARIABLE */ +{ yylhsminor.yy376 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy376 = yylhsminor.yy376; + break; + case 93: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ +{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy376 = yylhsminor.yy376; + break; + case 94: /* retention_list ::= retention */ + case 114: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==114); + case 117: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==117); + case 124: /* column_def_list ::= column_def */ yytestcase(yyruleno==124); + case 169: /* col_name_list ::= col_name */ yytestcase(yyruleno==169); + case 206: /* func_name_list ::= func_name */ yytestcase(yyruleno==206); + case 215: /* func_list ::= func */ yytestcase(yyruleno==215); + case 275: /* literal_list ::= signed_literal */ yytestcase(yyruleno==275); + case 325: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==325); + case 380: /* select_sublist ::= select_item */ yytestcase(yyruleno==380); + case 428: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==428); +{ yylhsminor.yy376 = createNodeList(pCxt, yymsp[0].minor.yy392); } + yymsp[0].minor.yy376 = yylhsminor.yy376; + break; + case 95: /* retention_list ::= retention_list NK_COMMA retention */ + case 125: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==125); + case 170: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==170); + case 207: /* func_name_list ::= func_name_list NK_COMMA func_name */ yytestcase(yyruleno==207); + case 216: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==216); + case 276: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==276); + case 326: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==326); + case 381: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==381); + case 429: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==429); +{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, yymsp[0].minor.yy392); } + yymsp[-2].minor.yy376 = yylhsminor.yy376; + break; + case 96: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ +{ yylhsminor.yy392 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; + break; + case 97: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + case 99: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==99); +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy89, yymsp[-5].minor.yy392, yymsp[-3].minor.yy376, yymsp[-1].minor.yy376, yymsp[0].minor.yy392); } + break; + case 98: /* cmd ::= CREATE TABLE multi_create_clause */ +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy376); } + break; + case 100: /* cmd ::= DROP TABLE multi_drop_clause */ +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy376); } + break; + case 101: /* cmd ::= DROP STABLE exists_opt full_table_name */ +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy89, yymsp[0].minor.yy392); } + break; + case 102: /* cmd ::= ALTER TABLE alter_table_clause */ + case 103: /* cmd ::= ALTER STABLE alter_table_clause */ yytestcase(yyruleno==103); + case 253: /* cmd ::= query_expression */ yytestcase(yyruleno==253); +{ pCxt->pRootNode = yymsp[0].minor.yy392; } + break; + case 104: /* alter_table_clause ::= full_table_name alter_table_options */ +{ yylhsminor.yy392 = createAlterTableOption(pCxt, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } + yymsp[-1].minor.yy392 = yylhsminor.yy392; + break; + case 105: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ +{ yylhsminor.yy392 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy449, yymsp[0].minor.yy112); } + yymsp[-4].minor.yy392 = yylhsminor.yy392; + break; + case 106: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ +{ yylhsminor.yy392 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy392, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy449); } + yymsp[-3].minor.yy392 = yylhsminor.yy392; + break; + case 107: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ +{ yylhsminor.yy392 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy449, yymsp[0].minor.yy112); } + yymsp[-4].minor.yy392 = yylhsminor.yy392; + break; + case 108: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ +{ yylhsminor.yy392 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy449, &yymsp[0].minor.yy449); } + yymsp[-4].minor.yy392 = yylhsminor.yy392; break; - case 106: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy456 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy537, &yymsp[0].minor.yy537); } - yymsp[-4].minor.yy456 = yylhsminor.yy456; - break; - case 107: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy456 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy537, yymsp[0].minor.yy388); } - yymsp[-4].minor.yy456 = yylhsminor.yy456; - break; - case 108: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy456 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy456, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy537); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; + case 109: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ +{ yylhsminor.yy392 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy449, yymsp[0].minor.yy112); } + yymsp[-4].minor.yy392 = yylhsminor.yy392; + break; + case 110: /* alter_table_clause ::= full_table_name DROP TAG column_name */ +{ yylhsminor.yy392 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy392, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy449); } + yymsp[-3].minor.yy392 = yylhsminor.yy392; break; - case 109: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy456 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy537, yymsp[0].minor.yy388); } - yymsp[-4].minor.yy456 = yylhsminor.yy456; + case 111: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ +{ yylhsminor.yy392 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy449, yymsp[0].minor.yy112); } + yymsp[-4].minor.yy392 = yylhsminor.yy392; break; - case 110: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy456 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy537, &yymsp[0].minor.yy537); } - yymsp[-4].minor.yy456 = yylhsminor.yy456; - break; - case 111: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ -{ yylhsminor.yy456 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy456, &yymsp[-2].minor.yy537, yymsp[0].minor.yy456); } - yymsp[-5].minor.yy456 = yylhsminor.yy456; + case 112: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ +{ yylhsminor.yy392 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy449, &yymsp[0].minor.yy449); } + yymsp[-4].minor.yy392 = yylhsminor.yy392; + break; + case 113: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ +{ yylhsminor.yy392 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy392, &yymsp[-2].minor.yy449, yymsp[0].minor.yy392); } + yymsp[-5].minor.yy392 = yylhsminor.yy392; break; - case 113: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ - case 116: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==116); -{ yylhsminor.yy632 = addNodeToList(pCxt, yymsp[-1].minor.yy632, yymsp[0].minor.yy456); } - yymsp[-1].minor.yy632 = yylhsminor.yy632; + case 115: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ + case 118: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==118); +{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-1].minor.yy376, yymsp[0].minor.yy392); } + yymsp[-1].minor.yy376 = yylhsminor.yy376; break; - case 114: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ -{ yylhsminor.yy456 = createCreateSubTableClause(pCxt, yymsp[-8].minor.yy649, yymsp[-7].minor.yy456, yymsp[-5].minor.yy456, yymsp[-4].minor.yy632, yymsp[-1].minor.yy632); } - yymsp[-8].minor.yy456 = yylhsminor.yy456; + case 116: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ +{ yylhsminor.yy392 = createCreateSubTableClause(pCxt, yymsp[-8].minor.yy89, yymsp[-7].minor.yy392, yymsp[-5].minor.yy392, yymsp[-4].minor.yy376, yymsp[-1].minor.yy376); } + yymsp[-8].minor.yy392 = yylhsminor.yy392; break; - case 117: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy456 = createDropTableClause(pCxt, yymsp[-1].minor.yy649, yymsp[0].minor.yy456); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + case 119: /* drop_table_clause ::= exists_opt full_table_name */ +{ yylhsminor.yy392 = createDropTableClause(pCxt, yymsp[-1].minor.yy89, yymsp[0].minor.yy392); } + yymsp[-1].minor.yy392 = yylhsminor.yy392; break; - case 118: /* specific_tags_opt ::= */ - case 149: /* tags_def_opt ::= */ yytestcase(yyruleno==149); - case 385: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==385); - case 402: /* group_by_clause_opt ::= */ yytestcase(yyruleno==402); - case 412: /* order_by_clause_opt ::= */ yytestcase(yyruleno==412); -{ yymsp[1].minor.yy632 = NULL; } + case 120: /* specific_tags_opt ::= */ + case 151: /* tags_def_opt ::= */ yytestcase(yyruleno==151); + case 388: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==388); + case 405: /* group_by_clause_opt ::= */ yytestcase(yyruleno==405); + case 416: /* order_by_clause_opt ::= */ yytestcase(yyruleno==416); +{ yymsp[1].minor.yy376 = NULL; } break; - case 119: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ -{ yymsp[-2].minor.yy632 = yymsp[-1].minor.yy632; } + case 121: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ +{ yymsp[-2].minor.yy376 = yymsp[-1].minor.yy376; } break; - case 120: /* full_table_name ::= table_name */ -{ yylhsminor.yy456 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy537, NULL); } - yymsp[0].minor.yy456 = yylhsminor.yy456; + case 122: /* full_table_name ::= table_name */ +{ yylhsminor.yy392 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy449, NULL); } + yymsp[0].minor.yy392 = yylhsminor.yy392; break; - case 121: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy456 = createRealTableNode(pCxt, &yymsp[-2].minor.yy537, &yymsp[0].minor.yy537, NULL); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 123: /* full_table_name ::= db_name NK_DOT table_name */ +{ yylhsminor.yy392 = createRealTableNode(pCxt, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy449, NULL); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 124: /* column_def ::= column_name type_name */ -{ yylhsminor.yy456 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy537, yymsp[0].minor.yy388, NULL); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + case 126: /* column_def ::= column_name type_name */ +{ yylhsminor.yy392 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy449, yymsp[0].minor.yy112, NULL); } + yymsp[-1].minor.yy392 = yylhsminor.yy392; break; - case 125: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy456 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy537, yymsp[-2].minor.yy388, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; + case 127: /* column_def ::= column_name type_name COMMENT NK_STRING */ +{ yylhsminor.yy392 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy449, yymsp[-2].minor.yy112, &yymsp[0].minor.yy0); } + yymsp[-3].minor.yy392 = yylhsminor.yy392; break; - case 126: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy388 = createDataType(TSDB_DATA_TYPE_BOOL); } + case 128: /* type_name ::= BOOL */ +{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_BOOL); } break; - case 127: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy388 = createDataType(TSDB_DATA_TYPE_TINYINT); } + case 129: /* type_name ::= TINYINT */ +{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; - case 128: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy388 = createDataType(TSDB_DATA_TYPE_SMALLINT); } + case 130: /* type_name ::= SMALLINT */ +{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; - case 129: /* type_name ::= INT */ - case 130: /* type_name ::= INTEGER */ yytestcase(yyruleno==130); -{ yymsp[0].minor.yy388 = createDataType(TSDB_DATA_TYPE_INT); } + case 131: /* type_name ::= INT */ + case 132: /* type_name ::= INTEGER */ yytestcase(yyruleno==132); +{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_INT); } break; - case 131: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy388 = createDataType(TSDB_DATA_TYPE_BIGINT); } + case 133: /* type_name ::= BIGINT */ +{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; - case 132: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy388 = createDataType(TSDB_DATA_TYPE_FLOAT); } + case 134: /* type_name ::= FLOAT */ +{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; - case 133: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy388 = createDataType(TSDB_DATA_TYPE_DOUBLE); } + case 135: /* type_name ::= DOUBLE */ +{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; - case 134: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy388 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } + case 136: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy112 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; - case 135: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy388 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } + case 137: /* type_name ::= TIMESTAMP */ +{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; - case 136: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy388 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } + case 138: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy112 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; - case 137: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy388 = createDataType(TSDB_DATA_TYPE_UTINYINT); } + case 139: /* type_name ::= TINYINT UNSIGNED */ +{ yymsp[-1].minor.yy112 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; - case 138: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy388 = createDataType(TSDB_DATA_TYPE_USMALLINT); } + case 140: /* type_name ::= SMALLINT UNSIGNED */ +{ yymsp[-1].minor.yy112 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; - case 139: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy388 = createDataType(TSDB_DATA_TYPE_UINT); } + case 141: /* type_name ::= INT UNSIGNED */ +{ yymsp[-1].minor.yy112 = createDataType(TSDB_DATA_TYPE_UINT); } break; - case 140: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy388 = createDataType(TSDB_DATA_TYPE_UBIGINT); } + case 142: /* type_name ::= BIGINT UNSIGNED */ +{ yymsp[-1].minor.yy112 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; - case 141: /* type_name ::= JSON */ -{ yymsp[0].minor.yy388 = createDataType(TSDB_DATA_TYPE_JSON); } + case 143: /* type_name ::= JSON */ +{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_JSON); } break; - case 142: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy388 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } + case 144: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy112 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; - case 143: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy388 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } + case 145: /* type_name ::= MEDIUMBLOB */ +{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; - case 144: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy388 = createDataType(TSDB_DATA_TYPE_BLOB); } + case 146: /* type_name ::= BLOB */ +{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_BLOB); } break; - case 145: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy388 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } + case 147: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy112 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; - case 146: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy388 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 148: /* type_name ::= DECIMAL */ +{ yymsp[0].minor.yy112 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 147: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy388 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 149: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy112 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 148: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy388 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 150: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ +{ yymsp[-5].minor.yy112 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 150: /* tags_def_opt ::= tags_def */ - case 321: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==321); - case 376: /* select_list ::= select_sublist */ yytestcase(yyruleno==376); -{ yylhsminor.yy632 = yymsp[0].minor.yy632; } - yymsp[0].minor.yy632 = yylhsminor.yy632; + case 152: /* tags_def_opt ::= tags_def */ + case 324: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==324); + case 379: /* select_list ::= select_sublist */ yytestcase(yyruleno==379); +{ yylhsminor.yy376 = yymsp[0].minor.yy376; } + yymsp[0].minor.yy376 = yylhsminor.yy376; break; - case 151: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ -{ yymsp[-3].minor.yy632 = yymsp[-1].minor.yy632; } + case 153: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ +{ yymsp[-3].minor.yy376 = yymsp[-1].minor.yy376; } break; - case 152: /* table_options ::= */ -{ yymsp[1].minor.yy456 = createTableOptions(pCxt); } + case 154: /* table_options ::= */ +{ yymsp[1].minor.yy392 = createTableOptions(pCxt); } break; - case 153: /* table_options ::= table_options COMMENT NK_STRING */ -{ ((STableOptions*)yymsp[-2].minor.yy456)->pComments = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 155: /* table_options ::= table_options COMMENT NK_STRING */ +{ ((STableOptions*)yymsp[-2].minor.yy392)->pComments = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 154: /* table_options ::= table_options KEEP integer_list */ - case 155: /* table_options ::= table_options KEEP variable_list */ yytestcase(yyruleno==155); -{ ((STableOptions*)yymsp[-2].minor.yy456)->pKeep = yymsp[0].minor.yy632; yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 156: /* table_options ::= table_options KEEP integer_list */ + case 157: /* table_options ::= table_options KEEP variable_list */ yytestcase(yyruleno==157); +{ ((STableOptions*)yymsp[-2].minor.yy392)->pKeep = yymsp[0].minor.yy376; yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 156: /* table_options ::= table_options TTL NK_INTEGER */ -{ ((STableOptions*)yymsp[-2].minor.yy456)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 158: /* table_options ::= table_options TTL NK_INTEGER */ +{ ((STableOptions*)yymsp[-2].minor.yy392)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 157: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ ((STableOptions*)yymsp[-4].minor.yy456)->pSma = yymsp[-1].minor.yy632; yylhsminor.yy456 = yymsp[-4].minor.yy456; } - yymsp[-4].minor.yy456 = yylhsminor.yy456; + case 159: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ +{ ((STableOptions*)yymsp[-4].minor.yy392)->pSma = yymsp[-1].minor.yy376; yylhsminor.yy392 = yymsp[-4].minor.yy392; } + yymsp[-4].minor.yy392 = yylhsminor.yy392; break; - case 158: /* table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ -{ ((STableOptions*)yymsp[-4].minor.yy456)->pFuncs = yymsp[-1].minor.yy632; yylhsminor.yy456 = yymsp[-4].minor.yy456; } - yymsp[-4].minor.yy456 = yylhsminor.yy456; + case 160: /* table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ +{ ((STableOptions*)yymsp[-4].minor.yy392)->pFuncs = yymsp[-1].minor.yy376; yylhsminor.yy392 = yymsp[-4].minor.yy392; } + yymsp[-4].minor.yy392 = yylhsminor.yy392; break; - case 159: /* table_options ::= table_options FILE_FACTOR NK_FLOAT */ -{ ((STableOptions*)yymsp[-2].minor.yy456)->pFilesFactor = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 161: /* table_options ::= table_options FILE_FACTOR NK_FLOAT */ +{ ((STableOptions*)yymsp[-2].minor.yy392)->pFilesFactor = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 160: /* table_options ::= table_options DELAY NK_INTEGER */ -{ ((STableOptions*)yymsp[-2].minor.yy456)->pDelay = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 162: /* table_options ::= table_options DELAY NK_INTEGER */ +{ ((STableOptions*)yymsp[-2].minor.yy392)->pDelay = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 161: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy456 = createTableOptions(pCxt); yylhsminor.yy456 = setTableAlterOption(pCxt, yylhsminor.yy456, &yymsp[0].minor.yy29); } - yymsp[0].minor.yy456 = yylhsminor.yy456; + case 163: /* alter_table_options ::= alter_table_option */ +{ yylhsminor.yy392 = createTableOptions(pCxt); yylhsminor.yy392 = setTableAlterOption(pCxt, yylhsminor.yy392, &yymsp[0].minor.yy221); } + yymsp[0].minor.yy392 = yylhsminor.yy392; break; - case 162: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy456 = setTableAlterOption(pCxt, yymsp[-1].minor.yy456, &yymsp[0].minor.yy29); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + case 164: /* alter_table_options ::= alter_table_options alter_table_option */ +{ yylhsminor.yy392 = setTableAlterOption(pCxt, yymsp[-1].minor.yy392, &yymsp[0].minor.yy221); } + yymsp[-1].minor.yy392 = yylhsminor.yy392; break; - case 163: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy29.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy29.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + case 165: /* alter_table_option ::= COMMENT NK_STRING */ +{ yymsp[-1].minor.yy221.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; - case 164: /* alter_table_option ::= KEEP integer_list */ - case 165: /* alter_table_option ::= KEEP variable_list */ yytestcase(yyruleno==165); -{ yymsp[-1].minor.yy29.type = TABLE_OPTION_KEEP; yymsp[-1].minor.yy29.pList = yymsp[0].minor.yy632; } + case 166: /* alter_table_option ::= KEEP integer_list */ + case 167: /* alter_table_option ::= KEEP variable_list */ yytestcase(yyruleno==167); +{ yymsp[-1].minor.yy221.type = TABLE_OPTION_KEEP; yymsp[-1].minor.yy221.pList = yymsp[0].minor.yy376; } break; - case 166: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy29.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy29.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + case 168: /* alter_table_option ::= TTL NK_INTEGER */ +{ yymsp[-1].minor.yy221.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy221.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; - case 169: /* col_name ::= column_name */ -{ yylhsminor.yy456 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy537); } - yymsp[0].minor.yy456 = yylhsminor.yy456; + case 171: /* col_name ::= column_name */ +{ yylhsminor.yy392 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy449); } + yymsp[0].minor.yy392 = yylhsminor.yy392; break; - case 170: /* cmd ::= SHOW DNODES */ + case 172: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL, NULL); } break; - case 171: /* cmd ::= SHOW USERS */ + case 173: /* cmd ::= SHOW USERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL, NULL); } break; - case 172: /* cmd ::= SHOW DATABASES */ + case 174: /* cmd ::= SHOW DATABASES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL, NULL); } break; - case 173: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy456, yymsp[0].minor.yy456); } + case 175: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy392, yymsp[0].minor.yy392); } break; - case 174: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy456, yymsp[0].minor.yy456); } + case 176: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy392, yymsp[0].minor.yy392); } break; - case 175: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy456, NULL); } + case 177: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy392, NULL); } break; - case 176: /* cmd ::= SHOW MNODES */ + case 178: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT, NULL, NULL); } break; - case 177: /* cmd ::= SHOW MODULES */ + case 179: /* cmd ::= SHOW MODULES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MODULES_STMT, NULL, NULL); } break; - case 178: /* cmd ::= SHOW QNODES */ + case 180: /* cmd ::= SHOW QNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT, NULL, NULL); } break; - case 179: /* cmd ::= SHOW FUNCTIONS */ + case 181: /* cmd ::= SHOW FUNCTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT, NULL, NULL); } break; - case 180: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } + case 182: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } break; - case 181: /* cmd ::= SHOW STREAMS */ + case 183: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT, NULL, NULL); } break; - case 182: /* cmd ::= SHOW ACCOUNTS */ + case 184: /* cmd ::= SHOW ACCOUNTS */ { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } break; - case 183: /* cmd ::= SHOW APPS */ + case 185: /* cmd ::= SHOW APPS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT, NULL, NULL); } break; - case 184: /* cmd ::= SHOW CONNECTIONS */ + case 186: /* cmd ::= SHOW CONNECTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT, NULL, NULL); } break; - case 185: /* cmd ::= SHOW LICENCE */ - case 186: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==186); + case 187: /* cmd ::= SHOW LICENCE */ + case 188: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==188); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCE_STMT, NULL, NULL); } break; - case 187: /* cmd ::= SHOW CREATE DATABASE db_name */ -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy537); } + case 189: /* cmd ::= SHOW CREATE DATABASE db_name */ +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy449); } break; - case 188: /* cmd ::= SHOW CREATE TABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy456); } + case 190: /* cmd ::= SHOW CREATE TABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy392); } break; - case 189: /* cmd ::= SHOW CREATE STABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy456); } + case 191: /* cmd ::= SHOW CREATE STABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy392); } break; - case 190: /* cmd ::= SHOW QUERIES */ + case 192: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT, NULL, NULL); } break; - case 191: /* cmd ::= SHOW SCORES */ + case 193: /* cmd ::= SHOW SCORES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT, NULL, NULL); } break; - case 192: /* cmd ::= SHOW TOPICS */ + case 194: /* cmd ::= SHOW TOPICS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT, NULL, NULL); } break; - case 193: /* cmd ::= SHOW VARIABLES */ + case 195: /* cmd ::= SHOW VARIABLES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLE_STMT, NULL, NULL); } break; - case 194: /* cmd ::= SHOW BNODES */ + case 196: /* cmd ::= SHOW BNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT, NULL, NULL); } break; - case 195: /* cmd ::= SHOW SNODES */ + case 197: /* cmd ::= SHOW SNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT, NULL, NULL); } break; - case 196: /* db_name_cond_opt ::= */ - case 201: /* from_db_opt ::= */ yytestcase(yyruleno==201); -{ yymsp[1].minor.yy456 = createDefaultDatabaseCondValue(pCxt); } + case 198: /* cmd ::= SHOW CLUSTER */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT, NULL, NULL); } break; - case 197: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy537); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + case 199: /* db_name_cond_opt ::= */ + case 204: /* from_db_opt ::= */ yytestcase(yyruleno==204); +{ yymsp[1].minor.yy392 = createDefaultDatabaseCondValue(pCxt); } break; - case 198: /* like_pattern_opt ::= */ - case 209: /* index_options ::= */ yytestcase(yyruleno==209); - case 236: /* into_opt ::= */ yytestcase(yyruleno==236); - case 383: /* where_clause_opt ::= */ yytestcase(yyruleno==383); - case 387: /* twindow_clause_opt ::= */ yytestcase(yyruleno==387); - case 392: /* sliding_opt ::= */ yytestcase(yyruleno==392); - case 394: /* fill_opt ::= */ yytestcase(yyruleno==394); - case 406: /* having_clause_opt ::= */ yytestcase(yyruleno==406); - case 414: /* slimit_clause_opt ::= */ yytestcase(yyruleno==414); - case 418: /* limit_clause_opt ::= */ yytestcase(yyruleno==418); -{ yymsp[1].minor.yy456 = NULL; } + case 200: /* db_name_cond_opt ::= db_name NK_DOT */ +{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy449); } + yymsp[-1].minor.yy392 = yylhsminor.yy392; break; - case 199: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + case 201: /* like_pattern_opt ::= */ + case 212: /* index_options ::= */ yytestcase(yyruleno==212); + case 239: /* into_opt ::= */ yytestcase(yyruleno==239); + case 386: /* where_clause_opt ::= */ yytestcase(yyruleno==386); + case 390: /* twindow_clause_opt ::= */ yytestcase(yyruleno==390); + case 395: /* sliding_opt ::= */ yytestcase(yyruleno==395); + case 397: /* fill_opt ::= */ yytestcase(yyruleno==397); + case 409: /* having_clause_opt ::= */ yytestcase(yyruleno==409); + case 418: /* slimit_clause_opt ::= */ yytestcase(yyruleno==418); + case 422: /* limit_clause_opt ::= */ yytestcase(yyruleno==422); +{ yymsp[1].minor.yy392 = NULL; } break; - case 200: /* table_name_cond ::= table_name */ -{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy537); } - yymsp[0].minor.yy456 = yylhsminor.yy456; + case 202: /* like_pattern_opt ::= LIKE NK_STRING */ +{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; - case 202: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy537); } + case 203: /* table_name_cond ::= table_name */ +{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy449); } + yymsp[0].minor.yy392 = yylhsminor.yy392; break; - case 205: /* func_name ::= function_name */ -{ yylhsminor.yy456 = createFunctionNode(pCxt, &yymsp[0].minor.yy537, NULL); } - yymsp[0].minor.yy456 = yylhsminor.yy456; + case 205: /* from_db_opt ::= FROM db_name */ +{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy449); } break; - case 206: /* cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy649, &yymsp[-3].minor.yy537, &yymsp[-1].minor.yy537, NULL, yymsp[0].minor.yy456); } + case 208: /* func_name ::= function_name */ +{ yylhsminor.yy392 = createFunctionNode(pCxt, &yymsp[0].minor.yy449, NULL); } + yymsp[0].minor.yy392 = yylhsminor.yy392; break; - case 207: /* cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, yymsp[-6].minor.yy649, &yymsp[-5].minor.yy537, &yymsp[-3].minor.yy537, yymsp[-1].minor.yy632, NULL); } + case 209: /* cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy89, &yymsp[-3].minor.yy449, &yymsp[-1].minor.yy449, NULL, yymsp[0].minor.yy392); } break; - case 208: /* cmd ::= DROP INDEX exists_opt index_name ON table_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-3].minor.yy649, &yymsp[-2].minor.yy537, &yymsp[0].minor.yy537); } + case 210: /* cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, yymsp[-6].minor.yy89, &yymsp[-5].minor.yy449, &yymsp[-3].minor.yy449, yymsp[-1].minor.yy376, NULL); } break; - case 210: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ -{ yymsp[-8].minor.yy456 = createIndexOption(pCxt, yymsp[-6].minor.yy632, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), NULL, yymsp[0].minor.yy456); } + case 211: /* cmd ::= DROP INDEX exists_opt index_name ON table_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-3].minor.yy89, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy449); } break; - case 211: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ -{ yymsp[-10].minor.yy456 = createIndexOption(pCxt, yymsp[-8].minor.yy632, releaseRawExprNode(pCxt, yymsp[-4].minor.yy456), releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), yymsp[0].minor.yy456); } + case 213: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ +{ yymsp[-8].minor.yy392 = createIndexOption(pCxt, yymsp[-6].minor.yy376, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), NULL, yymsp[0].minor.yy392); } break; - case 214: /* func ::= function_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy456 = createFunctionNode(pCxt, &yymsp[-3].minor.yy537, yymsp[-1].minor.yy632); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; + case 214: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ +{ yymsp[-10].minor.yy392 = createIndexOption(pCxt, yymsp[-8].minor.yy376, releaseRawExprNode(pCxt, yymsp[-4].minor.yy392), releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), yymsp[0].minor.yy392); } break; - case 215: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy649, &yymsp[-2].minor.yy537, yymsp[0].minor.yy456, NULL); } + case 217: /* func ::= function_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy392 = createFunctionNode(pCxt, &yymsp[-3].minor.yy449, yymsp[-1].minor.yy376); } + yymsp[-3].minor.yy392 = yylhsminor.yy392; break; - case 216: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy649, &yymsp[-2].minor.yy537, NULL, &yymsp[0].minor.yy537); } + case 218: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy89, &yymsp[-2].minor.yy449, yymsp[0].minor.yy392, NULL); } break; - case 217: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy649, &yymsp[0].minor.yy537); } + case 219: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS db_name */ +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy89, &yymsp[-2].minor.yy449, NULL, &yymsp[0].minor.yy449); } break; - case 218: /* cmd ::= DESC full_table_name */ - case 219: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==219); -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy456); } + case 220: /* cmd ::= DROP TOPIC exists_opt topic_name */ +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy89, &yymsp[0].minor.yy449); } break; - case 220: /* cmd ::= RESET QUERY CACHE */ + case 221: /* cmd ::= DESC full_table_name */ + case 222: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==222); +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy392); } + break; + case 223: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; - case 221: /* cmd ::= EXPLAIN analyze_opt explain_options query_expression */ -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy649, yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } + case 224: /* cmd ::= EXPLAIN analyze_opt explain_options query_expression */ +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy89, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } break; - case 223: /* analyze_opt ::= ANALYZE */ - case 231: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==231); - case 373: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==373); -{ yymsp[0].minor.yy649 = true; } + case 226: /* analyze_opt ::= ANALYZE */ + case 234: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==234); + case 376: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==376); +{ yymsp[0].minor.yy89 = true; } break; - case 224: /* explain_options ::= */ -{ yymsp[1].minor.yy456 = createDefaultExplainOptions(pCxt); } + case 227: /* explain_options ::= */ +{ yymsp[1].minor.yy392 = createDefaultExplainOptions(pCxt); } break; - case 225: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -{ yylhsminor.yy456 = setExplainVerbose(pCxt, yymsp[-2].minor.yy456, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 228: /* explain_options ::= explain_options VERBOSE NK_BOOL */ +{ yylhsminor.yy392 = setExplainVerbose(pCxt, yymsp[-2].minor.yy392, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 226: /* explain_options ::= explain_options RATIO NK_FLOAT */ -{ yylhsminor.yy456 = setExplainRatio(pCxt, yymsp[-2].minor.yy456, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 229: /* explain_options ::= explain_options RATIO NK_FLOAT */ +{ yylhsminor.yy392 = setExplainRatio(pCxt, yymsp[-2].minor.yy392, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 227: /* cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ -{ pCxt->pRootNode = createCompactStmt(pCxt, yymsp[-1].minor.yy632); } + case 230: /* cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ +{ pCxt->pRootNode = createCompactStmt(pCxt, yymsp[-1].minor.yy376); } break; - case 228: /* cmd ::= CREATE agg_func_opt FUNCTION function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-7].minor.yy649, &yymsp[-5].minor.yy537, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy388, yymsp[0].minor.yy652); } + case 231: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy89, yymsp[-8].minor.yy89, &yymsp[-5].minor.yy449, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy112, yymsp[0].minor.yy4); } break; - case 229: /* cmd ::= DROP FUNCTION function_name */ -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, &yymsp[0].minor.yy537); } + case 232: /* cmd ::= DROP FUNCTION function_name */ +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, &yymsp[0].minor.yy449); } break; - case 232: /* bufsize_opt ::= */ -{ yymsp[1].minor.yy652 = 0; } + case 235: /* bufsize_opt ::= */ +{ yymsp[1].minor.yy4 = 0; } break; - case 233: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ -{ yymsp[-1].minor.yy652 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 236: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ +{ yymsp[-1].minor.yy4 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 234: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-5].minor.yy649, &yymsp[-4].minor.yy537, yymsp[-2].minor.yy456, yymsp[-3].minor.yy456, yymsp[0].minor.yy456); } + case 237: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-5].minor.yy89, &yymsp[-4].minor.yy449, yymsp[-2].minor.yy392, yymsp[-3].minor.yy392, yymsp[0].minor.yy392); } break; - case 235: /* cmd ::= DROP STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy649, &yymsp[0].minor.yy537); } + case 238: /* cmd ::= DROP STREAM exists_opt stream_name */ +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy89, &yymsp[0].minor.yy449); } break; - case 237: /* into_opt ::= INTO full_table_name */ - case 354: /* from_clause ::= FROM table_reference_list */ yytestcase(yyruleno==354); - case 384: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==384); - case 407: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==407); -{ yymsp[-1].minor.yy456 = yymsp[0].minor.yy456; } + case 240: /* into_opt ::= INTO full_table_name */ + case 357: /* from_clause ::= FROM table_reference_list */ yytestcase(yyruleno==357); + case 387: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==387); + case 410: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==410); +{ yymsp[-1].minor.yy392 = yymsp[0].minor.yy392; } break; - case 238: /* stream_options ::= */ -{ yymsp[1].minor.yy456 = createStreamOptions(pCxt); } + case 241: /* stream_options ::= */ +{ yymsp[1].minor.yy392 = createStreamOptions(pCxt); } break; - case 239: /* stream_options ::= stream_options TRIGGER AT_ONCE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy456)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 242: /* stream_options ::= stream_options TRIGGER AT_ONCE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy392)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 240: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy456)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 243: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy392)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 241: /* stream_options ::= stream_options WATERMARK duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy456)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy456); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 244: /* stream_options ::= stream_options WATERMARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy392)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy392); yylhsminor.yy392 = yymsp[-2].minor.yy392; } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 242: /* cmd ::= KILL CONNECTION NK_INTEGER */ + case 245: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } break; - case 243: /* cmd ::= KILL QUERY NK_INTEGER */ + case 246: /* cmd ::= KILL QUERY NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_QUERY_STMT, &yymsp[0].minor.yy0); } break; - case 244: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + case 247: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 245: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy632); } + case 248: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy376); } break; - case 246: /* cmd ::= SPLIT VGROUP NK_INTEGER */ + case 249: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 247: /* dnode_list ::= DNODE NK_INTEGER */ -{ yymsp[-1].minor.yy632 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - break; - case 249: /* cmd ::= SYNCDB db_name REPLICA */ -{ pCxt->pRootNode = createSyncdbStmt(pCxt, &yymsp[-1].minor.yy537); } - break; - case 251: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; - break; - case 252: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; - break; - case 253: /* literal ::= NK_STRING */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; - break; - case 254: /* literal ::= NK_BOOL */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; - break; - case 255: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; - break; - case 256: /* literal ::= duration_literal */ - case 266: /* signed_literal ::= signed */ yytestcase(yyruleno==266); - case 284: /* expression ::= literal */ yytestcase(yyruleno==284); - case 285: /* expression ::= pseudo_column */ yytestcase(yyruleno==285); - case 286: /* expression ::= column_reference */ yytestcase(yyruleno==286); - case 287: /* expression ::= function_expression */ yytestcase(yyruleno==287); - case 288: /* expression ::= subquery */ yytestcase(yyruleno==288); - case 346: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==346); - case 350: /* boolean_primary ::= predicate */ yytestcase(yyruleno==350); - case 352: /* common_expression ::= expression */ yytestcase(yyruleno==352); - case 353: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==353); - case 355: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==355); - case 357: /* table_reference ::= table_primary */ yytestcase(yyruleno==357); - case 358: /* table_reference ::= joined_table */ yytestcase(yyruleno==358); - case 362: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==362); - case 409: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==409); - case 411: /* query_primary ::= query_specification */ yytestcase(yyruleno==411); -{ yylhsminor.yy456 = yymsp[0].minor.yy456; } - yymsp[0].minor.yy456 = yylhsminor.yy456; - break; - case 257: /* literal ::= NULL */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; - break; - case 258: /* literal ::= NK_QUESTION */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; - break; - case 259: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; - break; - case 260: /* signed ::= NK_INTEGER */ -{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy456 = yylhsminor.yy456; - break; - case 261: /* signed ::= NK_PLUS NK_INTEGER */ -{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - break; - case 262: /* signed ::= NK_MINUS NK_INTEGER */ + case 250: /* dnode_list ::= DNODE NK_INTEGER */ +{ yymsp[-1].minor.yy376 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + break; + case 252: /* cmd ::= SYNCDB db_name REPLICA */ +{ pCxt->pRootNode = createSyncdbStmt(pCxt, &yymsp[-1].minor.yy449); } + break; + case 254: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy392 = yylhsminor.yy392; + break; + case 255: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy392 = yylhsminor.yy392; + break; + case 256: /* literal ::= NK_STRING */ +{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy392 = yylhsminor.yy392; + break; + case 257: /* literal ::= NK_BOOL */ +{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy392 = yylhsminor.yy392; + break; + case 258: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy392 = yylhsminor.yy392; + break; + case 259: /* literal ::= duration_literal */ + case 269: /* signed_literal ::= signed */ yytestcase(yyruleno==269); + case 287: /* expression ::= literal */ yytestcase(yyruleno==287); + case 288: /* expression ::= pseudo_column */ yytestcase(yyruleno==288); + case 289: /* expression ::= column_reference */ yytestcase(yyruleno==289); + case 290: /* expression ::= function_expression */ yytestcase(yyruleno==290); + case 291: /* expression ::= subquery */ yytestcase(yyruleno==291); + case 349: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==349); + case 353: /* boolean_primary ::= predicate */ yytestcase(yyruleno==353); + case 355: /* common_expression ::= expression */ yytestcase(yyruleno==355); + case 356: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==356); + case 358: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==358); + case 360: /* table_reference ::= table_primary */ yytestcase(yyruleno==360); + case 361: /* table_reference ::= joined_table */ yytestcase(yyruleno==361); + case 365: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==365); + case 412: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==412); + case 415: /* query_primary ::= query_specification */ yytestcase(yyruleno==415); +{ yylhsminor.yy392 = yymsp[0].minor.yy392; } + yymsp[0].minor.yy392 = yylhsminor.yy392; + break; + case 260: /* literal ::= NULL */ +{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy392 = yylhsminor.yy392; + break; + case 261: /* literal ::= NK_QUESTION */ +{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy392 = yylhsminor.yy392; + break; + case 262: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy392 = yylhsminor.yy392; + break; + case 263: /* signed ::= NK_INTEGER */ +{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy392 = yylhsminor.yy392; + break; + case 264: /* signed ::= NK_PLUS NK_INTEGER */ +{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + break; + case 265: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + yymsp[-1].minor.yy392 = yylhsminor.yy392; break; - case 263: /* signed ::= NK_FLOAT */ -{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy456 = yylhsminor.yy456; + case 266: /* signed ::= NK_FLOAT */ +{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy392 = yylhsminor.yy392; break; - case 264: /* signed ::= NK_PLUS NK_FLOAT */ -{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + case 267: /* signed ::= NK_PLUS NK_FLOAT */ +{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; - case 265: /* signed ::= NK_MINUS NK_FLOAT */ + case 268: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + yymsp[-1].minor.yy392 = yylhsminor.yy392; break; - case 267: /* signed_literal ::= NK_STRING */ -{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy456 = yylhsminor.yy456; + case 270: /* signed_literal ::= NK_STRING */ +{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy392 = yylhsminor.yy392; break; - case 268: /* signed_literal ::= NK_BOOL */ -{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy456 = yylhsminor.yy456; + case 271: /* signed_literal ::= NK_BOOL */ +{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy392 = yylhsminor.yy392; break; - case 269: /* signed_literal ::= TIMESTAMP NK_STRING */ -{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + case 272: /* signed_literal ::= TIMESTAMP NK_STRING */ +{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } break; - case 270: /* signed_literal ::= duration_literal */ - case 324: /* star_func_para ::= expression */ yytestcase(yyruleno==324); - case 379: /* select_item ::= common_expression */ yytestcase(yyruleno==379); - case 423: /* search_condition ::= common_expression */ yytestcase(yyruleno==423); -{ yylhsminor.yy456 = releaseRawExprNode(pCxt, yymsp[0].minor.yy456); } - yymsp[0].minor.yy456 = yylhsminor.yy456; + case 273: /* signed_literal ::= duration_literal */ + case 327: /* star_func_para ::= expression */ yytestcase(yyruleno==327); + case 382: /* select_item ::= common_expression */ yytestcase(yyruleno==382); + case 427: /* search_condition ::= common_expression */ yytestcase(yyruleno==427); +{ yylhsminor.yy392 = releaseRawExprNode(pCxt, yymsp[0].minor.yy392); } + yymsp[0].minor.yy392 = yylhsminor.yy392; break; - case 271: /* signed_literal ::= NULL */ -{ yymsp[0].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL); } + case 274: /* signed_literal ::= NULL */ +{ yymsp[0].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL); } break; - case 289: /* expression ::= NK_LP expression NK_RP */ - case 351: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==351); -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy456)); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 292: /* expression ::= NK_LP expression NK_RP */ + case 354: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==354); +{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy392)); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 290: /* expression ::= NK_PLUS expression */ + case 293: /* expression ::= NK_PLUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy456)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy392)); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + yymsp[-1].minor.yy392 = yylhsminor.yy392; break; - case 291: /* expression ::= NK_MINUS expression */ + case 294: /* expression ::= NK_MINUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy456), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy392), NULL)); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + yymsp[-1].minor.yy392 = yylhsminor.yy392; break; - case 292: /* expression ::= expression NK_PLUS expression */ + case 295: /* expression ::= expression NK_PLUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 293: /* expression ::= expression NK_MINUS expression */ + case 296: /* expression ::= expression NK_MINUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 294: /* expression ::= expression NK_STAR expression */ + case 297: /* expression ::= expression NK_STAR expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 295: /* expression ::= expression NK_SLASH expression */ + case 298: /* expression ::= expression NK_SLASH expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 296: /* expression ::= expression NK_REM expression */ + case 299: /* expression ::= expression NK_REM expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 297: /* expression ::= column_reference NK_ARROW NK_STRING */ + case 300: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; - break; - case 298: /* expression_list ::= expression */ -{ yylhsminor.yy632 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy456)); } - yymsp[0].minor.yy632 = yylhsminor.yy632; - break; - case 299: /* expression_list ::= expression_list NK_COMMA expression */ -{ yylhsminor.yy632 = addNodeToList(pCxt, yymsp[-2].minor.yy632, releaseRawExprNode(pCxt, yymsp[0].minor.yy456)); } - yymsp[-2].minor.yy632 = yylhsminor.yy632; - break; - case 300: /* column_reference ::= column_name */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy537, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy537)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; - break; - case 301: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy537, &yymsp[0].minor.yy537, createColumnNode(pCxt, &yymsp[-2].minor.yy537, &yymsp[0].minor.yy537)); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; - break; - case 302: /* pseudo_column ::= ROWTS */ - case 303: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==303); - case 304: /* pseudo_column ::= QSTARTTS */ yytestcase(yyruleno==304); - case 305: /* pseudo_column ::= QENDTS */ yytestcase(yyruleno==305); - case 306: /* pseudo_column ::= WSTARTTS */ yytestcase(yyruleno==306); - case 307: /* pseudo_column ::= WENDTS */ yytestcase(yyruleno==307); - case 308: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==308); -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; - break; - case 309: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 310: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==310); -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy537, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy537, yymsp[-1].minor.yy632)); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; - break; - case 311: /* function_expression ::= CAST NK_LP expression AS type_name NK_RP */ -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), yymsp[-1].minor.yy388)); } - yymsp[-5].minor.yy456 = yylhsminor.yy456; - break; - case 312: /* function_expression ::= noarg_func NK_LP NK_RP */ -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy537, &yymsp[0].minor.yy0, createFunctionNodeNoArg(pCxt, &yymsp[-2].minor.yy537)); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; - break; - case 320: /* star_func_para_list ::= NK_STAR */ -{ yylhsminor.yy632 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy632 = yylhsminor.yy632; - break; - case 325: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 382: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==382); -{ yylhsminor.yy456 = createColumnNode(pCxt, &yymsp[-2].minor.yy537, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; - break; - case 326: /* predicate ::= expression compare_op expression */ - case 331: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==331); + yymsp[-2].minor.yy392 = yylhsminor.yy392; + break; + case 301: /* expression_list ::= expression */ +{ yylhsminor.yy376 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy392)); } + yymsp[0].minor.yy376 = yylhsminor.yy376; + break; + case 302: /* expression_list ::= expression_list NK_COMMA expression */ +{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, releaseRawExprNode(pCxt, yymsp[0].minor.yy392)); } + yymsp[-2].minor.yy376 = yylhsminor.yy376; + break; + case 303: /* column_reference ::= column_name */ +{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy449, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy449)); } + yymsp[0].minor.yy392 = yylhsminor.yy392; + break; + case 304: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy449, createColumnNode(pCxt, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy449)); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; + break; + case 305: /* pseudo_column ::= ROWTS */ + case 306: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==306); + case 307: /* pseudo_column ::= QSTARTTS */ yytestcase(yyruleno==307); + case 308: /* pseudo_column ::= QENDTS */ yytestcase(yyruleno==308); + case 309: /* pseudo_column ::= WSTARTTS */ yytestcase(yyruleno==309); + case 310: /* pseudo_column ::= WENDTS */ yytestcase(yyruleno==310); + case 311: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==311); +{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy392 = yylhsminor.yy392; + break; + case 312: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 313: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==313); +{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy449, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy449, yymsp[-1].minor.yy376)); } + yymsp[-3].minor.yy392 = yylhsminor.yy392; + break; + case 314: /* function_expression ::= CAST NK_LP expression AS type_name NK_RP */ +{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), yymsp[-1].minor.yy112)); } + yymsp[-5].minor.yy392 = yylhsminor.yy392; + break; + case 315: /* function_expression ::= noarg_func NK_LP NK_RP */ +{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy0, createFunctionNodeNoArg(pCxt, &yymsp[-2].minor.yy449)); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; + break; + case 323: /* star_func_para_list ::= NK_STAR */ +{ yylhsminor.yy376 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy376 = yylhsminor.yy376; + break; + case 328: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 385: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==385); +{ yylhsminor.yy392 = createColumnNode(pCxt, &yymsp[-2].minor.yy449, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; + break; + case 329: /* predicate ::= expression compare_op expression */ + case 334: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==334); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy416, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy380, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 327: /* predicate ::= expression BETWEEN expression AND expression */ + case 330: /* predicate ::= expression BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy456), releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy392); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy392), releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } - yymsp[-4].minor.yy456 = yylhsminor.yy456; + yymsp[-4].minor.yy392 = yylhsminor.yy392; break; - case 328: /* predicate ::= expression NOT BETWEEN expression AND expression */ + case 331: /* predicate ::= expression NOT BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy456), releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy392); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy392), releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } - yymsp[-5].minor.yy456 = yylhsminor.yy456; + yymsp[-5].minor.yy392 = yylhsminor.yy392; break; - case 329: /* predicate ::= expression IS NULL */ + case 332: /* predicate ::= expression IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), NULL)); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 330: /* predicate ::= expression IS NOT NULL */ + case 333: /* predicate ::= expression IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), NULL)); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; + yymsp[-3].minor.yy392 = yylhsminor.yy392; break; - case 332: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy416 = OP_TYPE_LOWER_THAN; } + case 335: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy380 = OP_TYPE_LOWER_THAN; } break; - case 333: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy416 = OP_TYPE_GREATER_THAN; } + case 336: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy380 = OP_TYPE_GREATER_THAN; } break; - case 334: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy416 = OP_TYPE_LOWER_EQUAL; } + case 337: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy380 = OP_TYPE_LOWER_EQUAL; } break; - case 335: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy416 = OP_TYPE_GREATER_EQUAL; } + case 338: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy380 = OP_TYPE_GREATER_EQUAL; } break; - case 336: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy416 = OP_TYPE_NOT_EQUAL; } + case 339: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy380 = OP_TYPE_NOT_EQUAL; } break; - case 337: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy416 = OP_TYPE_EQUAL; } + case 340: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy380 = OP_TYPE_EQUAL; } break; - case 338: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy416 = OP_TYPE_LIKE; } + case 341: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy380 = OP_TYPE_LIKE; } break; - case 339: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy416 = OP_TYPE_NOT_LIKE; } + case 342: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy380 = OP_TYPE_NOT_LIKE; } break; - case 340: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy416 = OP_TYPE_MATCH; } + case 343: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy380 = OP_TYPE_MATCH; } break; - case 341: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy416 = OP_TYPE_NMATCH; } + case 344: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy380 = OP_TYPE_NMATCH; } break; - case 342: /* compare_op ::= CONTAINS */ -{ yymsp[0].minor.yy416 = OP_TYPE_JSON_CONTAINS; } + case 345: /* compare_op ::= CONTAINS */ +{ yymsp[0].minor.yy380 = OP_TYPE_JSON_CONTAINS; } break; - case 343: /* in_op ::= IN */ -{ yymsp[0].minor.yy416 = OP_TYPE_IN; } + case 346: /* in_op ::= IN */ +{ yymsp[0].minor.yy380 = OP_TYPE_IN; } break; - case 344: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy416 = OP_TYPE_NOT_IN; } + case 347: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy380 = OP_TYPE_NOT_IN; } break; - case 345: /* in_predicate_value ::= NK_LP expression_list NK_RP */ -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy632)); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 348: /* in_predicate_value ::= NK_LP expression_list NK_RP */ +{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy376)); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 347: /* boolean_value_expression ::= NOT boolean_primary */ + case 350: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy456), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy392), NULL)); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + yymsp[-1].minor.yy392 = yylhsminor.yy392; break; - case 348: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 351: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 349: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 352: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); + yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 356: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy456 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy456, yymsp[0].minor.yy456, NULL); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 359: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy392 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy392, yymsp[0].minor.yy392, NULL); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 359: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy456 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy537, &yymsp[0].minor.yy537); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + case 362: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy392 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy449, &yymsp[0].minor.yy449); } + yymsp[-1].minor.yy392 = yylhsminor.yy392; break; - case 360: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy456 = createRealTableNode(pCxt, &yymsp[-3].minor.yy537, &yymsp[-1].minor.yy537, &yymsp[0].minor.yy537); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; + case 363: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy392 = createRealTableNode(pCxt, &yymsp[-3].minor.yy449, &yymsp[-1].minor.yy449, &yymsp[0].minor.yy449); } + yymsp[-3].minor.yy392 = yylhsminor.yy392; break; - case 361: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy456 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy456), &yymsp[0].minor.yy537); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + case 364: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy392 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy392), &yymsp[0].minor.yy449); } + yymsp[-1].minor.yy392 = yylhsminor.yy392; break; - case 363: /* alias_opt ::= */ -{ yymsp[1].minor.yy537 = nil_token; } + case 366: /* alias_opt ::= */ +{ yymsp[1].minor.yy449 = nil_token; } break; - case 364: /* alias_opt ::= table_alias */ -{ yylhsminor.yy537 = yymsp[0].minor.yy537; } - yymsp[0].minor.yy537 = yylhsminor.yy537; + case 367: /* alias_opt ::= table_alias */ +{ yylhsminor.yy449 = yymsp[0].minor.yy449; } + yymsp[0].minor.yy449 = yylhsminor.yy449; break; - case 365: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy537 = yymsp[0].minor.yy537; } + case 368: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy449 = yymsp[0].minor.yy449; } break; - case 366: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 367: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==367); -{ yymsp[-2].minor.yy456 = yymsp[-1].minor.yy456; } + case 369: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 370: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==370); +{ yymsp[-2].minor.yy392 = yymsp[-1].minor.yy392; } break; - case 368: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy456 = createJoinTableNode(pCxt, yymsp[-4].minor.yy164, yymsp[-5].minor.yy456, yymsp[-2].minor.yy456, yymsp[0].minor.yy456); } - yymsp[-5].minor.yy456 = yylhsminor.yy456; + case 371: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy392 = createJoinTableNode(pCxt, yymsp[-4].minor.yy372, yymsp[-5].minor.yy392, yymsp[-2].minor.yy392, yymsp[0].minor.yy392); } + yymsp[-5].minor.yy392 = yylhsminor.yy392; break; - case 369: /* join_type ::= */ -{ yymsp[1].minor.yy164 = JOIN_TYPE_INNER; } + case 372: /* join_type ::= */ +{ yymsp[1].minor.yy372 = JOIN_TYPE_INNER; } break; - case 370: /* join_type ::= INNER */ -{ yymsp[0].minor.yy164 = JOIN_TYPE_INNER; } + case 373: /* join_type ::= INNER */ +{ yymsp[0].minor.yy372 = JOIN_TYPE_INNER; } break; - case 371: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 374: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-8].minor.yy456 = createSelectStmt(pCxt, yymsp[-7].minor.yy649, yymsp[-6].minor.yy632, yymsp[-5].minor.yy456); - yymsp[-8].minor.yy456 = addWhereClause(pCxt, yymsp[-8].minor.yy456, yymsp[-4].minor.yy456); - yymsp[-8].minor.yy456 = addPartitionByClause(pCxt, yymsp[-8].minor.yy456, yymsp[-3].minor.yy632); - yymsp[-8].minor.yy456 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy456, yymsp[-2].minor.yy456); - yymsp[-8].minor.yy456 = addGroupByClause(pCxt, yymsp[-8].minor.yy456, yymsp[-1].minor.yy632); - yymsp[-8].minor.yy456 = addHavingClause(pCxt, yymsp[-8].minor.yy456, yymsp[0].minor.yy456); + yymsp[-8].minor.yy392 = createSelectStmt(pCxt, yymsp[-7].minor.yy89, yymsp[-6].minor.yy376, yymsp[-5].minor.yy392); + yymsp[-8].minor.yy392 = addWhereClause(pCxt, yymsp[-8].minor.yy392, yymsp[-4].minor.yy392); + yymsp[-8].minor.yy392 = addPartitionByClause(pCxt, yymsp[-8].minor.yy392, yymsp[-3].minor.yy376); + yymsp[-8].minor.yy392 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy392, yymsp[-2].minor.yy392); + yymsp[-8].minor.yy392 = addGroupByClause(pCxt, yymsp[-8].minor.yy392, yymsp[-1].minor.yy376); + yymsp[-8].minor.yy392 = addHavingClause(pCxt, yymsp[-8].minor.yy392, yymsp[0].minor.yy392); } break; - case 374: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy649 = false; } + case 377: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy89 = false; } break; - case 375: /* select_list ::= NK_STAR */ -{ yymsp[0].minor.yy632 = NULL; } + case 378: /* select_list ::= NK_STAR */ +{ yymsp[0].minor.yy376 = NULL; } break; - case 380: /* select_item ::= common_expression column_alias */ -{ yylhsminor.yy456 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy456), &yymsp[0].minor.yy537); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + case 383: /* select_item ::= common_expression column_alias */ +{ yylhsminor.yy392 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy392), &yymsp[0].minor.yy449); } + yymsp[-1].minor.yy392 = yylhsminor.yy392; break; - case 381: /* select_item ::= common_expression AS column_alias */ -{ yylhsminor.yy456 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), &yymsp[0].minor.yy537); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 384: /* select_item ::= common_expression AS column_alias */ +{ yylhsminor.yy392 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), &yymsp[0].minor.yy449); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 386: /* partition_by_clause_opt ::= PARTITION BY expression_list */ - case 403: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==403); - case 413: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==413); -{ yymsp[-2].minor.yy632 = yymsp[0].minor.yy632; } + case 389: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 406: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==406); + case 417: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==417); +{ yymsp[-2].minor.yy376 = yymsp[0].minor.yy376; } break; - case 388: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -{ yymsp[-5].minor.yy456 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), releaseRawExprNode(pCxt, yymsp[-1].minor.yy456)); } + case 391: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy392 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), releaseRawExprNode(pCxt, yymsp[-1].minor.yy392)); } break; - case 389: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ -{ yymsp[-3].minor.yy456 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy456)); } + case 392: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ +{ yymsp[-3].minor.yy392 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy392)); } break; - case 390: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy456 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), NULL, yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } + case 393: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy392 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), NULL, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } break; - case 391: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy456 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy456), releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } + case 394: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy392 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy392), releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } break; - case 393: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ -{ yymsp[-3].minor.yy456 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy456); } + case 396: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ +{ yymsp[-3].minor.yy392 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy392); } break; - case 395: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy456 = createFillNode(pCxt, yymsp[-1].minor.yy646, NULL); } + case 398: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy392 = createFillNode(pCxt, yymsp[-1].minor.yy102, NULL); } break; - case 396: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy456 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy632)); } + case 399: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy392 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy376)); } break; - case 397: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy646 = FILL_MODE_NONE; } + case 400: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy102 = FILL_MODE_NONE; } break; - case 398: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy646 = FILL_MODE_PREV; } + case 401: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy102 = FILL_MODE_PREV; } break; - case 399: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy646 = FILL_MODE_NULL; } + case 402: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy102 = FILL_MODE_NULL; } break; - case 400: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy646 = FILL_MODE_LINEAR; } + case 403: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy102 = FILL_MODE_LINEAR; } break; - case 401: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy646 = FILL_MODE_NEXT; } + case 404: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy102 = FILL_MODE_NEXT; } break; - case 404: /* group_by_list ::= expression */ -{ yylhsminor.yy632 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } - yymsp[0].minor.yy632 = yylhsminor.yy632; + case 407: /* group_by_list ::= expression */ +{ yylhsminor.yy376 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } + yymsp[0].minor.yy376 = yylhsminor.yy376; break; - case 405: /* group_by_list ::= group_by_list NK_COMMA expression */ -{ yylhsminor.yy632 = addNodeToList(pCxt, yymsp[-2].minor.yy632, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } - yymsp[-2].minor.yy632 = yylhsminor.yy632; + case 408: /* group_by_list ::= group_by_list NK_COMMA expression */ +{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } + yymsp[-2].minor.yy376 = yylhsminor.yy376; break; - case 408: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 411: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy456 = addOrderByClause(pCxt, yymsp[-3].minor.yy456, yymsp[-2].minor.yy632); - yylhsminor.yy456 = addSlimitClause(pCxt, yylhsminor.yy456, yymsp[-1].minor.yy456); - yylhsminor.yy456 = addLimitClause(pCxt, yylhsminor.yy456, yymsp[0].minor.yy456); + yylhsminor.yy392 = addOrderByClause(pCxt, yymsp[-3].minor.yy392, yymsp[-2].minor.yy376); + yylhsminor.yy392 = addSlimitClause(pCxt, yylhsminor.yy392, yymsp[-1].minor.yy392); + yylhsminor.yy392 = addLimitClause(pCxt, yylhsminor.yy392, yymsp[0].minor.yy392); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; + yymsp[-3].minor.yy392 = yylhsminor.yy392; + break; + case 413: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ yylhsminor.yy392 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy392, yymsp[0].minor.yy392); } + yymsp[-3].minor.yy392 = yylhsminor.yy392; break; - case 410: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ yylhsminor.yy456 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy456, yymsp[0].minor.yy456); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; + case 414: /* query_expression_body ::= query_expression_body UNION query_expression_body */ +{ yylhsminor.yy392 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy392, yymsp[0].minor.yy392); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 415: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 419: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==419); -{ yymsp[-1].minor.yy456 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 419: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 423: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==423); +{ yymsp[-1].minor.yy392 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 416: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 420: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==420); -{ yymsp[-3].minor.yy456 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 420: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 424: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==424); +{ yymsp[-3].minor.yy392 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 417: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 421: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==421); -{ yymsp[-3].minor.yy456 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 421: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 425: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==425); +{ yymsp[-3].minor.yy392 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 422: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy456); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 426: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy392); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 426: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy456 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), yymsp[-1].minor.yy626, yymsp[0].minor.yy209); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 430: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy392 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), yymsp[-1].minor.yy386, yymsp[0].minor.yy361); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 427: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy626 = ORDER_ASC; } + case 431: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy386 = ORDER_ASC; } break; - case 428: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy626 = ORDER_ASC; } + case 432: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy386 = ORDER_ASC; } break; - case 429: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy626 = ORDER_DESC; } + case 433: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy386 = ORDER_DESC; } break; - case 430: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy209 = NULL_ORDER_DEFAULT; } + case 434: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy361 = NULL_ORDER_DEFAULT; } break; - case 431: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy209 = NULL_ORDER_FIRST; } + case 435: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy361 = NULL_ORDER_FIRST; } break; - case 432: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy209 = NULL_ORDER_LAST; } + case 436: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy361 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index 869232002aa7bf2309794d94703097b4b1a1215b..96de3fb7356a6cf3acb4df828e3284c472f4ef07 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -17,3 +17,11 @@ TARGET_LINK_LIBRARIES( parserTest PUBLIC os util common nodes parser catalog transport gtest function planner qcom ) + +if(${BUILD_WINGETOPT}) + target_include_directories( + parserTest + PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" + ) + target_link_libraries(parserTest PUBLIC wingetopt) +endif() \ No newline at end of file diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 5723b93f8b50e89f9228697e0b9101db02a749b7..0f3e2fe414e18ef213f6b1b9257eb053a4c573a6 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -12,7 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - #include #include "stub.h" @@ -23,6 +22,9 @@ #pragma GCC diagnostic pop +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "mockCatalog.h" namespace { diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 3ef0eaed4289cf2b3595615b01af1a409ce43480..d36448b6869131b72f5d9c56be9bfe6e87d66532 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -40,13 +40,11 @@ public: virtual TableBuilder& setVgid(int16_t vgid) { schema()->vgId = vgid; - SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0, }; - - vgroup.epSet.eps[0] = (SEp){"dnode_1", 6030}; - vgroup.epSet.eps[1] = (SEp){"dnode_2", 6030}; - vgroup.epSet.eps[2] = (SEp){"dnode_3", 6030}; + SVgroupInfo vgroup = { vgid, 0, 0, {0}, 0}; + addEpIntoEpSet(&vgroup.epSet, "dnode_1", 6030); + addEpIntoEpSet(&vgroup.epSet, "dnode_2", 6030); + addEpIntoEpSet(&vgroup.epSet, "dnode_3", 6030); vgroup.epSet.inUse = 0; - vgroup.epSet.numOfEps = 3; meta_->vgs.emplace_back(vgroup); return *this; @@ -148,7 +146,7 @@ public: meta_[db][tbname]->schema->uid = id_++; meta_[db][tbname]->schema->tableType = TSDB_CHILD_TABLE; - SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0,}; + SVgroupInfo vgroup = { vgid, 0, 0, {0}, 0}; addEpIntoEpSet(&vgroup.epSet, "dnode_1", 6030); addEpIntoEpSet(&vgroup.epSet, "dnode_2", 6030); addEpIntoEpSet(&vgroup.epSet, "dnode_3", 6030); diff --git a/source/libs/parser/test/parserInsertTest.cpp b/source/libs/parser/test/parserInsertTest.cpp index d292fcf8b0b7987cf6f98e29e4a08cc15452db50..4aec141b5537fdb9b9773fe67dbe1474d85fcabc 100644 --- a/source/libs/parser/test/parserInsertTest.cpp +++ b/source/libs/parser/test/parserInsertTest.cpp @@ -15,6 +15,7 @@ #include +#include "os.h" #include "parInt.h" using namespace std; diff --git a/source/libs/parser/test/parserTestMain.cpp b/source/libs/parser/test/parserTestMain.cpp index 8cd69418b9842eea3c3af243a7f93cb6f51dfd55..95faadfc34cd9428f9ef49ae1a1addba1179aa3d 100644 --- a/source/libs/parser/test/parserTestMain.cpp +++ b/source/libs/parser/test/parserTestMain.cpp @@ -14,14 +14,20 @@ */ #include +#include +#include #include #include -#include "mockCatalog.h" +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif +#include "os.h" #include "parserTestUtil.h" #include "parToken.h" #include "functionMgt.h" +#include "mockCatalog.h" bool g_isDump = false; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 5427749d09d977f339cbdfd85cb156d44694b86e..b13c0a88889da5209a890862f376b25be9ac0b35 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -22,6 +22,7 @@ typedef struct SLogicPlanContext { } SLogicPlanContext; typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, SSelectStmt*, SLogicNode**); +typedef int32_t (*FCreateSetOpLogicNode)(SLogicPlanContext*, SSetOperator*, SLogicNode**); static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode); static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogicNode** pLogicNode); @@ -84,13 +85,19 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { +static int32_t rewriteExprForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { nodesWalkExprs(pExprs, doNameExpr, NULL); SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); return cxt.errCode; } +static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) { + SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; + nodesRewriteExprs(pTarget, doRewriteExpr, &cxt); + return cxt.errCode; +} + static int32_t pushLogicNode(SLogicPlanContext* pCxt, SLogicNode** pOldRoot, SLogicNode* pNewRoot) { if (NULL == pNewRoot->pChildren) { pNewRoot->pChildren = nodesMakeList(); @@ -343,7 +350,9 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr) } pCol->node.resType = pExpr->resType; strcpy(pCol->colName, pExpr->aliasName); - strcpy(pCol->tableAlias, pStmtName); + if (NULL != pStmtName) { + strcpy(pCol->tableAlias, pStmtName); + } return pCol; } @@ -430,10 +439,10 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { - code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY); + code = rewriteExprForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY); } if (TSDB_CODE_SUCCESS == code) { - code = rewriteExpr(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); + code = rewriteExprForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); } if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { @@ -469,7 +478,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm } if (TSDB_CODE_SUCCESS == code) { - code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW); + code = rewriteExprForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW); } if (TSDB_CODE_SUCCESS == code) { @@ -720,7 +729,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { - code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT); + code = rewriteExprForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT); } // set the output @@ -768,6 +777,171 @@ static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele return code; } +static int32_t createSetOpChildLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, FCreateSetOpLogicNode func, SLogicNode** pRoot) { + SLogicNode* pNode = NULL; + int32_t code = func(pCxt, pSetOperator, &pNode); + if (TSDB_CODE_SUCCESS == code && NULL != pNode) { + code = pushLogicNode(pCxt, pRoot, pNode); + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pNode); + } + return code; +} + +static int32_t createSetOpSortLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) { + if (NULL == pSetOperator->pOrderByList) { + return TSDB_CODE_SUCCESS; + } + + SSortLogicNode* pSort = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SORT); + if (NULL == pSort) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + + pSort->node.pTargets = nodesCloneList(pSetOperator->pProjectionList); + if (NULL == pSort->node.pTargets) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + if (TSDB_CODE_SUCCESS == code) { + pSort->pSortKeys = nodesCloneList(pSetOperator->pOrderByList); + if (NULL == pSort->pSortKeys) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pSort; + } else { + nodesDestroyNode(pSort); + } + + return code; +} + +static int32_t createSetOpProjectLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) { + SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT); + if (NULL == pProject) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (NULL != pSetOperator->pLimit) { + pProject->limit = ((SLimitNode*)pSetOperator->pLimit)->limit; + pProject->offset = ((SLimitNode*)pSetOperator->pLimit)->offset; + } else { + pProject->limit = -1; + pProject->offset = -1; + } + + int32_t code = TSDB_CODE_SUCCESS; + + pProject->pProjections = nodesCloneList(pSetOperator->pProjectionList); + if (NULL == pProject->pProjections) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByProjections(pCxt, NULL, pSetOperator->pProjectionList, &pProject->node.pTargets); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pProject; + } else { + nodesDestroyNode(pProject); + } + + return code; +} + +static int32_t createSetOpAggLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) { + SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG); + if (NULL == pAgg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + pAgg->pGroupKeys = nodesCloneList(pSetOperator->pProjectionList); + if (NULL == pAgg->pGroupKeys) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + // rewrite the expression in subsequent clauses + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExprs(pAgg->pGroupKeys, pSetOperator->pOrderByList); + } + + // set the output + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys, &pAgg->node.pTargets); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pAgg; + } else { + nodesDestroyNode(pAgg); + } + + return code; +} + +static int32_t createSetOpLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) { + SLogicNode* pSetOp = NULL; + int32_t code = TSDB_CODE_SUCCESS; + switch (pSetOperator->opType) { + case SET_OP_TYPE_UNION_ALL: + code = createSetOpProjectLogicNode(pCxt, pSetOperator, &pSetOp); + break; + case SET_OP_TYPE_UNION: + code = createSetOpAggLogicNode(pCxt, pSetOperator, &pSetOp); + break; + default: + code = -1; + break; + } + + SLogicNode* pLeft = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = createQueryLogicNode(pCxt, pSetOperator->pLeft, &pLeft); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pSetOp->pChildren, (SNode*)pLeft); + } + SLogicNode* pRight = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = createQueryLogicNode(pCxt, pSetOperator->pRight, &pRight); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pSetOp->pChildren, (SNode*)pRight); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pSetOp; + } else { + nodesDestroyNode(pSetOp); + } + + return code; +} + +static int32_t createSetOperatorLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) { + SLogicNode* pRoot = NULL; + int32_t code = createSetOpLogicNode(pCxt, pSetOperator, &pRoot); + if (TSDB_CODE_SUCCESS == code) { + code = createSetOpChildLogicNode(pCxt, pSetOperator, createSetOpSortLogicNode, &pRoot); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = pRoot; + } else { + nodesDestroyNode(pRoot); + } + + return code; +} + static int32_t getMsgType(ENodeType sqlType) { return (QUERY_NODE_CREATE_TABLE_STMT == sqlType || QUERY_NODE_CREATE_MULTI_TABLE_STMT == sqlType) ? TDMT_VND_CREATE_TABLE : TDMT_VND_SUBMIT; } @@ -791,6 +965,8 @@ static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogi return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt, pLogicNode); case QUERY_NODE_EXPLAIN_STMT: return createQueryLogicNode(pCxt, ((SExplainStmt*)pStmt)->pQuery, pLogicNode); + case QUERY_NODE_SET_OPERATOR: + return createSetOperatorLogicNode(pCxt, (SSetOperator*)pStmt, pLogicNode); default: break; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 34bee3fd0b7dcda03e364a381e2520128bb58274..832135e90eaa9bdf90c15c48158f784fee3ba1ac 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -228,6 +228,8 @@ static int32_t cpdMergeConds(SNode** pDst, SNodeList** pSrc) { if (NULL == pLogicCond) { return TSDB_CODE_OUT_OF_MEMORY; } + pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL; + pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pLogicCond->condType = LOGIC_COND_TYPE_AND; pLogicCond->pParameterList = *pSrc; *pDst = (SNode*)pLogicCond; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index caeab20970c59906308ecad94ceb3ea2d4688e84..745029730fcc6cc0fb19d15c025936791ac18bb1 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -38,7 +38,7 @@ typedef struct SPhysiPlanContext { static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; - if (NULL != pStmtName) { + if (NULL != pStmtName && '\0' != pStmtName[0]) { return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName); } if ('\0' == pCol->tableAlias[0]) { @@ -47,7 +47,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { return sprintf(pKey, "%s.%s", pCol->tableAlias, pCol->colName); } - if (NULL != pStmtName) { + if (NULL != pStmtName && '\0' != pStmtName[0]) { return sprintf(pKey, "%s.%s", pStmtName, ((SExprNode*)pNode)->aliasName); } return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); diff --git a/source/libs/planner/src/planScaleOut.c b/source/libs/planner/src/planScaleOut.c index 2b5fd12e226f20571acf47e7f332168bc5159d93..7bb97b59d70b1c827d7ce7831fe3d9573b8cd1d6 100644 --- a/source/libs/planner/src/planScaleOut.c +++ b/source/libs/planner/src/planScaleOut.c @@ -129,7 +129,7 @@ static int32_t pushHierarchicalPlan(SNodeList* pParentsGroup, SNodeList* pCurren return code; } -static int32_t doScaleOut(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t* pLevel, SNodeList* pParentsGroup) { +static int32_t doScaleOut(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pParentsGroup) { SNodeList* pCurrentGroup = nodesMakeList(); if (NULL == pCurrentGroup) { return TSDB_CODE_OUT_OF_MEMORY; @@ -138,13 +138,13 @@ static int32_t doScaleOut(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32 int32_t code = TSDB_CODE_SUCCESS; switch (pSubplan->subplanType) { case SUBPLAN_TYPE_MERGE: - code = scaleOutForMerge(pCxt, pSubplan, *pLevel, pCurrentGroup); + code = scaleOutForMerge(pCxt, pSubplan, level, pCurrentGroup); break; case SUBPLAN_TYPE_SCAN: - code = scaleOutForScan(pCxt, pSubplan, *pLevel, pCurrentGroup); + code = scaleOutForScan(pCxt, pSubplan, level, pCurrentGroup); break; case SUBPLAN_TYPE_MODIFY: - code = scaleOutForModify(pCxt, pSubplan, *pLevel, pCurrentGroup); + code = scaleOutForModify(pCxt, pSubplan, level, pCurrentGroup); break; default: break; @@ -152,13 +152,12 @@ static int32_t doScaleOut(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32 if (TSDB_CODE_SUCCESS == code) { code = pushHierarchicalPlan(pParentsGroup, pCurrentGroup); - ++(*pLevel); } if (TSDB_CODE_SUCCESS == code) { SNode* pChild; FOREACH(pChild, pSubplan->pChildren) { - code = doScaleOut(pCxt, (SLogicSubplan*)pChild, pLevel, pCurrentGroup); + code = doScaleOut(pCxt, (SLogicSubplan*)pChild, level + 1, pCurrentGroup); if (TSDB_CODE_SUCCESS != code) { break; } @@ -194,7 +193,7 @@ int32_t scaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQue } SScaleOutContext cxt = { .pPlanCxt = pCxt, .subplanId = 1 }; - int32_t code = doScaleOut(&cxt, pLogicSubplan, &(pPlan->totalLevel), pPlan->pTopSubplans); + int32_t code = doScaleOut(&cxt, pLogicSubplan, 0, pPlan->pTopSubplans); if (TSDB_CODE_SUCCESS == code) { *pLogicPlan = pPlan; } else { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index e54cf339347ce7955c830149b6f0e4f6655ef84e..b419414ca6ddaa3454ca4d371ea6b6f52d1f44fe 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -45,7 +45,17 @@ typedef struct SCtjInfo { SLogicSubplan* pSubplan; } SCtjInfo; -typedef bool (*FSplFindSplitNode)(SLogicSubplan* pSubplan, SStsInfo* pInfo); +typedef struct SUaInfo { + SProjectLogicNode* pProject; + SLogicSubplan* pSubplan; +} SUaInfo; + +typedef struct SUnInfo { + SAggLogicNode* pAgg; + SLogicSubplan* pSubplan; +} SUnInfo; + +typedef bool (*FSplFindSplitNode)(SLogicSubplan* pSubplan, void* pInfo); static SLogicSubplan* splCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pScan, int32_t flag) { SLogicSubplan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); @@ -132,16 +142,10 @@ static bool stsFindSplitNode(SLogicSubplan* pSubplan, SStsInfo* pInfo) { static int32_t stsSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { SStsInfo info = {0}; - if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_STS, stsFindSplitNode, &info)) { + if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_STS, (FSplFindSplitNode)stsFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; } - if (NULL == info.pSubplan->pChildren) { - info.pSubplan->pChildren = nodesMakeList(); - if (NULL == info.pSubplan->pChildren) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } - int32_t code = nodesListStrictAppend(info.pSubplan->pChildren, splCreateScanSubplan(pCxt, info.pScan, SPLIT_FLAG_STS)); + int32_t code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, splCreateScanSubplan(pCxt, info.pScan, SPLIT_FLAG_STS)); if (TSDB_CODE_SUCCESS == code) { code = splCreateExchangeNode(pCxt, info.pSubplan, info.pScan, SUBPLAN_TYPE_MERGE); } @@ -173,7 +177,7 @@ static SLogicNode* ctjMatchByNode(SLogicNode* pNode) { return NULL; } -static bool ctjFindSplitNode(SLogicSubplan* pSubplan, SStsInfo* pInfo) { +static bool ctjFindSplitNode(SLogicSubplan* pSubplan, SCtjInfo* pInfo) { SLogicNode* pSplitNode = ctjMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { pInfo->pScan = (SScanLogicNode*)pSplitNode; @@ -184,18 +188,172 @@ static bool ctjFindSplitNode(SLogicSubplan* pSubplan, SStsInfo* pInfo) { static int32_t ctjSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { SCtjInfo info = {0}; - if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_CTJ, ctjFindSplitNode, &info)) { + if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_CTJ, (FSplFindSplitNode)ctjFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; } - if (NULL == info.pSubplan->pChildren) { - info.pSubplan->pChildren = nodesMakeList(); - if (NULL == info.pSubplan->pChildren) { - return TSDB_CODE_OUT_OF_MEMORY; + int32_t code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, splCreateScanSubplan(pCxt, info.pScan, SPLIT_FLAG_CTJ)); + if (TSDB_CODE_SUCCESS == code) { + code = splCreateExchangeNode(pCxt, info.pSubplan, info.pScan, info.pSubplan->subplanType); + } + ++(pCxt->groupId); + pCxt->split = true; + return code; +} + +static SLogicNode* uaMatchByNode(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pNode) && LIST_LENGTH(pNode->pChildren) > 1) { + return pNode; + } + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + SLogicNode* pSplitNode = uaMatchByNode((SLogicNode*)pChild); + if (NULL != pSplitNode) { + return pSplitNode; + } + } + return NULL; +} + +static bool uaFindSplitNode(SLogicSubplan* pSubplan, SUaInfo* pInfo) { + SLogicNode* pSplitNode = uaMatchByNode(pSubplan->pNode); + if (NULL != pSplitNode) { + pInfo->pProject = (SProjectLogicNode*)pSplitNode; + pInfo->pSubplan = pSubplan; + } + return NULL != pSplitNode; +} + +static SLogicSubplan* uaCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode) { + SLogicSubplan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); + if (NULL == pSubplan) { + return NULL; + } + pSubplan->id.groupId = pCxt->groupId; + pSubplan->subplanType = SUBPLAN_TYPE_SCAN; + pSubplan->pNode = pNode; + return pSubplan; +} + +static int32_t uaCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SProjectLogicNode* pProject) { + SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); + if (NULL == pExchange) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pExchange->srcGroupId = pCxt->groupId; + // pExchange->precision = pScan->pMeta->tableInfo.precision; + pExchange->node.pTargets = nodesCloneList(pProject->node.pTargets); + if (NULL == pExchange->node.pTargets) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + + if (NULL == pProject->node.pParent) { + pSubplan->pNode = (SLogicNode*)pExchange; + nodesDestroyNode(pProject); + return TSDB_CODE_SUCCESS; + } + + SNode* pNode; + FOREACH(pNode, pProject->node.pParent->pChildren) { + if (nodesEqualNode(pNode, pProject)) { + REPLACE_NODE(pExchange); + nodesDestroyNode(pNode); + return TSDB_CODE_SUCCESS; + } + } + nodesDestroyNode(pExchange); + return TSDB_CODE_FAILED; +} + +static int32_t uaSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { + SUaInfo info = {0}; + if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)uaFindSplitNode, &info)) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = TSDB_CODE_SUCCESS; + + SNode* pChild = NULL; + FOREACH(pChild, info.pProject->node.pChildren) { + code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, uaCreateSubplan(pCxt, (SLogicNode*)pChild)); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(NULL); + } else { + break; } } - int32_t code = nodesListStrictAppend(info.pSubplan->pChildren, splCreateScanSubplan(pCxt, info.pScan, SPLIT_FLAG_CTJ)); if (TSDB_CODE_SUCCESS == code) { - code = splCreateExchangeNode(pCxt, info.pSubplan, info.pScan, info.pSubplan->subplanType); + nodesClearList(info.pProject->node.pChildren); + info.pProject->node.pChildren = NULL; + code = uaCreateExchangeNode(pCxt, info.pSubplan, info.pProject); + } + ++(pCxt->groupId); + pCxt->split = true; + return code; +} + +static SLogicNode* unMatchByNode(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) && LIST_LENGTH(pNode->pChildren) > 1) { + return pNode; + } + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + SLogicNode* pSplitNode = uaMatchByNode((SLogicNode*)pChild); + if (NULL != pSplitNode) { + return pSplitNode; + } + } + return NULL; +} + +static int32_t unCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SAggLogicNode* pAgg) { + SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); + if (NULL == pExchange) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pExchange->srcGroupId = pCxt->groupId; + // pExchange->precision = pScan->pMeta->tableInfo.precision; + pExchange->node.pTargets = nodesCloneList(pAgg->node.pTargets); + if (NULL == pExchange->node.pTargets) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + + return nodesListMakeAppend(&pAgg->node.pChildren, pExchange); +} + +static bool unFindSplitNode(SLogicSubplan* pSubplan, SUnInfo* pInfo) { + SLogicNode* pSplitNode = unMatchByNode(pSubplan->pNode); + if (NULL != pSplitNode) { + pInfo->pAgg = (SAggLogicNode*)pSplitNode; + pInfo->pSubplan = pSubplan; + } + return NULL != pSplitNode; +} + +static int32_t unSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { + SUnInfo info = {0}; + if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)unFindSplitNode, &info)) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = TSDB_CODE_SUCCESS; + + SNode* pChild = NULL; + FOREACH(pChild, info.pAgg->node.pChildren) { + code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, uaCreateSubplan(pCxt, (SLogicNode*)pChild)); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(NULL); + } else { + break; + } + } + if (TSDB_CODE_SUCCESS == code) { + nodesClearList(info.pAgg->node.pChildren); + info.pAgg->node.pChildren = NULL; + code = unCreateExchangeNode(pCxt, info.pSubplan, info.pAgg); } ++(pCxt->groupId); pCxt->split = true; @@ -205,6 +363,8 @@ static int32_t ctjSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { static const SSplitRule splitRuleSet[] = { { .pName = "SuperTableScan", .splitFunc = stsSplit }, { .pName = "ChildTableJoin", .splitFunc = ctjSplit }, + { .pName = "UnionAll", .splitFunc = uaSplit }, + { .pName = "Union", .splitFunc = unSplit } }; static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule)); diff --git a/source/libs/planner/test/CMakeLists.txt b/source/libs/planner/test/CMakeLists.txt index e068cd8698d8ddf5b7211fa984bea5dee3a66c7b..f0c59b969f0827a3c1a24c424819d097c71891ba 100644 --- a/source/libs/planner/test/CMakeLists.txt +++ b/source/libs/planner/test/CMakeLists.txt @@ -22,3 +22,12 @@ TARGET_INCLUDE_DIRECTORIES( PRIVATE "${TD_SOURCE_DIR}/source/libs/planner/inc" PRIVATE "${TD_SOURCE_DIR}/source/libs/parser/test" ) + + +if(${BUILD_WINGETOPT}) + target_include_directories( + plannerTest + PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" + ) + target_link_libraries(plannerTest PUBLIC wingetopt) +endif() \ No newline at end of file diff --git a/2.0/src/client/inc/tscProfile.h b/source/libs/planner/test/planSetOpTest.cpp similarity index 52% rename from 2.0/src/client/inc/tscProfile.h rename to source/libs/planner/test/planSetOpTest.cpp index 193fac0fb00eaa20c883215e5708eea8f07f1bc9..5ace503959f0987fe0d96cdcca04b8869bf2389e 100644 --- a/2.0/src/client/inc/tscProfile.h +++ b/source/libs/planner/test/planSetOpTest.cpp @@ -13,26 +13,23 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSCPROFILE_H -#define TDENGINE_TSCPROFILE_H +#include "planTestUtil.h" +#include "planner.h" -#ifdef __cplusplus -extern "C" { -#endif +using namespace std; -#include "tsclient.h" +class PlanSetOpTest : public PlannerTestBase { -void tscAddIntoSqlList(SSqlObj *pSql); -void tscRemoveFromSqlList(SSqlObj *pSql); -void tscAddIntoStreamList(SSqlStream *pStream); -void tscRemoveFromStreamList(SSqlStream *pStream, SSqlObj *pSqlObj); -int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj); -void tscKillQuery(STscObj *pObj, uint32_t killId); -void tscKillStream(STscObj *pObj, uint32_t killId); -void tscKillConnection(STscObj *pObj); +}; -#ifdef __cplusplus +TEST_F(PlanSetOpTest, unionAll) { + useDb("root", "test"); + + run("select c1, c2 from t1 where c1 > 10 union all select c1, c2 from t1 where c1 > 20"); } -#endif -#endif // TDENGINE_TSCPROFILE_H +TEST_F(PlanSetOpTest, union) { + useDb("root", "test"); + + run("select c1, c2 from t1 where c1 > 10 union select c1, c2 from t1 where c1 > 20"); +} diff --git a/source/libs/planner/test/plannerTestMain.cpp b/source/libs/planner/test/planTestMain.cpp similarity index 98% rename from source/libs/planner/test/plannerTestMain.cpp rename to source/libs/planner/test/planTestMain.cpp index 2878dc7954d234ef43c7002abacde163fe0a0671..84cf75c5be8aede5873e1547c05c469c9f81a2bb 100644 --- a/source/libs/planner/test/plannerTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -16,7 +16,7 @@ #include #include - +#include "getopt.h" #include "mockCatalog.h" #include "planTestUtil.h" diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 25457d3e411ea9b4be253409540d2e31cefe142d..f124970875ceedc582d133ecf0185975f4fab3ad 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include "planTestUtil.h" #include @@ -62,7 +63,11 @@ public: doScaleOutLogicPlan(&cxt, pLogicSubplan, &pLogicPlan); SQueryPlan* pPlan = nullptr; - doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan, NULL); + doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan); + + if (g_isDump) { + dump(); + } } catch (...) { dump(); throw; @@ -87,6 +92,7 @@ private: string splitLogicPlan_; string scaledLogicPlan_; string physiPlan_; + vector physiSubplans_; }; void reset() { @@ -115,6 +121,10 @@ private: cout << res_.scaledLogicPlan_ << endl; cout << "physical plan : " << endl; cout << res_.physiPlan_ << endl; + cout << "physical subplan : " << endl; + for (const auto& subplan : res_.physiSubplans_) { + cout << subplan << endl; + } } void doParseSql(const string& sql, SQuery** pQuery) { @@ -153,9 +163,17 @@ private: res_.scaledLogicPlan_ = toString((SNode*)(*pLogicPlan)); } - void doCreatePhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList) { + void doCreatePhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan) { + SArray* pExecNodeList = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SQueryNodeAddr)); DO_WITH_THROW(createPhysiPlan, pCxt, pLogicPlan, pPlan, pExecNodeList); res_.physiPlan_ = toString((SNode*)(*pPlan)); + SNode* pNode; + FOREACH(pNode, (*pPlan)->pSubplans) { + SNode* pSubplan; + FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { + res_.physiSubplans_.push_back(toString(pSubplan)); + } + } } void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 932107dfc1a9ffcdaa95300478a9609628582795..9e37784cab9be3e44d4547967b277b7c97a03c70 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -157,6 +157,28 @@ int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } +int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SRetrieveFuncReq funcReq = {0}; + funcReq.numOfFuncs = 1; + funcReq.ignoreCodeComment = true; + funcReq.pFuncNames = taosArrayInit(1, strlen(input) + 1); + taosArrayPush(funcReq.pFuncNames, input); + + int32_t bufLen = tSerializeSRetrieveFuncReq(NULL, 0, &funcReq); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSRetrieveFuncReq(pBuf, bufLen, &funcReq); + + taosArrayDestroy(funcReq.pFuncNames); + + *msg = pBuf; + *msgLen = bufLen; + + return TSDB_CODE_SUCCESS; +} int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { SUseDbOutput *pOut = output; @@ -379,6 +401,31 @@ int32_t queryProcessGetIndexRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_SUCCESS; } +int32_t queryProcessRetrieveFuncRsp(void *output, char *msg, int32_t msgSize) { + SRetrieveFuncRsp out = {0}; + + if (NULL == output || NULL == msg || msgSize <= 0) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + if (tDeserializeSRetrieveFuncRsp(msg, msgSize, &out) != 0) { + qError("tDeserializeSRetrieveFuncRsp failed, msgSize:%d", msgSize); + return TSDB_CODE_INVALID_MSG; + } + + if (1 != out.numOfFuncs) { + qError("invalid func num returned, numOfFuncs:%d", out.numOfFuncs); + return TSDB_CODE_INVALID_MSG; + } + + SFuncInfo * funcInfo = taosArrayGet(out.pFuncInfos, 0); + + memcpy(output, funcInfo, sizeof(*funcInfo)); + taosArrayDestroy(out.pFuncInfos); + + return TSDB_CODE_SUCCESS; +} + void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryBuildTableMetaReqMsg; @@ -386,6 +433,7 @@ void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_MND_QNODE_LIST)] = queryBuildQnodeListMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryBuildGetDBCfgMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryBuildGetIndexMsg; + queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryBuildRetrieveFuncMsg; queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp; @@ -393,6 +441,7 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_QNODE_LIST)] = queryProcessQnodeListRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryProcessGetDbCfgRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryProcessGetIndexRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryProcessRetrieveFuncRsp; } #pragma GCC diagnostic pop diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index d62f9f04b88c94f93294d4a66a0765c00477a9eb..cc82349feedb20f15760bbf26a32237673867544 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -95,6 +95,9 @@ typedef struct SQWPhaseInput { } SQWPhaseInput; typedef struct SQWPhaseOutput { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } SQWPhaseOutput; diff --git a/source/libs/qworker/inc/qworkerMsg.h b/source/libs/qworker/inc/qworkerMsg.h index c0f4d6d1574c0e9b843d600ac81c84a2a6264f50..fb1950d16bb6dafcbf8c44537e94a33d51b58cf3 100644 --- a/source/libs/qworker/inc/qworkerMsg.h +++ b/source/libs/qworker/inc/qworkerMsg.h @@ -43,7 +43,8 @@ void qwFreeFetchRsp(void *msg); int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp); int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp); int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *rsp, int32_t code); -int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn); +int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn); +int32_t qwRegisterHbBrokenLinkArg(SQWorkerMgmt *mgmt, uint64_t sId, SQWConnInfo *pConn); #ifdef __cplusplus } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index f97d7ffae0b344716f4947f436bb89e0ad665fd0..a74c28cf5395c7e37b382654e89234d5e8f93ba1 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -948,7 +948,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex DataSinkHandle sinkHandle = NULL; SQWTaskCtx *ctx = NULL; - QW_ERR_JRET(qwRegisterBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo)); + QW_ERR_JRET(qwRegisterQueryBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo)); QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, NULL)); @@ -1285,23 +1285,51 @@ _return: QW_RET(TSDB_CODE_SUCCESS); } +int32_t qwProcessHbLinkBroken(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { + int32_t code = 0; + SSchedulerHbRsp rsp = {0}; + SQWSchStatus *sch = NULL; + + QW_ERR_RET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); + + QW_LOCK(QW_WRITE, &sch->hbConnLock); + + if (qwMsg->connInfo.handle == sch->hbConnInfo.handle) { + tmsgReleaseHandle(sch->hbConnInfo.handle, TAOS_CONN_SERVER); + sch->hbConnInfo.handle = NULL; + sch->hbConnInfo.ahandle = NULL; + + QW_DLOG("release hb handle due to connection broken, handle:%p", qwMsg->connInfo.handle); + } else { + QW_DLOG("ignore hb connection broken, handle:%p, currentHandle:%p", qwMsg->connInfo.handle, sch->hbConnInfo.handle); + } + + QW_UNLOCK(QW_WRITE, &sch->hbConnLock); + + qwReleaseScheduler(QW_READ, mgmt); + + QW_RET(TSDB_CODE_SUCCESS); +} + int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; SQWSchStatus *sch = NULL; - uint64_t seqId = 0; - void *origHandle = NULL; - memcpy(&rsp.epId, &req->epId, sizeof(req->epId)); + if (qwMsg->code) { + QW_RET(qwProcessHbLinkBroken(mgmt, qwMsg, req)); + } QW_ERR_JRET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); + QW_ERR_JRET(qwRegisterHbBrokenLinkArg(mgmt, req->sId, &qwMsg->connInfo)); + QW_LOCK(QW_WRITE, &sch->hbConnLock); if (sch->hbConnInfo.handle) { tmsgReleaseHandle(sch->hbConnInfo.handle, TAOS_CONN_SERVER); } - + memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo)); memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId)); @@ -1314,7 +1342,14 @@ int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { _return: + memcpy(&rsp.epId, &req->epId, sizeof(req->epId)); + qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code); + + if (code) { + tmsgReleaseHandle(qwMsg->connInfo.handle, TAOS_CONN_SERVER); + } + QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); QW_RET(TSDB_CODE_SUCCESS); diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 15a42d3a3157411ba3a9bbbdaa6d0576d41ce7d4..6723eb21c1c6ad96db086fc7e08c9826249fc5c7 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -286,7 +286,7 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { } -int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { +int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { STaskDropReq * req = (STaskDropReq *)rpcMallocCont(sizeof(STaskDropReq)); if (NULL == req) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(STaskDropReq)); @@ -313,6 +313,42 @@ int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { return TSDB_CODE_SUCCESS; } +int32_t qwRegisterHbBrokenLinkArg(SQWorkerMgmt *mgmt, uint64_t sId, SQWConnInfo *pConn) { + SSchedulerHbReq req = {0}; + req.header.vgId = mgmt->nodeId; + req.sId = sId; + + int32_t msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); + if (msgSize < 0) { + QW_SCH_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + void *msg = rpcMallocCont(msgSize); + if (NULL == msg) { + QW_SCH_ELOG("calloc %d failed", msgSize); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { + QW_SCH_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); + taosMemoryFree(msg); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SRpcMsg pMsg = { + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .msgType = TDMT_VND_QUERY_HEARTBEAT, + .pCont = msg, + .contLen = sizeof(SSchedulerHbReq), + .code = TSDB_CODE_RPC_NETWORK_UNAVAIL, + }; + + tmsgRegisterBrokenLinkArg(&mgmt->msgCb, &pMsg); + + return TSDB_CODE_SUCCESS; +} + + int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { @@ -587,10 +623,14 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { } uint64_t sId = req.sId; - SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .code = pMsg->code}; qwMsg.connInfo.handle = pMsg->handle; qwMsg.connInfo.ahandle = pMsg->ahandle; + if (TSDB_CODE_RPC_NETWORK_UNAVAIL == pMsg->code) { + QW_SCH_DLOG("receive Hb msg due to network broken, error:%s", tstrerror(pMsg->code)); + } + QW_SCH_DLOG("processHb start, node:%p, handle:%p", node, pMsg->handle); QW_ERR_RET(qwProcessHb(mgmt, &qwMsg, &req)); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 7179fa6984b7a5564fac9daf4d393b34c2c4ba7f..5a57a47df84a86b3c4f9d590cac1b3beda849fd8 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -27,6 +27,9 @@ #pragma GCC diagnostic ignored "-Wpointer-arith" #include +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" @@ -277,8 +280,8 @@ int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, uint64_t taskId, struct SSub qwtTestSinkMaxBlockNum = taosRand() % 100 + 1; qwtTestSinkQueryEnd = false; - *pTaskInfo = (qTaskInfo_t)qwtTestCaseIdx+1; - *handle = (DataSinkHandle)qwtTestCaseIdx+2; + *pTaskInfo = (qTaskInfo_t)((char*)qwtTestCaseIdx+1); + *handle = (DataSinkHandle)((char*)qwtTestCaseIdx+2); ++qwtTestCaseIdx; @@ -435,9 +438,16 @@ void stubSetStringToPlan() { static Stub stub; stub.set(qStringToSubplan, qwtStringToPlan); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qStringToSubplan", result); +#endif +#ifdef LINUX AddrAny any("libplanner.so"); std::map result; any.get_global_func_addr_dynsym("^qStringToSubplan$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtStringToPlan); } @@ -448,9 +458,16 @@ void stubSetExecTask() { static Stub stub; stub.set(qExecTask, qwtExecTask); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qExecTask", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^qExecTask$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtExecTask); } @@ -463,9 +480,16 @@ void stubSetCreateExecTask() { static Stub stub; stub.set(qCreateExecTask, qwtCreateExecTask); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qCreateExecTask", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^qCreateExecTask$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtCreateExecTask); } @@ -476,9 +500,16 @@ void stubSetAsyncKillTask() { static Stub stub; stub.set(qAsyncKillTask, qwtKillTask); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qAsyncKillTask", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^qAsyncKillTask$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtKillTask); } @@ -489,9 +520,16 @@ void stubSetDestroyTask() { static Stub stub; stub.set(qDestroyTask, qwtDestroyTask); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qDestroyTask", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^qDestroyTask$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtDestroyTask); } @@ -503,9 +541,16 @@ void stubSetDestroyDataSinker() { static Stub stub; stub.set(dsDestroyDataSinker, qwtDestroyDataSinker); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsDestroyDataSinker", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^dsDestroyDataSinker$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtDestroyDataSinker); } @@ -516,9 +561,16 @@ void stubSetGetDataLength() { static Stub stub; stub.set(dsGetDataLength, qwtGetDataLength); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsGetDataLength", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^dsGetDataLength$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtGetDataLength); } @@ -529,9 +581,16 @@ void stubSetEndPut() { static Stub stub; stub.set(dsEndPut, qwtEndPut); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsEndPut", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^dsEndPut$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtEndPut); } @@ -542,9 +601,16 @@ void stubSetPutDataBlock() { static Stub stub; stub.set(dsPutDataBlock, qwtPutDataBlock); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsPutDataBlock", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^dsPutDataBlock$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtPutDataBlock); } @@ -555,9 +621,16 @@ void stubSetRpcSendResponse() { static Stub stub; stub.set(rpcSendResponse, qwtRpcSendResponse); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendResponse", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendResponse$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtRpcSendResponse); } @@ -568,9 +641,16 @@ void stubSetGetDataBlock() { static Stub stub; stub.set(dsGetDataBlock, qwtGetDataBlock); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsGetDataBlock", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^dsGetDataBlock$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtGetDataBlock); } diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 977fab2e85d8f782476c6faa56ca9d0deb3efd6a..99c33d3165d306bd89cb1411b6de417065b182ea 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -215,8 +215,10 @@ typedef struct SFilterPCtx { } SFilterPCtx; typedef struct SFltTreeStat { - int32_t code; - bool scalarMode; + int32_t code; + int8_t precision; + bool scalarMode; + SFilterInfo* info; } SFltTreeStat; typedef struct SFltScalarCtx { @@ -308,7 +310,6 @@ struct SFilterInfo { #define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGetData(((SColumnInfoData *)(fi)->data), (ri))) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) -#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc) #define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX) #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid]) @@ -321,7 +322,6 @@ struct SFilterInfo { #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) -#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) #define FILTER_UNIT_COL_IDX(u) ((u)->left.idx) #define FILTER_UNIT_OPTR(u) ((u)->compare.optr) #define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func) diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index f15116bdb768b12d75992c1ff16f5243685ba2cc..adbdd13a8435773b9002d59d7ed4047dcbc5fc76 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -56,6 +56,8 @@ static FORCE_INLINE double getVectorDoubleValue_BOOL(void *src, int32_t index) { return (double)*((bool *)src + index); } +double getVectorDoubleValue_JSON(void *src, int32_t index); + static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { _getDoubleValue_fn_t p = NULL; if (srcType == TSDB_DATA_TYPE_TINYINT) { @@ -80,6 +82,8 @@ static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) p = getVectorDoubleValue_DOUBLE; } else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) { p = getVectorDoubleValue_BIGINT; + } else if (srcType == TSDB_DATA_TYPE_JSON) { + p = getVectorDoubleValue_JSON; } else if (srcType == TSDB_DATA_TYPE_BOOL) { p = getVectorDoubleValue_BOOL; } else { diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 191143de1274f7e715f68e577e4bd2cae54449f4..e24162bccbd97657d8097309dd38076c9f71f35e 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -57,28 +57,24 @@ OptrStr gOptrStr[] = { {OP_TYPE_IS_NOT_UNKNOWN, "not unknown"}, // json operator - {OP_TYPE_JSON_GET_VALUE, "json get"}, + {OP_TYPE_JSON_GET_VALUE, "->"}, {OP_TYPE_JSON_CONTAINS, "json contains"} }; bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(maxv, minr); - //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result >= 0; } bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(maxv, minr); - //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result > 0; } bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(minv, maxr); - //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result <= 0; } bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(minv, maxr); - //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result < 0; } bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { @@ -170,7 +166,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch, compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8, - compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch + compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch, compareJsonContainsKey }; int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { @@ -221,7 +217,12 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { assert(0); } } - + + if (optr == OP_TYPE_JSON_CONTAINS && type == TSDB_DATA_TYPE_JSON) { + return 28; + } + + switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: comparFn = 1; break; @@ -1049,6 +1050,8 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { cell = cell->pNext; } + colDataDestroy(out.columnData); + taosMemoryFree(out.columnData); } else { filterAddFieldFromNode(info, node->pRight, &right); @@ -1778,9 +1781,9 @@ int32_t fltInitValFieldData(SFilterInfo *info) { bytes = (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; fi->data = taosMemoryCalloc(1, bytes); - } else if (type != TSDB_DATA_TYPE_JSON){ + } else{ if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE -/* +/* fi->data = taosMemoryCalloc(dType->bytes, tDataTypes[type].bytes); for (int32_t a = 0; a < dType->bytes; ++a) { int64_t *v = taosArrayGet(var->arr, a); @@ -1791,31 +1794,29 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } else { fi->data = taosMemoryCalloc(1, sizeof(int64_t)); } - } else{ // type == TSDB_DATA_TYPE_JSON - // fi->data = null; use fi->desc as data, because json value is variable, so use tVariant (fi->desc) } - if(type != TSDB_DATA_TYPE_JSON) { - if (dType->type == type) { - assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type); + if (dType->type == type) { + assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type); + } else { + SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; + out.columnData->info.type = type; + if (IS_VAR_DATA_TYPE(type)) { + out.columnData->info.bytes = bytes; } else { - SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; - out.columnData->info.type = type; - if (IS_VAR_DATA_TYPE(type)) { - out.columnData->info.bytes = bytes; - } else { - out.columnData->info.bytes = tDataTypes[type].bytes; - } - - // todo refactor the convert - int32_t code = doConvertDataType(var, &out); - if (code != TSDB_CODE_SUCCESS) { - qError("convert value to type[%d] failed", type); - return TSDB_CODE_TSC_INVALID_OPERATION; - } + out.columnData->info.bytes = tDataTypes[type].bytes; + } - memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes); + // todo refactor the convert + int32_t code = doConvertDataType(var, &out); + if (code != TSDB_CODE_SUCCESS) { + qError("convert value to type[%d] failed", type); + return TSDB_CODE_TSC_INVALID_OPERATION; } + + memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes); + colDataDestroy(out.columnData); + taosMemoryFree(out.columnData); } // match/nmatch for nchar type need convert from ucs4 to mbs @@ -2556,11 +2557,7 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit); if (unit->right.type == FLD_TYPE_VALUE) { - if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){ // json value is tVariant - info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit); - }else{ - info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); - } + info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); } else { info->cunits[i].valData = NULL; } @@ -2886,18 +2883,8 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); - if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ - if (!colData){ // for json->'key' is null - (*p)[i] = 1; - }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is null - colData = POINTER_SHIFT(colData, CHAR_BYTES); - (*p)[i] = colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL); - }else{ - (*p)[i] = 0; - } - }else{ - (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); - } + (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); + if ((*p)[i] == 0) { all = false; } @@ -2921,19 +2908,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); - if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ - if (!colData) { // for json->'key' is not null - (*p)[i] = 0; - }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is not null - colData = POINTER_SHIFT(colData, CHAR_BYTES); - (*p)[i] = !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL); - }else{ // for json->'key' is not null - (*p)[i] = 1; - } - }else { - (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); - } - + (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); if ((*p)[i] == 0) { all = false; } @@ -3389,6 +3364,12 @@ int32_t filterGetTimeRangeImpl(SFilterInfo *info, STimeWindow *win, bool * filterGetRangeRes(prev, &tra); win->skey = tra.s; win->ekey = tra.e; + if (FILTER_GET_FLAG(tra.sflag, RANGE_FLG_EXCLUDE)) { + win->skey++; + } + if (FILTER_GET_FLAG(tra.eflag, RANGE_FLG_EXCLUDE)) { + win->ekey--; + } } filterFreeRangeCtx(prev); @@ -3517,7 +3498,40 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } - if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { + if (QUERY_NODE_VALUE == nodeType(*pNode)) { + if (!FILTER_GET_FLAG(stat->info->options, FLT_OPTION_TIMESTAMP)) { + return DEAL_RES_CONTINUE; + } + + SValueNode *valueNode = (SValueNode *)*pNode; + if (TSDB_DATA_TYPE_BINARY != valueNode->node.resType.type) { + return DEAL_RES_CONTINUE; + } + +#if 0 + if (stat->precision < 0) { + //TODO + return DEAL_RES_CONTINUE; + } + + char *timeStr = valueNode->datum.p; + if (taosParseTime(valueNode->datum.p, &valueNode->datum.i, valueNode->node.resType.bytes, stat->precision, tsDaylight) != + TSDB_CODE_SUCCESS) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); + } + TODO +#else + return DEAL_RES_CONTINUE; +#endif + } + + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + SColumnNode *colNode = (SColumnNode *)*pNode; + stat->precision = colNode->node.resType.precision; + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_NODE_LIST == nodeType(*pNode)) { return DEAL_RES_CONTINUE; } @@ -3566,6 +3580,11 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } + if (OP_TYPE_JSON_CONTAINS == node->opType) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) { SNode *t = node->pLeft; node->pLeft = node->pRight; @@ -3676,16 +3695,19 @@ int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) info = *pInfo; info->options = options; - SFltTreeStat stat1 = {0}; - FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat1)); + SFltTreeStat stat = {0}; + stat.precision = -1; + stat.info = info; + + FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); - info->scalarMode = stat1.scalarMode; + info->scalarMode = stat.scalarMode; if (!info->scalarMode) { FLT_ERR_JRET(fltInitFromNode(pNode, info, options)); } else { info->sclCtx.node = pNode; - FLT_ERR_JRET(fltOptimizeNodes(info, &info->sclCtx.node, &stat1)); + FLT_ERR_JRET(fltOptimizeNodes(info, &info->sclCtx.node, &stat)); } return code; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 337a773d5c00d28f4a1cf78bf7c11a89dd84aa39..145afbe984a8f19e30871cd91bd62f9c5520cf32 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -132,6 +132,7 @@ void sclFreeRes(SHashObj *res) { void sclFreeParam(SScalarParam *param) { if (param->columnData != NULL) { colDataDestroy(param->columnData); + taosMemoryFree(param->columnData); } if (param->pHashFilter != NULL) { @@ -605,37 +606,48 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) { EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { STargetNode *target = (STargetNode *)pNode; - + if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); ctx->code = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } - SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId); + int32_t index = -1; + for(int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) { + SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, i); + if (pb->info.blockId == target->dataBlockId) { + index = i; + break; + } + } + + if (index == -1) { + sclError("column tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index); + if (target->slotId >= taosArrayGetSize(block->pDataBlock)) { sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); ctx->code = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } + // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock, SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId); - + SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); if (NULL == res) { sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); ctx->code = TSDB_CODE_QRY_APP_ERROR; return DEAL_RES_ERROR; } - - for (int32_t i = 0; i < res->numOfRows; ++i) { - if (colDataIsNull(res->columnData, res->numOfRows, i, NULL)) { - colDataAppend(col, i, NULL, true); - } else { - char *p = colDataGetData(res->columnData, i); - colDataAppend(col, i, p, false); - } - } + + colDataAssign(col, res->columnData, res->numOfRows); + block->info.rows = res->numOfRows; sclFreeParam(res); taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 02c6006f60d08e2aab5361948b36817d99f446ed..3fa4797273dafd1223cc5973af2a250fe549d0f4 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -4,6 +4,7 @@ #include "ttime.h" #include "sclInt.h" #include "sclvector.h" +#include "tjson.h" typedef float (*_float_fn)(float); typedef double (*_double_fn)(double); @@ -304,7 +305,7 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP continue; } - char *in = pInputData->pData + pInputData->varmeta.offset[i]; + char *in = colDataGetData(pInputData, i); out[i] = lenFn(in, type); } @@ -395,11 +396,8 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int16_t dataLen = 0; for (int32_t i = 0; i < inputNum; ++i) { - if (pInput[i].numOfRows == 1) { - input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[0]; - } else { - input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[k]; - } + int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k; + input[i] = colDataGetData(pInputData[i], rowIdx); ret = concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen); if (ret != TSDB_CODE_SUCCESS) { @@ -473,11 +471,8 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p break; } - if (pInput[i].numOfRows == 1) { - input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[0]; - } else { - input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[k]; - } + int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k; + input[i] = colDataGetData(pInputData[i], rowIdx); ret = concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen); if (ret != TSDB_CODE_SUCCESS) { @@ -534,7 +529,7 @@ static int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScala continue; } - char *input = pInputData->pData + pInputData->varmeta.offset[i]; + char *input = colDataGetData(pInput[0].columnData, i); int32_t len = varDataLen(input); if (type == TSDB_DATA_TYPE_VARCHAR) { for (int32_t j = 0; j < len; ++j) { @@ -575,8 +570,8 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar colDataAppendNULL(pOutputData, i); continue; } - char *input = pInputData->pData + pInputData->varmeta.offset[i]; + char *input = colDataGetData(pInput[0].columnData, i); int32_t len = varDataLen(input); int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE; trimFn(input, output, type, charLen); @@ -593,10 +588,6 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar } int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - if (inputNum != 2 && inputNum!= 3) { - return TSDB_CODE_FAILED; - } - int32_t subPos = 0; GET_TYPED_DATA(subPos, int32_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); if (subPos == 0) { //subPos needs to be positive or negative values; @@ -606,7 +597,7 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t subLen = INT16_MAX; if (inputNum == 3) { GET_TYPED_DATA(subLen, int32_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); - if (subLen < 0) { //subLen cannot be negative + if (subLen < 0 || subLen > INT16_MAX) { //subLen cannot be negative return TSDB_CODE_FAILED; } subLen = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? subLen : subLen * TSDB_NCHAR_SIZE; @@ -615,38 +606,34 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - char *input = pInputData->pData + pInputData->varmeta.offset[0]; - char *output = NULL; - int32_t outputLen = pInputData->varmeta.length * pInput->numOfRows; char *outputBuf = taosMemoryCalloc(outputLen, 1); - output = outputBuf; + char *output = outputBuf; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { colDataAppendNULL(pOutputData, i); continue; } - + char *input = colDataGetData(pInput[0].columnData, i); int32_t len = varDataLen(input); int32_t startPosBytes; if (subPos > 0) { startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? subPos - 1 : (subPos - 1) * TSDB_NCHAR_SIZE; - startPosBytes = MIN(startPosBytes, len); + startPosBytes = TMIN(startPosBytes, len); } else { startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? len + subPos : len + subPos * TSDB_NCHAR_SIZE; - startPosBytes = MAX(startPosBytes, 0); + startPosBytes = TMAX(startPosBytes, 0); } - int32_t resLen = MIN(subLen, len - startPosBytes); + int32_t resLen = TMIN(subLen, len - startPosBytes); if (resLen > 0) { memcpy(varDataVal(output), varDataVal(input) + startPosBytes, resLen); } varDataSetLen(output, resLen); colDataAppend(pOutputData, i , output, false); - input += varDataTLen(input); output += varDataTLen(output); } @@ -659,27 +646,22 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int16_t inputType = pInput[0].columnData->info.type; int16_t outputType = pOutput[0].columnData->info.type; - if (outputType != TSDB_DATA_TYPE_BIGINT && outputType != TSDB_DATA_TYPE_UBIGINT && - outputType != TSDB_DATA_TYPE_VARCHAR && outputType != TSDB_DATA_TYPE_NCHAR && - outputType != TSDB_DATA_TYPE_TIMESTAMP) { - return TSDB_CODE_FAILED; - } int64_t outputLen = pOutput[0].columnData->info.bytes; - char *input = NULL; + if (IS_VAR_DATA_TYPE(outputType)) { + int32_t factor = (TSDB_DATA_TYPE_NCHAR == outputType) ? TSDB_NCHAR_SIZE : 1; + outputLen = outputLen * factor + VARSTR_HEADER_SIZE; + } + char *outputBuf = taosMemoryCalloc(outputLen * pInput[0].numOfRows, 1); char *output = outputBuf; - if (IS_VAR_DATA_TYPE(inputType)) { - input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0]; - } else { - input = pInput[0].columnData->pData; - } for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { colDataAppendNULL(pOutput->columnData, i); continue; } + char *input = colDataGetData(pInput[0].columnData, i); switch(outputType) { case TSDB_DATA_TYPE_BIGINT: { @@ -734,9 +716,10 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t len = sprintf(varDataVal(output), "%.*s", (int32_t)(outputLen - VARSTR_HEADER_SIZE), *(int8_t *)input ? "true" : "false"); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_BINARY) { - int32_t len = sprintf(varDataVal(output), "%.*s", (int32_t)(outputLen - VARSTR_HEADER_SIZE), varDataVal(input)); + int32_t len = TMIN(varDataLen(input), outputLen - VARSTR_HEADER_SIZE); + len = sprintf(varDataVal(output), "%.*s", len, varDataVal(input)); varDataSetLen(output, len); - } else if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { //not support return TSDB_CODE_FAILED; } else { @@ -767,7 +750,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = MIN(outputLen, varDataLen(input) + VARSTR_HEADER_SIZE); + int32_t len = TMIN(outputLen, varDataLen(input) + VARSTR_HEADER_SIZE); memcpy(output, input, len); varDataSetLen(output, len - VARSTR_HEADER_SIZE); } else { @@ -789,11 +772,6 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } colDataAppend(pOutput->columnData, i, output, false); - if (IS_VAR_DATA_TYPE(inputType)) { - input += varDataTLen(input); - } else { - input += tDataTypes[inputType].bytes; - } if (IS_VAR_DATA_TYPE(outputType)) { output += varDataTLen(output); } else { @@ -809,13 +787,13 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(pInput); - char *input = pInput[0].columnData->pData; for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { colDataAppendNULL(pOutput->columnData, i); continue; } + char *input = colDataGetData(pInput[0].columnData, i); char fraction[20] = {0}; bool hasFraction = false; NUM_TO_STRING(type, input, sizeof(fraction), fraction); @@ -832,7 +810,8 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { timeVal = timeVal / (1000 * 1000 * 1000); } else { - assert(0); + colDataAppendNULL(pOutput->columnData, i); + continue; } hasFraction = true; memmove(fraction, fraction + TSDB_TIME_PRECISION_SEC_DIGITS, TSDB_TIME_PRECISION_SEC_DIGITS); @@ -862,7 +841,6 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * varDataSetLen(buf, len); colDataAppend(pOutput->columnData, i, buf, false); - input += tDataTypes[type].bytes; } pOutput->numOfRows = pInput->numOfRows; @@ -874,12 +852,12 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP int32_t type = GET_PARAM_TYPE(pInput); int32_t timePrec = GET_PARAM_PRECISON(pInput); - char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0]; for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { colDataAppendNULL(pOutput->columnData, i); continue; } + char *input = colDataGetData(pInput[0].columnData, i); int64_t timeVal = 0; int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal); @@ -889,8 +867,60 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP } colDataAppend(pOutput->columnData, i, (char *)&timeVal, false); + } + + pOutput->numOfRows = pInput->numOfRows; + + return TSDB_CODE_SUCCESS; +} + +int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + int32_t type = GET_PARAM_TYPE(pInput); + if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) { + return TSDB_CODE_FAILED; + } + + if (inputNum != 1) { + return TSDB_CODE_FAILED; + } + + char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0]; + char *tmp = taosMemoryCalloc(pInput[0].columnData->info.bytes + 1, 1); + for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { + if (colDataIsNull_s(pInput[0].columnData, i)) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } + + if(type == TSDB_DATA_TYPE_NCHAR){ + if (varDataTLen(input) > TSDB_MAX_JSON_TAG_LEN){ + colDataAppendNULL(pOutput->columnData, i); + continue; + } + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), tmp); + if (len < 0) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } + tmp[len] = 0; + }else{ + if (varDataLen(input) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + colDataAppendNULL(pOutput->columnData, i); + continue; + } + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataTLen(input)] = 0; + } + + if(!tjsonValidateJson(tmp)){ + colDataAppendNULL(pOutput->columnData, i); + continue; + } + + colDataAppend(pOutput->columnData, i, input, false); input += varDataTLen(input); } + taosMemoryFree(tmp); pOutput->numOfRows = pInput->numOfRows; @@ -907,19 +937,14 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); - char *input = NULL; - if (IS_VAR_DATA_TYPE(type)) { - input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0]; - } else { - input = pInput[0].columnData->pData; - } - for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { colDataAppendNULL(pOutput->columnData, i); continue; } + char *input = colDataGetData(pInput[0].columnData, i); + if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ int32_t ret = convertStringToTimestamp(type, input, TSDB_TIME_PRECISION_NANO, &timeVal); if (ret != TSDB_CODE_SUCCESS) { @@ -969,7 +994,8 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS){ timeVal = timeVal * factor; } else { - assert(0); + colDataAppendNULL(pOutput->columnData, i); + continue; } break; } @@ -983,7 +1009,8 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { timeVal = timeVal * factor; } else { - assert(0); + colDataAppendNULL(pOutput->columnData, i); + continue; } break; } @@ -997,7 +1024,8 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { timeVal = timeVal * factor / factor / 60 * 60 * factor; } else { - assert(0); + colDataAppendNULL(pOutput->columnData, i); + continue; } break; } @@ -1011,7 +1039,8 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { timeVal = timeVal * factor / factor / 3600 * 3600 * factor; } else { - assert(0); + colDataAppendNULL(pOutput->columnData, i); + continue; } break; } @@ -1025,7 +1054,8 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { timeVal = timeVal * factor / factor / 86400* 86400 * factor; } else { - assert(0); + colDataAppendNULL(pOutput->columnData, i); + continue; } break; } @@ -1039,7 +1069,8 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { timeVal = timeVal * factor / factor / 604800 * 604800* factor; } else { - assert(0); + colDataAppendNULL(pOutput->columnData, i); + continue; } break; } @@ -1078,11 +1109,6 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } colDataAppend(pOutput->columnData, i, (char *)&timeVal, false); - if (IS_VAR_DATA_TYPE(type)) { - input += varDataTLen(input); - } else { - input += tDataTypes[type].bytes; - } } pOutput->numOfRows = pInput->numOfRows; @@ -1097,28 +1123,25 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); } - char *input[2]; - for (int32_t k = 0; k < 2; ++k) { - int32_t type = GET_PARAM_TYPE(&pInput[k]); - if (type != TSDB_DATA_TYPE_BIGINT && type != TSDB_DATA_TYPE_TIMESTAMP && - type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) { - return TSDB_CODE_FAILED; - } - - if (IS_VAR_DATA_TYPE(type)) { - input[k] = pInput[k].columnData->pData + pInput[k].columnData->varmeta.offset[0]; - } else { - input[k] = pInput[k].columnData->pData; + int32_t numOfRows = 0; + for (int32_t i = 0; i < inputNum; ++i) { + if (pInput[i].numOfRows > numOfRows) { + numOfRows = pInput[i].numOfRows; } } - for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { + char *input[2]; + for (int32_t i = 0; i < numOfRows; ++i) { + bool hasNull = false; for (int32_t k = 0; k < 2; ++k) { - if (colDataIsNull_s(pInput[0].columnData, i)) { - colDataAppendNULL(pOutput->columnData, i); - continue; + if (colDataIsNull_s(pInput[k].columnData, i)) { + hasNull = true; + break; } + int32_t rowIdx = (pInput[k].numOfRows == 1) ? 0 : i; + input[k] = colDataGetData(pInput[k].columnData, rowIdx); + int32_t type = GET_PARAM_TYPE(&pInput[k]); if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ int32_t ret = convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k]); @@ -1148,18 +1171,18 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p timeVal[k] = timeVal[k] * 1000; } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { timeVal[k] = timeVal[k]; - } - } - - if (pInput[k].numOfRows != 1) { - if (IS_VAR_DATA_TYPE(type)) { - input[k] += varDataTLen(input[k]); } else { - input[k] += tDataTypes[type].bytes; + colDataAppendNULL(pOutput->columnData, i); + continue; } } } + if (hasNull) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } + int64_t result = (timeVal[0] >= timeVal[1]) ? (timeVal[0] - timeVal[1]) : (timeVal[1] - timeVal[0]); @@ -1220,7 +1243,7 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p colDataAppend(pOutput->columnData, i, (char *)&result, false); } - pOutput->numOfRows = pInput->numOfRows; + pOutput->numOfRows = numOfRows; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 0131a94fec7f98c9038fe1c61e3f00e0047a48a7..d484738328554cc5d830f5e31d0332b485c5a9f1 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -25,6 +25,78 @@ #include "tdatablock.h" #include "ttypes.h" +#define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData)) +#define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData)) + +void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){ + switch (outType) { + case TSDB_DATA_TYPE_BOOL: { + GET_TYPED_DATA(*((bool *)outData), bool, inType, inData); + break; + } + case TSDB_DATA_TYPE_TINYINT: { + GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_INT: { + GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_UINT: { + GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + GET_TYPED_DATA(*((float *)outData), float, inType, inData); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + GET_TYPED_DATA(*((double *)outData), double, inType, inData); + break; + } + default:{ + ASSERT(0); + } + } +} + +void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){ + char *tmp = taosMemoryMalloc(varDataTLen(inData)); + int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp); + if (len < 0) { + sclError("castConvert taosUcs4ToMbs error 1"); + } + + tmp[len] = 0; + + ASSERT(outType == TSDB_DATA_TYPE_DOUBLE); + double value = strtod(tmp, NULL); + + *((double *)outData) = value; + taosMemoryFreeClear(tmp); +} + typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index); int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) { @@ -61,6 +133,20 @@ int64_t getVectorBigintValue_BOOL(void *src, int32_t index) { return (int64_t)*((bool *)src + index); } +int64_t getVectorBigintValue_JSON(void *src, int32_t index){ + ASSERT(!colDataIsNull_var(((SColumnInfoData*)src), index)); + char *data = colDataGetVarData((SColumnInfoData*)src, index); + double out = 0; + if (*data == TSDB_DATA_TYPE_NULL){ + return 0; + } else if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY + convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); + } else { + convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); + } + return (int64_t)out; +} + _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) { _getBigintValue_fn_t p = NULL; if(srcType==TSDB_DATA_TYPE_TINYINT) { @@ -87,6 +173,8 @@ _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) { p = getVectorBigintValue_BIGINT; }else if(srcType==TSDB_DATA_TYPE_BOOL) { p = getVectorBigintValue_BOOL; + }else if(srcType==TSDB_DATA_TYPE_JSON) { + p = getVectorBigintValue_JSON; }else { assert(0); } @@ -213,7 +301,8 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in func = varToUnsigned; } else if (IS_FLOAT_TYPE(outType)) { func = varToFloat; - } else if (outType == TSDB_DATA_TYPE_NCHAR) { + } else if (outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar + ASSERT(inType == TSDB_DATA_TYPE_VARCHAR); func = varToNchar; vton = true; } else { @@ -228,14 +317,28 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in continue; } - char* data = colDataGetData(pIn->columnData, i); + char* data = colDataGetVarData(pIn->columnData, i); + int32_t convertType = inType; + if(inType == TSDB_DATA_TYPE_JSON){ + if(*data == TSDB_DATA_TYPE_NULL) { + colDataAppendNULL(pOut->columnData, i); + continue; + } + else if(*data == TSDB_DATA_TYPE_NCHAR) { + data += CHAR_BYTES; + convertType = TSDB_DATA_TYPE_NCHAR; + } else { + convertNumberToNumber(data+CHAR_BYTES, colDataGetNumData(pOut->columnData, i), *data, outType); + continue; + } + } if (vton) { memcpy(tmp, data, varDataTLen(data)); } else { - if (TSDB_DATA_TYPE_VARCHAR == inType) { + if (TSDB_DATA_TYPE_VARCHAR == convertType) { memcpy(tmp, varDataVal(data), varDataLen(data)); tmp[varDataLen(data)] = 0; - } else { + } else if (TSDB_DATA_TYPE_NCHAR == convertType){ ASSERT(varDataLen(data) <= bufSize); int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp); @@ -256,6 +359,66 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in return TSDB_CODE_SUCCESS; } +double getVectorDoubleValue_JSON(void *src, int32_t index){ + char *data = colDataGetVarData((SColumnInfoData*)src, index); + double out = 0; + if (*data == TSDB_DATA_TYPE_NULL){ + return out; + } else if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY + convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); + } else { + convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); + } + return out; +} + +void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){ + if(optr == OP_TYPE_JSON_CONTAINS) { + return; + } + + if(typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON){ + return; + } + + if(typeLeft == TSDB_DATA_TYPE_JSON){ + typeLeft = **pLeftData; + (*pLeftData) ++; + } + if(typeRight == TSDB_DATA_TYPE_JSON){ + typeRight = **pRightData; + (*pRightData) ++; + } + if(typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL){ + *isNull = true; + return; + } + int8_t type = vectorGetConvertType(typeLeft, typeRight); + + if(type == 0) { + *fp = filterGetCompFunc(typeLeft, optr); + return; + } + + *fp = filterGetCompFunc(type, optr); + + if(typeLeft == TSDB_DATA_TYPE_NCHAR) { + convertStringToDouble(*pLeftData, pLeftOut, typeLeft, type); + *pLeftData = pLeftOut; + } else if(typeLeft != type) { + convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type); + *pLeftData = pLeftOut; + } + + if(typeRight == TSDB_DATA_TYPE_NCHAR) { + convertStringToDouble(*pRightData, pRightOut, typeRight, type); + *pRightData = pRightOut; + } else if(typeRight != type) { + convertNumberToNumber(*pRightData, pRightOut, typeRight, type); + *pRightData = pRightOut; + } +} + // TODO opt performance int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { SColumnInfoData* pInputCol = pIn->columnData; @@ -422,24 +585,24 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { } int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { -/* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG VARB JSON DECI BLOB */ +/* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB */ /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/*BOOL*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 0, 12, 13, 14, 7, 0, 0, 0, -/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 0, 0, 0, -/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 0, 0, 0, -/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 7, 0, 0, 0, -/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 7, 0, 0, 0, -/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 7, 0, 0, 0, -/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 7, 0, 0, 0, +/*BOOL*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 0, 12, 13, 14, 0, 7, 0, 0, +/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, +/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, +/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 0, 7, 0, 0, +/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 0, 7, 0, 0, +/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, +/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 7, 7, 7, 0, 0, 0, 0, -/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 9, 7, 7, 0, 0, 0, +/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 9, 7, 0, 7, 0, 0, /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, -/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 7, 0, 0, 0, -/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 7, 0, 0, 0, -/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 7, 0, 0, 0, -/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, -/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, +/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0, +/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 0, +/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; @@ -530,7 +693,7 @@ enum { static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SScalarParam* pParam, int32_t type) { SColumnInfoData* pCol = pParam->columnData; - if (IS_VAR_DATA_TYPE(pCol->info.type)) { + if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) { pDest->numOfRows = pParam->numOfRows; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; @@ -564,11 +727,12 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, 0); - } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + + getVectorDoubleValueFnRight(RIGHT_COL, 0); } } } @@ -583,12 +747,12 @@ static void vectorMathBigintAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, 0); } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); - } } } @@ -614,6 +778,50 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) { } } +char *getJsonValue(char *json, char *key){ //todo + int16_t cols = kvRowNCols(json); + for (int i = 0; i < cols; ++i) { + SColIdx *pColIdx = kvRowColIdxAt(json, i); + char *data = kvRowColVal(json, pColIdx); + if(i == 0){ + if(*data == TSDB_DATA_TYPE_NULL) { + return NULL; + } + continue; + } + if(memcmp(key, data, varDataTLen(data)) == 0){ + return data + varDataTLen(data); + } + } + return NULL; +} + +void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + SColumnInfoData *pOutputCol = pOut->columnData; + + int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1; + + pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); + + char *pRightData = colDataGetVarData(pRight->columnData, 0); + for (; i >= 0 && i < pLeft->numOfRows; i += step) { + if (colDataIsNull_var(pLeft->columnData, i)) { + colDataSetNull_var(pOutputCol, i); + pOutputCol->hasNull = true; + continue; + } + char *pLeftData = colDataGetVarData(pLeft->columnData, i); + char *value = getJsonValue(pLeftData, pRightData); + if (!value) { + colDataSetNull_var(pOutputCol, i); + pOutputCol->hasNull = true; + continue; + } + colDataAppend(pOutputCol, i, value, false); + } +} + void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { SColumnInfoData *pOutputCol = pOut->columnData; @@ -636,17 +844,12 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, i); - } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j]; + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore } + *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, i); } - } else if (pLeft->numOfRows == 1) { vectorMathBigintAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); } else if (pRight->numOfRows == 1) { @@ -659,17 +862,12 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, i); - } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j]; + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore } + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, i); } - } else if (pLeft->numOfRows == 1) { vectorMathAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); } else if (pRight->numOfRows == 1) { @@ -692,11 +890,12 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - *output = (getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, 0)) * factor; - } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = (getVectorDoubleValueFnLeft(LEFT_COL, i) + - getVectorDoubleValueFnRight(RIGHT_COL, 0)) * factor; } } } @@ -711,12 +910,12 @@ static void vectorMathBigintSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } *output = (getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, 0)) * factor; } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); - } } } @@ -740,17 +939,12 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, i); - } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j]; + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore } + *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, i); } - } else if (pLeft->numOfRows == 1) { vectorMathBigintSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i); } else if (pRight->numOfRows == 1) { @@ -763,17 +957,12 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, i); - } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j]; + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore } + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, i); } - } else if (pLeft->numOfRows == 1) { vectorMathSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i); } else if (pRight->numOfRows == 1) { @@ -796,11 +985,12 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData* colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, 0); - } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + * getVectorDoubleValueFnRight(RIGHT_COL, 0); } } } @@ -822,17 +1012,13 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam double *output = (double *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, i); - } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j]; + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore } + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + * getVectorDoubleValueFnRight(RIGHT_COL, i); } - } else if (pLeft->numOfRows == 1) { vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); } else if (pRight->numOfRows == 1) { @@ -860,39 +1046,37 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p double *output = (double *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { // check for the 0 value for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, i); - } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j]; + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore } + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + /getVectorDoubleValueFnRight(RIGHT_COL, i); } - } else if (pLeft->numOfRows == 1) { - if (colDataIsNull_f(pLeftCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (colDataIsNull_s(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, 0) / getVectorDoubleValueFnRight(pRightCol->pData, i); - } - pOutputCol->hasNull = pRightCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pRightCol->nullbitmap, BitmapLen(pRight->numOfRows)); + if (colDataIsNull_s(pRightCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft(LEFT_COL, 0) + / getVectorDoubleValueFnRight(RIGHT_COL, i); } } } else if (pRight->numOfRows == 1) { - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, 0); - } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(pLeft->numOfRows)); + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + / getVectorDoubleValueFnRight(RIGHT_COL, 0); } } } @@ -920,13 +1104,13 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_f(pLeftCol->nullbitmap, i) || colDataIsNull_f(pRightCol->nullbitmap, i)) { + if (colDataIsNull_s(pLeftCol, i) || colDataIsNull_s(pRightCol, i)) { colDataAppendNULL(pOutputCol, i); continue; } - double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i); - double rx = getVectorDoubleValueFnRight(pRightCol->pData, i); + double lx = getVectorDoubleValueFnLeft(LEFT_COL, i); + double rx = getVectorDoubleValueFnRight(RIGHT_COL, i); if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) { colDataAppendNULL(pOutputCol, i); continue; @@ -935,17 +1119,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *output = lx - ((int64_t)(lx / rx)) * rx; } } else if (pLeft->numOfRows == 1) { - double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, 0); - if (colDataIsNull_f(pLeftCol->nullbitmap, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value + double lx = getVectorDoubleValueFnLeft(LEFT_COL, 0); + if (colDataIsNull_s(pLeftCol, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { - if (colDataIsNull_f(pRightCol->nullbitmap, i)) { + if (colDataIsNull_s(pRightCol, i)) { colDataAppendNULL(pOutputCol, i); continue; } - double rx = getVectorDoubleValueFnRight(pRightCol->pData, i); + double rx = getVectorDoubleValueFnRight(RIGHT_COL, i); if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { colDataAppendNULL(pOutputCol, i); continue; @@ -955,17 +1139,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } } } else if (pRight->numOfRows == 1) { - double rx = getVectorDoubleValueFnRight(pRightCol->pData, 0); - if (colDataIsNull_f(pRightCol->nullbitmap, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value + double rx = getVectorDoubleValueFnRight(RIGHT_COL, 0); + if (colDataIsNull_s(pRightCol, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { - if (colDataIsNull_f(pLeftCol->nullbitmap, i)) { + if (colDataIsNull_s(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; } - double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i); + double lx = getVectorDoubleValueFnLeft(LEFT_COL, i); if (isnan(lx) || isinf(lx)) { colDataAppendNULL(pOutputCol, i); continue; @@ -995,12 +1179,11 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO double *output = (double *)pOutputCol->pData; for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) { - *output = - getVectorDoubleValueFnLeft(pLeftCol->pData, i); - } - - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(pLeft->numOfRows)); + if (colDataIsNull_s(pLeft->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = - getVectorDoubleValueFnLeft(LEFT_COL, i); } doReleaseVec(pLeftCol, leftConvert); @@ -1063,15 +1246,15 @@ static void vectorBitAndHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRigh double *output = (double *)pOutputCol->pData; - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) & getVectorBigintValueFnRight(pRightCol->pData, 0); - } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, 0); } } } @@ -1093,17 +1276,12 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int64_t *output = (int64_t *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) & getVectorBigintValueFnRight(pRightCol->pData, i); - } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] & pRightCol->nullbitmap[j]; + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore } + *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, i); } - } else if (pLeft->numOfRows == 1) { vectorBitAndHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); } else if (pRight->numOfRows == 1) { @@ -1120,16 +1298,16 @@ static void vectorBitOrHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRight int64_t *output = (int64_t *)pOutputCol->pData; - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { - int64_t rx = getVectorBigintValueFnRight(pRightCol->pData, 0); + int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, 0); for (; i >= 0 && i < numOfRows; i += step, output += 1) { - *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) | rx; - } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorBigintValueFnLeft(LEFT_COL, i) | rx; } } } @@ -1151,15 +1329,11 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int64_t *output = (int64_t *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) | getVectorBigintValueFnRight(pRightCol->pData, i); - } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j]; + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore } + *output = getVectorBigintValueFnLeft(LEFT_COL, i) | getVectorBigintValueFnRight(RIGHT_COL, i); } } else if (pLeft->numOfRows == 1) { vectorBitOrHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); @@ -1181,6 +1355,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * if (pRight->pHashFilter != NULL) { for (; i >= 0 && i < pLeft->numOfRows; i += step) { if (colDataIsNull_s(pLeft->columnData, i)) { + colDataAppendNULL(pOut->columnData, i); continue; } @@ -1194,35 +1369,62 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step) { if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOut->columnData, i); continue; // TODO set null or ignore } char *pLeftData = colDataGetData(pLeft->columnData, i); char *pRightData = colDataGetData(pRight->columnData, i); + + int64_t leftOut = 0; + int64_t rightOut = 0; + bool isJsonnull = false; + convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); + if(isJsonnull){ + colDataAppendNULL(pOut->columnData, i); + continue; // TODO set null or ignore + } bool res = filterDoCompare(fp, optr, pLeftData, pRightData); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } } else if (pRight->numOfRows == 1) { - char *pRightData = colDataGetData(pRight->columnData, 0); ASSERT(pLeft->pHashFilter == NULL); - for (; i >= 0 && i < pLeft->numOfRows; i += step) { if (colDataIsNull_s(pLeft->columnData, i)) { + colDataAppendNULL(pOut->columnData, i); continue; } char *pLeftData = colDataGetData(pLeft->columnData, i); + char *pRightData = colDataGetData(pRight->columnData, 0); + int64_t leftOut = 0; + int64_t rightOut = 0; + bool isJsonnull = false; + convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); + if(isJsonnull){ + colDataAppendNULL(pOut->columnData, i); + continue; // TODO set null or ignore + } bool res = filterDoCompare(fp, optr, pLeftData, pRightData); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } } else if (pLeft->numOfRows == 1) { - char *pLeftData = colDataGetData(pLeft->columnData, 0); for (; i >= 0 && i < pRight->numOfRows; i += step) { if (colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOut->columnData, i); continue; } + char *pLeftData = colDataGetData(pLeft->columnData, 0); char *pRightData = colDataGetData(pLeft->columnData, i); + int64_t leftOut = 0; + int64_t rightOut = 0; + bool isJsonnull = false; + convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); + if(isJsonnull){ + colDataAppendNULL(pOut->columnData, i); + continue; // TODO set null or ignore + } bool res = filterDoCompare(fp, optr, pLeftData, pRightData); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } @@ -1303,6 +1505,10 @@ void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH); } +void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_JSON_CONTAINS); +} + void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0; @@ -1371,8 +1577,13 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { return vectorBitOr; case OP_TYPE_IS_TRUE: return vectorIsTrue; + case OP_TYPE_JSON_GET_VALUE: + return vectorJsonArrow; + case OP_TYPE_JSON_CONTAINS: + return vectorJsonContains; default: assert(0); return NULL; } } + diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 42998aba003217dcfa42dff2fe3203a299529425..59c3104e96c0320804ba4f17dd0a013146b27a2d 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -27,6 +27,9 @@ #pragma GCC diagnostic ignored "-Wpointer-arith" #include +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" @@ -242,7 +245,7 @@ TEST(timerangeTest, greater) { int32_t code = filterGetTimeRange(opNode1, &win, &isStrict); ASSERT_EQ(code, 0); ASSERT_EQ(isStrict, true); - ASSERT_EQ(win.skey, tsmall); + ASSERT_EQ(win.skey, tsmall+1); ASSERT_EQ(win.ekey, INT64_MAX); //filterFreeInfo(filter); nodesDestroyNode(opNode1); @@ -265,6 +268,37 @@ TEST(timerangeTest, greater_and_lower) { flttMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + //SFilterInfo *filter = NULL; + //int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); + //ASSERT_EQ(code, 0); + STimeWindow win = {0}; + bool isStrict = false; + int32_t code = filterGetTimeRange(logicNode, &win, &isStrict); + ASSERT_EQ(isStrict, true); + ASSERT_EQ(code, 0); + ASSERT_EQ(win.skey, tsmall+1); + ASSERT_EQ(win.ekey, tbig-1); + //filterFreeInfo(filter); + nodesDestroyNode(logicNode); +} + +TEST(timerangeTest, greater_equal_and_lower_equal) { + SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; + bool eRes[5] = {false, false, true, true, true}; + SScalarParam res = {0}; + int64_t tsmall = 222, tbig = 333; + flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pcol, pval); + flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pcol, pval); + SNode *list[2] = {0}; + list[0] = opNode1; + list[1] = opNode2; + + flttMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + //SFilterInfo *filter = NULL; //int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); //ASSERT_EQ(code, 0); @@ -279,6 +313,7 @@ TEST(timerangeTest, greater_and_lower) { nodesDestroyNode(logicNode); } + TEST(timerangeTest, greater_and_lower_not_strict) { SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode1 = NULL, *logicNode2 = NULL; bool eRes[5] = {false, false, true, true, true}; @@ -321,8 +356,8 @@ TEST(timerangeTest, greater_and_lower_not_strict) { int32_t code = filterGetTimeRange(logicNode1, &win, &isStrict); ASSERT_EQ(isStrict, false); ASSERT_EQ(code, 0); - ASSERT_EQ(win.skey, tsmall1); - ASSERT_EQ(win.ekey, tbig2); + ASSERT_EQ(win.skey, tsmall1+1); + ASSERT_EQ(win.ekey, tbig2-1); //filterFreeInfo(filter); nodesDestroyNode(logicNode1); } @@ -364,7 +399,7 @@ TEST(columnTest, smallint_column_greater_double_value) { keep = filterRangeExecute(filter, &stat, 1, rowNum); ASSERT_EQ(keep, true); - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -419,7 +454,7 @@ TEST(columnTest, int_column_greater_smallint_value) { keep = filterRangeExecute(filter, &stat, 1, rowNum); ASSERT_EQ(keep, false); - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -463,7 +498,7 @@ TEST(columnTest, int_column_in_double_list) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -527,7 +562,7 @@ TEST(columnTest, binary_column_in_binary_list) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -575,7 +610,7 @@ TEST(columnTest, binary_column_like_binary) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -622,7 +657,7 @@ TEST(columnTest, binary_column_is_null) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -669,7 +704,7 @@ TEST(columnTest, binary_column_is_not_null) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -706,7 +741,7 @@ TEST(opTest, smallint_column_greater_int_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -744,7 +779,7 @@ TEST(opTest, smallint_value_add_int_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -787,7 +822,7 @@ TEST(opTest, bigint_column_multi_binary_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -830,7 +865,7 @@ TEST(opTest, smallint_column_and_binary_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -868,7 +903,7 @@ TEST(opTest, smallint_column_or_float_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -906,7 +941,7 @@ TEST(opTest, smallint_column_or_double_value) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -950,7 +985,7 @@ TEST(opTest, binary_column_is_true) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1021,7 +1056,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1090,7 +1125,7 @@ TEST(filterModelogicTest, same_column_and_or_and) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1159,7 +1194,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1228,7 +1263,7 @@ TEST(filterModelogicTest, same_column_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1299,7 +1334,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); diff --git a/source/libs/scalar/test/scalar/CMakeLists.txt b/source/libs/scalar/test/scalar/CMakeLists.txt index 6cbac1e25d7e5f7be1931129a82c143047555029..03418d17ac2a952998081053a1789c5ac2102145 100644 --- a/source/libs/scalar/test/scalar/CMakeLists.txt +++ b/source/libs/scalar/test/scalar/CMakeLists.txt @@ -8,11 +8,12 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(scalarTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( scalarTest - PUBLIC os util common gtest qcom function nodes scalar + PUBLIC os util common gtest qcom function nodes scalar parser ) TARGET_INCLUDE_DIRECTORIES( scalarTest PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/" + PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc" PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc" ) diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 09d6528dd80a2a5000aad4b5fb94671c71c2f2dd..f6140a75d3f9c5493cbb631f3664b1d9f530001d 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -27,6 +27,9 @@ #pragma GCC diagnostic ignored "-Wpointer-arith" #include +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" @@ -38,6 +41,7 @@ #include "scalar.h" #include "nodes.h" #include "tlog.h" +#include "parUtil.h" #define _DEBUG_PRINT_ 0 @@ -92,6 +96,7 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s blockDataEnsureCapacity(res, rows); *dataBlockId = taosArrayGetSize(pBlockList) - 1; + res->info.blockId = *dataBlockId; *slotId = 0; } else { SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); @@ -909,6 +914,264 @@ TEST(constantTest, greater_and_lower) { nodesDestroyNode(res); } +void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){ + char keyVar[32] = {0}; + memcpy(varDataVal(keyVar), key, strlen(key)); + varDataLen(keyVar) = strlen(key); + + SNode *pLeft = NULL, *pRight = NULL; + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, keyVar); + scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, varDataLen(json), 1, json); + scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight); +} + +void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData){ + int32_t resType = TSDB_DATA_TYPE_NULL; + if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || + opType == OP_TYPE_DIV || opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){ + resType = TSDB_DATA_TYPE_DOUBLE; + }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){ + resType = TSDB_DATA_TYPE_BIGINT; + }else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || + opType == OP_TYPE_LOWER_THAN || opType == OP_TYPE_LOWER_EQUAL || + opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL || + opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE || + opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || + opType == OP_TYPE_NMATCH){ + resType = TSDB_DATA_TYPE_BOOL; + } + + SNode *right = NULL; + scltMakeValueNode(&right, rightType, rightData); + scltMakeOpNode(opNode, opType, resType, *opNode, right); + + SColumnInfo colInfo = createColumnInfo(1, resType, tDataTypes[resType].bytes); + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, 1, &colInfo); + scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode); +} + +void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, double exceptValue, EOperatorType opType){ + SArray *blockList = taosArrayInit(2, POINTER_BYTES); + SSDataBlock *src = NULL; + SNode *opNode = NULL; + + makeJsonArrow(&src, &opNode, json, (char*)key); + taosArrayPush(blockList, &src); + + makeOperator(&opNode, blockList, opType, rightType, rightData); + + int32_t code = scalarCalculate(opNode, blockList, NULL); + ASSERT_EQ(code, 0); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, 1); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + + if(colDataIsNull_f(column->nullbitmap, 0)){ + ASSERT_EQ(DBL_MAX, exceptValue); + printf("result:NULL\n"); + + }else if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV || + opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){ + double tmp = *((double *)colDataGetData(column, 0)); + ASSERT_TRUE(tmp == exceptValue); + printf("result:%lf\n", tmp); + }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){ + ASSERT_EQ(*((int64_t *)colDataGetData(column, 0)), exceptValue); + printf("result:%ld\n", *((int64_t *)colDataGetData(column, 0))); + }else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || opType == OP_TYPE_LOWER_THAN || + opType == OP_TYPE_LOWER_EQUAL || opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL || + opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE || + opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || opType == OP_TYPE_NMATCH){ + ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue); + printf("result:%d\n", *((bool *)colDataGetData(column, 0))); + } + + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); +} + +TEST(columnTest, json_column_arith_op) { + scltInitLogFile(); + char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}"; + + SKVRowBuilder kvRowBuilder; + tdInitKVRowBuilder(&kvRowBuilder); + parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0); + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); + + const int32_t len = 8; + EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV, + OP_TYPE_MOD, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR}; + int32_t input[len] = {1, 8, 2, 2, 3, 0, -4, 9}; + + printf("--------------------json int---------------------\n"); + char *key = "k1"; + double eRes[len] = {5.0, -4, 8.0, 2.0, 1.0, -4, 4&-4, 4|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]); + } + + printf("--------------------json string---------------------\n"); + + key = "k2"; + double eRes1[len] = {1.0, -8, 0, 0, 0, 0, 0, 9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]); + } + + printf("---------------------json null--------------------\n"); + + key = "k3"; + double eRes2[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]); + } + + printf("---------------------json bool--------------------\n"); + + key = "k4"; + double eRes3[len] = {2.0, -7, 2, 0.5, 1, -1, 1&-4, 1|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]); + } + + printf("----------------------json double-------------------\n"); + + key = "k5"; + double eRes4[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]); + } + + printf("---------------------json not exist--------------------\n"); + + key = "k10"; + double eRes5[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]); + } +} + +void *prepareNchar(char* rightData){ + int32_t len = 0; + int32_t inputLen = strlen(rightData); + + char* t = (char*)taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + taosMbsToUcs4(rightData, inputLen, (TdUcs4*) varDataVal(t), inputLen * TSDB_NCHAR_SIZE, &len); + varDataSetLen(t, len); + return t; +} + +TEST(columnTest, json_column_logic_op) { + scltInitLogFile(); + char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}"; + + SKVRowBuilder kvRowBuilder; + tdInitKVRowBuilder(&kvRowBuilder); + parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0); + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); + + const int32_t len = 9; + const int32_t len1 = 4; + EOperatorType op[len+len1] = {OP_TYPE_GREATER_THAN, OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, OP_TYPE_EQUAL, OP_TYPE_NOT_EQUAL, + OP_TYPE_IS_NULL, OP_TYPE_IS_NOT_NULL, OP_TYPE_IS_TRUE, OP_TYPE_LIKE, OP_TYPE_NOT_LIKE, OP_TYPE_MATCH, OP_TYPE_NMATCH}; + + int32_t input[len] = {1, 8, 2, 2, 3, 0, 0, 0, 0}; + char *inputNchar[len1] = {"hell_", "hel%", "hell", "llll"}; + + printf("--------------------json int---------------------\n"); + char *key = "k1"; + bool eRes[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]); + } + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i]); + taosMemoryFree(rightData); + } + + printf("--------------------json string---------------------\n"); + + key = "k2"; + bool eRes1[len+len1] = {false, false, true, true, false, false, false, true, false, true, false, true, true}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]); + } + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i]); + taosMemoryFree(rightData); + } + + printf("--------------------json null---------------------\n"); + + key = "k3"; + double eRes2[len+len1] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, true, false, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]); + } + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i]); + taosMemoryFree(rightData); + } + + printf("--------------------json bool---------------------\n"); + + key = "k4"; + bool eRes3[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]); + } + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i]); + taosMemoryFree(rightData); + } + + printf("--------------------json double---------------------\n"); + + key = "k5"; + bool eRes4[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]); + } + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i]); + taosMemoryFree(rightData); + } + + printf("--------------------json double---------------------\n"); + + key = "k6"; + bool eRes5[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, true}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]); + } + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i]); + taosMemoryFree(rightData); + } + + printf("---------------------json not exist--------------------\n"); + + key = "k10"; + double eRes10[len+len1] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, true, false, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i]); + } + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes10[i], op[i]); + taosMemoryFree(rightData); + } +} + TEST(columnTest, smallint_value_add_int_column) { scltInitLogFile(); @@ -928,7 +1191,7 @@ TEST(columnTest, smallint_value_add_int_column) { int16_t dataBlockId = 0, slotId = 0; scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); - + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); @@ -3097,3 +3360,4 @@ int main(int argc, char** argv) { } #pragma GCC diagnostic pop + diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index d1def1bef116ccca66647d173f41efabda85126d..0f6961018ce4cbbba9e4129bed2e42a8e3e0d64e 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -52,14 +52,26 @@ typedef struct SSchHbTrans { typedef struct SSchApiStat { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SSchApiStat; typedef struct SSchRuntimeStat { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SSchRuntimeStat; typedef struct SSchJobStat { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SSchJobStat; typedef struct SSchedulerStat { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 6cc435fee4b2be4dad426cd6c052c24919c8943c..c0b3ae7055d6b0d72784c40f7345476dc5126de7 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -2749,4 +2749,15 @@ void schedulerDestroy(void) { taosCloseRef(schMgmt.jobRef); schMgmt.jobRef = 0; } + + if (schMgmt.hbConnections) { + void *pIter = taosHashIterate(schMgmt.hbConnections, NULL); + while (pIter != NULL) { + SSchHbTrans *hb = pIter; + schFreeRpcCtx(&hb->rpcCtx); + pIter = taosHashIterate(schMgmt.hbConnections, pIter); + } + taosHashCleanup(schMgmt.hbConnections); + schMgmt.hbConnections = NULL; + } } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 5d9322a5753abe0a1ff9f15cd4e6f59183a3cc73..fc0e05aaf106fb11d8daa9be9a55e510aac58ff5 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -25,7 +25,9 @@ #pragma GCC diagnostic ignored "-Wformat" #include - +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" @@ -275,9 +277,16 @@ void schtSetPlanToString() { static Stub stub; stub.set(qSubPlanToString, schtPlanToString); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qSubPlanToString", result); +#endif +#ifdef LINUX AddrAny any("libplanner.so"); std::map result; any.get_global_func_addr_dynsym("^qSubPlanToString$", result); +#endif for (const auto& f : result) { stub.set(f.second, schtPlanToString); } @@ -288,9 +297,16 @@ void schtSetExecNode() { static Stub stub; stub.set(qSetSubplanExecutionNode, schtExecNode); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qSetSubplanExecutionNode", result); +#endif +#ifdef LINUX AddrAny any("libplanner.so"); std::map result; any.get_global_func_addr_dynsym("^qSetSubplanExecutionNode$", result); +#endif for (const auto& f : result) { stub.set(f.second, schtExecNode); } @@ -301,9 +317,16 @@ void schtSetRpcSendRequest() { static Stub stub; stub.set(rpcSendRequest, schtRpcSendRequest); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRequest", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRequest$", result); +#endif for (const auto& f : result) { stub.set(f.second, schtRpcSendRequest); } @@ -324,9 +347,16 @@ void schtSetAsyncSendMsgToServer() { static Stub stub; stub.set(asyncSendMsgToServer, schtAsyncSendMsgToServer); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("asyncSendMsgToServer", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^asyncSendMsgToServer$", result); +#endif for (const auto& f : result) { stub.set(f.second, schtAsyncSendMsgToServer); } @@ -382,6 +412,7 @@ void *schtCreateFetchRspThread(void *param) { schReleaseJob(job); assert(code == 0); + return NULL; } @@ -413,6 +444,7 @@ void *schtFetchRspThread(void *aa) { assert(code == 0 || code); } + return NULL; } void schtFreeQueryJob(int32_t freeThread) { @@ -595,6 +627,7 @@ void* schtRunJobThread(void *aa) { schedulerDestroy(); + return NULL; } void* schtFreeJobThread(void *aa) { @@ -602,6 +635,7 @@ void* schtFreeJobThread(void *aa) { taosUsleep(taosRand() % 100); schtFreeQueryJob(1); } + return NULL; } diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 941478c9d68062106bf5f373bcda6e06fbffb79c..b4cbe493f2deda18206b2663f82eb6b013e5936a 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -14,7 +14,7 @@ */ #include "syncEnv.h" -#include +// #include SSyncEnv *gSyncEnv = NULL; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index e260ef4654fbf93c404b827de2265269c503fccf..a3068d69a951f03014386d734adda73088b6de77 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1200,3 +1200,16 @@ static void syncFreeNode(void* param) { taosMemoryFree(pNode); } + +const char* syncStr(ESyncState state) { + switch (state) { + case TAOS_SYNC_STATE_FOLLOWER: + return "FOLLOWER"; + case TAOS_SYNC_STATE_CANDIDATE: + return "CANDIDATE"; + case TAOS_SYNC_STATE_LEADER: + return "LEADER"; + default: + return "ERROR"; + } +} diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 3b20dab0db2be67eccb77137baf2dadbcaec2e6e..3110c0b2a338e04d8aec6bf61ea5f9019234c69d 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -16,6 +16,8 @@ #include "syncUtil.h" #include "syncEnv.h" +void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); + // ---- encode / decode uint64_t syncUtilAddr2U64(const char* host, uint16_t port) { uint64_t u64; diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index f052d08a1e480435e5e1fb9f799c45aef436689e..09d20156f476a93b1de432ac9b87972fb691ad1f 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -8,6 +8,7 @@ #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" +#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOClientTest.cpp b/source/libs/sync/test/syncIOClientTest.cpp index c146ff4e925cb67085a53d678345e7f6a7cc2806..492b2e4349cc5a19473f6af936b1bfd45bbb553c 100644 --- a/source/libs/sync/test/syncIOClientTest.cpp +++ b/source/libs/sync/test/syncIOClientTest.cpp @@ -4,6 +4,7 @@ #include "syncInt.h" #include "syncMessage.h" #include "syncUtil.h" +#include "tdatablock.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLogStoreCheck.cpp b/source/libs/sync/test/syncLogStoreCheck.cpp index ae725ac5a25bece41f228770996e7bc94db71e96..431b291ca797847a00556f5aeb6d231f48061dc6 100644 --- a/source/libs/sync/test/syncLogStoreCheck.cpp +++ b/source/libs/sync/test/syncLogStoreCheck.cpp @@ -6,6 +6,7 @@ #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" +#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index c4a5f6cae4ee99c732d13c4bfbfbe2cf5e1b2006..27e1009335a82e42e0d8e8e1b00f8cfe709e9534 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -6,6 +6,7 @@ #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" +#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotTest.cpp b/source/libs/sync/test/syncSnapshotTest.cpp index 7ad14e0f5e41dd0d6abdbd806fcb6f5ac750ad1f..5dd9ea9fcff94be6cb4083a8bf62bbc789e7b340 100644 --- a/source/libs/sync/test/syncSnapshotTest.cpp +++ b/source/libs/sync/test/syncSnapshotTest.cpp @@ -8,6 +8,7 @@ #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" +#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index cff76a3ae7e821c144269e217ec0f4902b59fca7..ef09d2a0a4e66b52f942ae6525a0dd9814a4da03 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -8,6 +8,7 @@ #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" +#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 3a62b19c10453c8f21c3a87da586ad2c11b2d2fd..01490030f2f80383af39b521461ea3c47e732dd1 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,5 +1,5 @@ # tdb -add_library(tdb "") +add_library(tdb STATIC "") target_sources(tdb PRIVATE "src/db/tdbPCache.c" diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 8a95635ddc6a4cb8d6bba41061e1e31a8654ef0e..0dfebe98208810b6688ea75cbaba3699c40d810b 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -42,14 +42,17 @@ struct SBTree { ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \ TDB_FLAG_IS(flags, TDB_BTREE_ROOT | TDB_BTREE_LEAF) || TDB_FLAG_IS(flags, 0)) -typedef struct __attribute__((__packed__)) { + +#pragma pack(push,1) +typedef struct { TDB_BTREE_PAGE_COMMON_HDR } SLeafHdr; -typedef struct __attribute__((__packed__)) { - TDB_BTREE_PAGE_COMMON_HDR; +typedef struct { + TDB_BTREE_PAGE_COMMON_HDR SPgno pgno; // right-most child } SIntHdr; +#pragma pack(pop) typedef struct { u8 flags; diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 86272b3cad19d3b96198a9a4d0583c1aa14db06b..ce753ad930c0e3abfb289627347e80201cea3ca5 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -28,11 +28,10 @@ struct SPCache { SPage lru; }; -#define PCACHE_PAGE_HASH(pPgid) \ - ({ \ - u32 *t = (u32 *)((pPgid)->fileid); \ - t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno; \ - }) +static inline int tdbPCachePageHash(const SPgid *pPgid) { + u32 *t = (u32 *)((pPgid)->fileid); + return t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno; +} #define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL) static int tdbPCacheOpenImpl(SPCache *pCache); @@ -130,7 +129,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) SPage *pPage; // 1. Search the hash table - pPage = pCache->pgHash[PCACHE_PAGE_HASH(pPgid) % pCache->nHash]; + pPage = pCache->pgHash[tdbPCachePageHash(pPgid) % pCache->nHash]; while (pPage) { if (TDB_IS_SAME_PAGE(&(pPage->pgid), pPgid)) break; pPage = pPage->pHashNext; @@ -218,7 +217,7 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { SPage **ppPage; int h; - h = PCACHE_PAGE_HASH(&(pPage->pgid)); + h = tdbPCachePageHash(&(pPage->pgid)); for (ppPage = &(pCache->pgHash[h % pCache->nHash]); *ppPage != pPage; ppPage = &((*ppPage)->pHashNext)) ; ASSERT(*ppPage == pPage); @@ -230,7 +229,7 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) { int h; - h = PCACHE_PAGE_HASH(&(pPage->pgid)) % pCache->nHash; + h = tdbPCachePageHash(&(pPage->pgid)) % pCache->nHash; pPage->pHashNext = pCache->pgHash[h]; pCache->pgHash[h] = pPage; diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index ea75f7fbeb7865ea600f68703fcc1fc46ea9ca80..05939579d8385b1cdefdbfc7d1a59aa64a597896 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -435,17 +435,20 @@ static int tdbPageDefragment(SPage *pPage) { } /* ---------------------------------------------------------------------------------------------------------- */ -typedef struct __attribute__((__packed__)) { + +#pragma pack(push,1) +typedef struct { u16 cellNum; u16 cellBody; u16 cellFree; u16 nFree; } SPageHdr; -typedef struct __attribute__((__packed__)) { +typedef struct { u16 szCell; u16 nxOffset; } SFreeCell; +#pragma pack(pop) // cellNum static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } @@ -517,17 +520,19 @@ SPageMethods pageMethods = { setPageFreeCellInfo // setFreeCellInfo }; -typedef struct __attribute__((__packed__)) { +#pragma pack(push,1) +typedef struct { u8 cellNum[3]; u8 cellBody[3]; u8 cellFree[3]; u8 nFree[3]; } SPageHdrL; -typedef struct __attribute__((__packed__)) { +typedef struct { u8 szCell[3]; u8 nxOffset[3]; } SFreeCellL; +#pragma pack(pop) // cellNum static inline int getLPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 7672ccf34e372e1ef7cf52cd995366480979082e..8835b7f759328ffff0af1bd5a89f9dc97e85d11c 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -15,13 +15,15 @@ #include "tdbInt.h" -typedef struct __attribute__((__packed__)) { +#pragma pack(push,1) +typedef struct { u8 hdrString[16]; u16 pageSize; SPgno freePage; u32 nFreePages; u8 reserved[102]; } SFileHdr; +#pragma pack(pop) TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct"); diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 77a1ee5f6ee4c0bdd341d6cd293efc7ffa33fa9e..3707a922ee84d46849d57647a6bb6f3853a508d2 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -47,10 +47,13 @@ typedef struct { void (*setFreeCellInfo)(SCell *pCell, int szCell, int nxOffset); } SPageMethods; +#pragma pack(push,1) + // Page footer -typedef struct __attribute__((__packed__)) { +typedef struct { u8 cksm[4]; } SPageFtr; +#pragma pack(pop) struct SPage { tdb_spinlock_t lock; @@ -80,22 +83,23 @@ struct SPage { #define P_LOCK_BUSY 1 #define P_LOCK_FAIL -1 +static inline int tdbTryLockPage(tdb_spinlock_t *pLock) { + int ret; + if (tdbSpinlockTrylock(pLock) == 0) { + ret = P_LOCK_SUCC; + } else if (errno == EBUSY) { + ret = P_LOCK_BUSY; + } else { + ret = P_LOCK_FAIL; + } + return ret; +} + #define TDB_INIT_PAGE_LOCK(pPage) tdbSpinlockInit(&((pPage)->lock), 0) #define TDB_DESTROY_PAGE_LOCK(pPage) tdbSpinlockDestroy(&((pPage)->lock)) #define TDB_LOCK_PAGE(pPage) tdbSpinlockLock(&((pPage)->lock)) #define TDB_UNLOCK_PAGE(pPage) tdbSpinlockUnlock(&((pPage)->lock)) -#define TDB_TRY_LOCK_PAGE(pPage) \ - ({ \ - int ret; \ - if (tdbSpinlockTrylock(&((pPage)->lock)) == 0) { \ - ret = P_LOCK_SUCC; \ - } else if (errno == EBUSY) { \ - ret = P_LOCK_BUSY; \ - } else { \ - ret = P_LOCK_FAIL; \ - } \ - ret; \ - }) +#define TDB_TRY_LOCK_PAGE(pPage) tdbTryLockPage(&((pPage)->lock)) // APIs #define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 7085e71567bfb976ed12c42132e5c98a26e5aa16..cffd3f27469598c58a31513e11d9b72b8539fc5f 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -31,20 +31,20 @@ extern "C" { int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size); -#define TDB_REALLOC(PTR, SIZE) \ - ({ \ - void *nPtr; \ - if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) { \ - nPtr = tdbOsRealloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \ - if (nPtr) { \ - ((int *)nPtr)[0] = (SIZE); \ - nPtr = (char *)nPtr + sizeof(int); \ - } \ - } else { \ - nPtr = (PTR); \ - } \ - nPtr; \ - }) +static inline void *tdbRealloc(void *ptr, size_t size) { + void *nPtr; + if ((ptr) == NULL || ((int *)(ptr))[-1] < (size)) { + nPtr = tdbOsRealloc((ptr) ? (char *)(ptr) - sizeof(int) : NULL, (size) + sizeof(int)); + if (nPtr) { + ((int *)nPtr)[0] = (size); + nPtr = (char *)nPtr + sizeof(int); + } + } else { + nPtr = (ptr); + } + return nPtr; +} +#define TDB_REALLOC(PTR, SIZE) tdbRealloc(PTR, SIZE) #define TDB_FREE(PTR) \ do { \ diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index e50045d30b41abb363d28f5958645829d2c893ee..18a3a28bab44adf6213edc3c1975e88518ad16cd 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -366,7 +366,7 @@ const STfsFile *tfsReaddir(STfsDir *pTfsDir) { void tfsClosedir(STfsDir *pTfsDir) { if (pTfsDir) { if (pTfsDir->pDir != NULL) { - taosCloseDir(pTfsDir->pDir); + taosCloseDir(&pTfsDir->pDir); pTfsDir->pDir = NULL; } taosMemoryFree(pTfsDir); @@ -455,7 +455,7 @@ static int32_t tfsFormatDir(char *idir, char *odir) { } char tmp[PATH_MAX] = {0}; - if (realpath(wep.we_wordv[0], tmp) == NULL) { + if (taosRealPath(wep.we_wordv[0], tmp, PATH_MAX) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); wordfree(&wep); return -1; @@ -499,7 +499,7 @@ static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pTfsDir) { char adir[TMPNAME_LEN * 2] = "\0"; if (pTfsDir->pDir != NULL) { - taosCloseDir(pTfsDir->pDir); + taosCloseDir(&pTfsDir->pDir); pTfsDir->pDir = NULL; } diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 1b8cbb0f0407052e35294a990652286dd205627b..e18538cf279731e356b4c255a2cf75285746ecfd 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -158,6 +158,7 @@ typedef struct { char secured : 2; char spi : 2; + char user[TSDB_UNI_LEN]; uint64_t ahandle; // ahandle assigned by client uint32_t code; // del later uint32_t msgType; @@ -186,23 +187,23 @@ typedef enum { Normal, Quit, Release, Register } STransMsgType; typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } ConnStatus; #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) -#define RPC_RESERVE_SIZE (sizeof(STranConnCtx)) +#define RPC_RESERVE_SIZE (sizeof(STranConnCtx)) -#define RPC_MSG_OVERHEAD (sizeof(SRpcHead) + sizeof(SRpcDigest)) -#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) -#define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) +#define RPC_MSG_OVERHEAD (sizeof(SRpcHead) + sizeof(SRpcDigest)) +#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) +#define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) #define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) -#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) -#define rpcIsReq(type) (type & 1U) +#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) +#define rpcIsReq(type) (type & 1U) #define TRANS_RESERVE_SIZE (sizeof(STranConnCtx)) -#define TRANS_MSG_OVERHEAD (sizeof(STransMsgHead)) -#define transHeadFromCont(cont) ((STransMsgHead*)((char*)cont - sizeof(STransMsgHead))) -#define transContFromHead(msg) (msg + sizeof(STransMsgHead)) +#define TRANS_MSG_OVERHEAD (sizeof(STransMsgHead)) +#define transHeadFromCont(cont) ((STransMsgHead*)((char*)cont - sizeof(STransMsgHead))) +#define transContFromHead(msg) (msg + sizeof(STransMsgHead)) #define transMsgLenFromCont(contLen) (contLen + sizeof(STransMsgHead)) -#define transContLenFromMsg(msgLen) (msgLen - sizeof(STransMsgHead)); -#define transIsReq(type) (type & 1U) +#define transContLenFromMsg(msgLen) (msgLen - sizeof(STransMsgHead)); +#define transIsReq(type) (type & 1U) int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index ad50948a021ac663bcc012ff6cbf08f86a354e4e..eaca9b0fc765ddee699db235733d9508dd45dc36 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -63,13 +63,14 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + int (*retry)(void* parent, SRpcMsg*, SEpSet*); - int32_t refCount; - void* parent; - void* idPool; // handle to ID pool - void* tmrCtrl; // handle to timer - SHashObj* hash; // handle returned by hash utility - void* tcphandle; // returned handle from TCP initialization + int32_t refCount; + void* parent; + void* idPool; // handle to ID pool + void* tmrCtrl; // handle to timer + SHashObj* hash; // handle returned by hash utility + void* tcphandle; // returned handle from TCP initialization TdThreadMutex mutex; } SRpcInfo; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index ebb90338cd42303243f44be81bceef25f1160b63..fa517d6d61cca34395dd5648a4146a446b9e2e64 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -39,6 +39,7 @@ void* rpcOpen(const SRpcInit* pInit) { // register callback handle pRpc->cfp = pInit->cfp; pRpc->afp = pInit->afp; + pRpc->retry = pInit->rfp; if (pInit->connType == TAOS_CONN_SERVER) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; @@ -100,11 +101,10 @@ void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) { SRpcMsg rpcMsg; memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.contLen = sizeof(SEpSet); - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - if (rpcMsg.pCont == NULL) return; - - memcpy(rpcMsg.pCont, pEpSet, sizeof(SEpSet)); + SMEpSet msg = {.epSet = *pEpSet}; + int32_t len = tSerializeSMEpSet(NULL, 0, &msg); + rpcMsg.pCont = rpcMallocCont(len); + tSerializeSMEpSet(rpcMsg.pCont, len, &msg); rpcMsg.code = TSDB_CODE_RPC_REDIRECT; rpcMsg.handle = thandle; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 9dc12a3dece380e15f6ee410047793de77c4984b..b43b8a1e0c30cabd1eb50e302c604b75ba5436c3 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -31,12 +31,8 @@ typedef struct SCliConn { int hThrdIdx; STransCtx ctx; - bool broken; // link broken or not - ConnStatus status; // - int release; // 1: release - // spi configure - char spi; - char secured; + bool broken; // link broken or not + ConnStatus status; // char* ip; uint32_t port; @@ -44,7 +40,6 @@ typedef struct SCliConn { // debug and log info struct sockaddr_in addr; struct sockaddr_in locaddr; - } SCliConn; typedef struct SCliMsg { @@ -102,6 +97,8 @@ static void cliSendCb(uv_write_t* req, int status); static void cliConnCb(uv_connect_t* req, int status); static void cliAsyncCb(uv_async_t* handle); +static void cliAppCb(SCliConn* pConn, STransMsg* pMsg); + static SCliConn* cliCreateConn(SCliThrdObj* thrd); static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle or not*/); static void cliDestroy(uv_handle_t* handle); @@ -129,6 +126,15 @@ static void transDestroyConnCtx(STransConnCtx* ctx); static SCliThrdObj* createThrdObj(); static void destroyThrdObj(SCliThrdObj* pThrd); +static void cliWalkCb(uv_handle_t* handle, void* arg); + +#define CLI_RELEASE_UV(loop) \ + do { \ + uv_walk(loop, cliWalkCb, NULL); \ + uv_run(loop, UV_RUN_DEFAULT); \ + uv_loop_close(loop); \ + } while (0); + // snprintf may cause performance problem #define CONN_CONSTRUCT_HASH_KEY(key, ip, port) \ do { \ @@ -212,8 +218,10 @@ static void destroyThrdObj(SCliThrdObj* pThrd); } \ } while (0) -#define CONN_NO_PERSIST_BY_APP(conn) (((conn)->status == ConnNormal || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) -#define CONN_RELEASE_BY_SERVER(conn) (((conn)->status == ConnRelease || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) +#define CONN_NO_PERSIST_BY_APP(conn) \ + (((conn)->status == ConnNormal || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) +#define CONN_RELEASE_BY_SERVER(conn) \ + (((conn)->status == ConnRelease || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) #define REQUEST_NO_RESP(msg) ((msg)->noResp == 1) #define REQUEST_PERSIS_HANDLE(msg) ((msg)->persistHandle == 1) @@ -288,10 +296,9 @@ void cliHandleResp(SCliConn* conn) { tDebug("%s cli conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } - tDebug("%s cli conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pTransInst->label, conn, TMSG_INFO(pHead->msgType), - taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), taosInetNtoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen); - - conn->secured = pHead->secured; + tDebug("%s cli conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pTransInst->label, conn, + TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), + taosInetNtoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen); if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { tTrace("except, server continue send while cli ignore it"); @@ -306,7 +313,8 @@ void cliHandleResp(SCliConn* conn) { if (pCtx == NULL || pCtx->pSem == NULL) { tTrace("%s cli conn %p handle resp", pTransInst->label, conn); - (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); + cliAppCb(conn, &transMsg); + //(pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); } else { tTrace("%s cli conn(sync) %p handle resp", pTransInst->label, conn); memcpy((char*)pCtx->pRsp, (char*)&transMsg, sizeof(transMsg)); @@ -355,10 +363,12 @@ void cliHandleExcept(SCliConn* pConn) { if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { transMsg.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType); - tDebug("%s cli conn %p construct ahandle %p by %s", CONN_GET_INST_LABEL(pConn), pConn, transMsg.ahandle, TMSG_INFO(transMsg.msgType)); + tDebug("%s cli conn %p construct ahandle %p by %s", CONN_GET_INST_LABEL(pConn), pConn, transMsg.ahandle, + TMSG_INFO(transMsg.msgType)); if (transMsg.ahandle == NULL) { transMsg.ahandle = transCtxDumpBrokenlinkVal(&pConn->ctx, (int32_t*)&(transMsg.msgType)); - tDebug("%s cli conn %p construct ahandle %p due to brokenlink", CONN_GET_INST_LABEL(pConn), pConn, transMsg.ahandle); + tDebug("%s cli conn %p construct ahandle %p due to brokenlink", CONN_GET_INST_LABEL(pConn), pConn, + transMsg.ahandle); } } else { transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; @@ -370,7 +380,8 @@ void cliHandleExcept(SCliConn* pConn) { once = true; continue; } - (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); + cliAppCb(pConn, &transMsg); + //(pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); } else { tTrace("%s cli conn(sync) %p handle except", pTransInst->label, pConn); memcpy((char*)(pCtx->pRsp), (char*)(&transMsg), sizeof(transMsg)); @@ -600,39 +611,21 @@ void cliSend(SCliConn* pConn) { pMsg->pCont = (void*)rpcMallocCont(0); pMsg->contLen = 0; } - STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); - pHead->ahandle = pCtx != NULL ? (uint64_t)pCtx->ahandle : 0; - int msgLen = transMsgLenFromCont(pMsg->contLen); - if (!pConn->secured) { - char* buf = taosMemoryCalloc(1, msgLen + sizeof(STransUserMsg)); - memcpy(buf, (char*)pHead, msgLen); - - STransUserMsg* uMsg = (STransUserMsg*)(buf + msgLen); - memcpy(uMsg->user, pTransInst->user, tListLen(uMsg->user)); - memcpy(uMsg->secret, pTransInst->secret, tListLen(uMsg->secret)); - - // to avoid mem leak - destroyUserdata(pMsg); - - pMsg->pCont = (char*)buf + sizeof(STransMsgHead); - pMsg->contLen = msgLen + sizeof(STransUserMsg) - sizeof(STransMsgHead); - - pHead = (STransMsgHead*)buf; - pHead->secured = 1; - msgLen += sizeof(STransUserMsg); - } - + STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + pHead->ahandle = pCtx != NULL ? (uint64_t)pCtx->ahandle : 0; pHead->noResp = REQUEST_NO_RESP(pMsg) ? 1 : 0; pHead->persist = REQUEST_PERSIS_HANDLE(pMsg) ? 1 : 0; pHead->msgType = pMsg->msgType; pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0; + memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user)); uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); - tDebug("%s cli conn %p %s is send to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), - taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); + tDebug("%s cli conn %p %s is send to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, + TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), + taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); if (pHead->persist == 1) { CONN_SET_PERSIST_BY_APP(pConn); @@ -671,9 +664,11 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { destroyCmsg(pMsg); destroyConnPool(pThrd->pool); uv_timer_stop(&pThrd->timer); + uv_walk(pThrd->loop, cliWalkCb, NULL); + pThrd->quit = true; - uv_stop(pThrd->loop); + // uv_stop(pThrd->loop); } static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = pMsg->msg.handle; @@ -786,7 +781,6 @@ static void* cliWorkThread(void* arg) { SCliThrdObj* pThrd = (SCliThrdObj*)arg; setThreadName("trans-cli-work"); uv_run(pThrd->loop, UV_RUN_DEFAULT); - return NULL; } @@ -851,8 +845,8 @@ static void destroyThrdObj(SCliThrdObj* pThrd) { if (pThrd == NULL) { return; } - uv_stop(pThrd->loop); taosThreadJoin(pThrd->thread, NULL); + CLI_RELEASE_UV(pThrd->loop); taosThreadMutexDestroy(&pThrd->msgMtx); transDestroyAsyncPool(pThrd->asyncPool); @@ -874,6 +868,11 @@ void cliSendQuit(SCliThrdObj* thrd) { msg->type = Quit; transSendAsync(thrd->asyncPool, &msg->q); } +void cliWalkCb(uv_handle_t* handle, void* arg) { + if (!uv_is_closing(handle)) { + uv_close(handle, NULL); + } +} int cliRBChoseIdx(STrans* pTransInst) { int64_t index = pTransInst->index; @@ -882,6 +881,18 @@ int cliRBChoseIdx(STrans* pTransInst) { } return index % pTransInst->numOfThreads; } +void cliAppCb(SCliConn* pConn, STransMsg* transMsg) { + SCliThrdObj* pThrd = pConn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; + + if (transMsg->code == TSDB_CODE_RPC_REDIRECT && pTransInst->retry != NULL) { + SMEpSet emsg = {0}; + tDeserializeSMEpSet(transMsg->pCont, transMsg->contLen, &emsg); + pTransInst->retry(pTransInst, transMsg, &(emsg.epSet)); + } else { + pTransInst->cfp(pTransInst->parent, transMsg, NULL); + } +} void transCloseClient(void* arg) { SCliObj* cli = arg; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 03b97b6fa1a948768c74316f66c1e7eaaa670986..9e53811fd381a26b7fd35997041f63069a505761 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -195,7 +195,7 @@ SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) void transDestroyAsyncPool(SAsyncPool* pool) { for (int i = 0; i < pool->nAsync; i++) { uv_async_t* async = &(pool->asyncs[i]); - uv_close((uv_handle_t*)async, NULL); + // uv_close((uv_handle_t*)async, NULL); SAsyncItem* item = async->data; taosThreadMutexDestroy(&item->mtx); taosMemoryFree(item); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 50ce0bddcd96ba82045c4775823944ef49a0e325..59a30051ef1fb961305836c6771de3d193b2068f 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -30,7 +30,6 @@ typedef struct SSrvConn { uv_timer_t pTimer; queue queue; - int ref; int persist; // persist connection or not SConnBuffer readBuf; // read buf, int inType; @@ -46,7 +45,6 @@ typedef struct SSrvConn { struct sockaddr_in addr; struct sockaddr_in locaddr; - char secured; int spi; char info[64]; char user[TSDB_UNI_LEN]; // user ID for the link @@ -93,27 +91,6 @@ typedef struct SServerObj { static const char* notify = "a"; -#define CONN_SHOULD_RELEASE(conn, head) \ - do { \ - if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ - conn->status = ConnRelease; \ - transClearBuffer(&conn->readBuf); \ - transFreeMsg(transContFromHead((char*)head)); \ - tTrace("server conn %p received release request", conn); \ - \ - STransMsg tmsg = {.code = 0, .handle = (void*)conn, .ahandle = NULL}; \ - SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); \ - srvMsg->msg = tmsg; \ - srvMsg->type = Release; \ - srvMsg->pConn = conn; \ - if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ - return; \ - } \ - uvStartSendRespInternal(srvMsg); \ - return; \ - } \ - } while (0) - static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); @@ -126,10 +103,14 @@ static void uvWorkerAsyncCb(uv_async_t* handle); static void uvAcceptAsyncCb(uv_async_t* handle); static void uvShutDownCb(uv_shutdown_t* req, int status); -static void uvFreeCb(uv_handle_t* handle) { - // - taosMemoryFree(handle); -} +/* + * time-consuming task throwed into BG work thread + */ +static void uvWorkDoTask(uv_work_t* req); +static void uvWorkAfterTask(uv_work_t* req, int status); + +static void uvWalkCb(uv_handle_t* handle, void* arg); +static void uvFreeCb(uv_handle_t* handle); static void uvStartSendRespInternal(SSrvMsg* smsg); static void uvPrepareSendData(SSrvMsg* msg, uv_buf_t* wb); @@ -146,7 +127,8 @@ static void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd); static void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd); static void uvHandleResp(SSrvMsg* msg, SWorkThrdObj* thrd); static void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd); -static void (*transAsyncHandle[])(SSrvMsg* msg, SWorkThrdObj* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, uvHandleRegister}; +static void (*transAsyncHandle[])(SSrvMsg* msg, SWorkThrdObj* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, + uvHandleRegister}; static void uvDestroyConn(uv_handle_t* handle); @@ -158,6 +140,34 @@ static void* transAcceptThread(void* arg); static bool addHandleToWorkloop(void* arg); static bool addHandleToAcceptloop(void* arg); +#define CONN_SHOULD_RELEASE(conn, head) \ + do { \ + if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ + conn->status = ConnRelease; \ + transClearBuffer(&conn->readBuf); \ + transFreeMsg(transContFromHead((char*)head)); \ + tTrace("server conn %p received release request", conn); \ + \ + STransMsg tmsg = {.code = 0, .handle = (void*)conn, .ahandle = NULL}; \ + SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); \ + srvMsg->msg = tmsg; \ + srvMsg->type = Release; \ + srvMsg->pConn = conn; \ + if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ + return; \ + } \ + uvStartSendRespInternal(srvMsg); \ + return; \ + } \ + } while (0) + +#define SRV_RELEASE_UV(loop) \ + do { \ + uv_walk(loop, uvWalkCb, NULL); \ + uv_run(loop, UV_RUN_DEFAULT); \ + uv_loop_close(loop); \ + } while (0); + void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SSrvConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; @@ -176,16 +186,16 @@ static void uvHandleReq(SSrvConn* pConn) { uint32_t msgLen = pBuf->len; STransMsgHead* pHead = (STransMsgHead*)msg; - if (pHead->secured == 1) { - STransUserMsg* uMsg = (STransUserMsg*)((char*)msg + msgLen - sizeof(STransUserMsg)); - memcpy(pConn->user, uMsg->user, tListLen(uMsg->user)); - memcpy(pConn->secret, uMsg->secret, tListLen(uMsg->secret)); - } pHead->code = htonl(pHead->code); pHead->msgLen = htonl(pHead->msgLen); - if (pHead->secured == 1) { - pHead->msgLen -= sizeof(STransUserMsg); - } + memcpy(pConn->user, pHead->user, strlen(pHead->user)); + + // TODO(dengyihao): time-consuming task throwed into BG Thread + // uv_work_t* wreq = taosMemoryMalloc(sizeof(uv_work_t)); + // wreq->data = pConn; + // uv_read_stop((uv_stream_t*)pConn->pTcp); + // transRefSrvHandle(pConn); + // uv_queue_work(((SWorkThrdObj*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask); CONN_SHOULD_RELEASE(pConn, pHead); @@ -209,12 +219,13 @@ static void uvHandleReq(SSrvConn* pConn) { } if (pConn->status == ConnNormal && pHead->noResp == 0) { transRefSrvHandle(pConn); - tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen); + tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType), + taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), + ntohs(pConn->locaddr.sin_port), transMsg.contLen); } else { - tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, resp:%d ", pConn, TMSG_INFO(transMsg.msgType), - taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), - transMsg.contLen, pHead->noResp); + tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, resp:%d ", pConn, + TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), + taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen, pHead->noResp); // no ref here } @@ -338,12 +349,6 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); pHead->ahandle = (uint64_t)pMsg->ahandle; - // pHead->secured = pMsg->code == 0 ? 1 : 0; // - if (!pConn->secured) { - pConn->secured = pMsg->code == 0 ? 1 : 0; - } - pHead->secured = pConn->secured; - if (pConn->status == ConnNormal) { pHead->msgType = pConn->inType + 1; } else { @@ -354,8 +359,9 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { char* msg = (char*)pHead; int32_t len = transMsgLenFromCont(pMsg->contLen); - tDebug("server conn %p %s is sent to %s:%d, local info: %s:%d", pConn, TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); + tDebug("server conn %p %s is sent to %s:%d, local info: %s:%d", pConn, TMSG_INFO(pHead->msgType), + taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), + ntohs(pConn->locaddr.sin_port)); pHead->msgLen = htonl(len); wb->base = msg; @@ -367,7 +373,7 @@ static void uvStartSendRespInternal(SSrvMsg* smsg) { uvPrepareSendData(smsg, &wb); SSrvConn* pConn = smsg->pConn; - uv_timer_stop(&pConn->pTimer); + // uv_timer_stop(&pConn->pTimer); uv_write(&pConn->pWriter, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnSendCb); } static void uvStartSendResp(SSrvMsg* smsg) { @@ -436,36 +442,17 @@ void uvWorkerAsyncCb(uv_async_t* handle) { static void uvWalkCb(uv_handle_t* handle, void* arg) { if (!uv_is_closing(handle)) { uv_close(handle, NULL); - // uv_unref(handle); - tDebug("handle: %p -----test----", handle); } } -#define MAKE_VALGRIND_HAPPY(loop) \ - do { \ - uv_walk(loop, uvWalkCb, NULL); \ - uv_run(loop, UV_RUN_DEFAULT); \ - uv_loop_close(loop); \ - } while (0); +static void uvFreeCb(uv_handle_t* handle) { + // + taosMemoryFree(handle); +} static void uvAcceptAsyncCb(uv_async_t* async) { SServerObj* srv = async->data; tDebug("close server port %d", srv->port); uv_walk(srv->loop, uvWalkCb, NULL); - // uv_close((uv_handle_t*)async, NULL); - // uv_close((uv_handle_t*)&srv->server, NULL); - // uv_stop(srv->loop); - // uv_print_all_handles(srv->loop, stderr); - // int ref = uv_loop_alive(srv->loop); - // assert(ref == 0); - // tError("active size %d", ref); - // uv_stop(srv->loop); - // uv_run(srv->loop, UV_RUN_DEFAULT); - // fprintf(stderr, "------------------------------------"); - // uv_print_all_handles(srv->loop, stderr); - - // int ret = uv_loop_close(srv->loop); - // tError("(loop)->active_reqs.count: %d, ret: %d", (srv->loop)->active_reqs.count, ret); - // assert(ret == 0); } static void uvShutDownCb(uv_shutdown_t* req, int status) { @@ -476,6 +463,24 @@ static void uvShutDownCb(uv_shutdown_t* req, int status) { taosMemoryFree(req); } +static void uvWorkDoTask(uv_work_t* req) { + // doing time-consumeing task + // only auth conn currently, add more func later + tTrace("server conn %p start to be processed in BG Thread", req->data); + return; +} + +static void uvWorkAfterTask(uv_work_t* req, int status) { + if (status != 0) { + tTrace("server conn %p failed to processed ", req->data); + } + // Done time-consumeing task + // add more func later + // this func called in main loop + tTrace("server conn %p already processed ", req->data); + taosMemoryFree(req); +} + void uvOnAcceptCb(uv_stream_t* stream, int status) { if (status == -1) { return; @@ -532,8 +537,8 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->pTransInst = pThrd->pTransInst; /* init conn timer*/ - uv_timer_init(pThrd->loop, &pConn->pTimer); - pConn->pTimer.data = pConn; + // uv_timer_init(pThrd->loop, &pConn->pTimer); + // pConn->pTimer.data = pConn; pConn->hostThrd = pThrd; @@ -677,17 +682,15 @@ static void uvDestroyConn(uv_handle_t* handle) { SWorkThrdObj* thrd = conn->hostThrd; tDebug("server conn %p destroy", conn); - uv_timer_stop(&conn->pTimer); + // uv_timer_stop(&conn->pTimer); transQueueDestroy(&conn->srvMsgs); QUEUE_REMOVE(&conn->queue); taosMemoryFree(conn->pTcp); - // taosMemoryFree(conn); + taosMemoryFree(conn); if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) { tTrace("work thread quit"); - // uv_walk(thrd->loop, uvWalkCb, NULL); - uv_loop_close(thrd->loop); - uv_stop(thrd->loop); + uv_walk(thrd->loop, uvWalkCb, NULL); } } @@ -749,9 +752,7 @@ End: void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) { thrd->quit = true; if (QUEUE_IS_EMPTY(&thrd->conn)) { - // uv_walk(thrd->loop, uvWalkCb, NULL); - uv_loop_close(thrd->loop); - uv_stop(thrd->loop); + uv_walk(thrd->loop, uvWalkCb, NULL); } else { destroyAllConn(thrd); } @@ -802,7 +803,7 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { return; } taosThreadJoin(pThrd->thread, NULL); - // MAKE_VALGRIND_HAPPY(pThrd->loop); + SRV_RELEASE_UV(pThrd->loop); transDestroyAsyncPool(pThrd->asyncPool); taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); @@ -822,7 +823,7 @@ void transCloseServer(void* arg) { uv_async_send(srv->pAcceptAsync); taosThreadJoin(srv->thread, NULL); - MAKE_VALGRIND_HAPPY(srv->loop); + SRV_RELEASE_UV(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { sendQuitToWorkThrd(srv->pThreadObj[i]); @@ -845,10 +846,8 @@ void transRefSrvHandle(void* handle) { if (handle == NULL) { return; } - SSrvConn* conn = handle; - int ref = T_REF_INC((SSrvConn*)handle); - UNUSED(ref); + tDebug("server conn %p ref count: %d", handle, ref); } void transUnrefSrvHandle(void* handle) { diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index cdf09081679fb138f45b802b27a400d49ceace2d..2b1c4fa623e24d0c2a09dacf642e41a08418abdc 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -12,8 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include - #include #include "os.h" #include "rpcLog.h" diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index 27a468a86fb7e3f011d5826ef524fe5bfb55aa6d..3f1a7805b43d7d4d18c065e59338cd3b6b648709 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -12,8 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include - #include #include "os.h" #include "rpcLog.h" diff --git a/source/libs/transport/test/transportTests.cpp b/source/libs/transport/test/transportTests.cpp index 2488dda3f6c8fd2c2a466b377bf4225094282c02..35009c7686dec2495f69eb3a363f50406fa98a9c 100644 --- a/source/libs/transport/test/transportTests.cpp +++ b/source/libs/transport/test/transportTests.cpp @@ -156,14 +156,14 @@ TEST_F(TransCtxEnv, mergeTest) { STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; @@ -176,14 +176,14 @@ TEST_F(TransCtxEnv, mergeTest) { STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; @@ -198,7 +198,7 @@ TEST_F(TransCtxEnv, mergeTest) { STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryCalloc(1, 11); memcpy(val1.val, val.c_str(), val.size()); @@ -206,7 +206,7 @@ TEST_F(TransCtxEnv, mergeTest) { key++; } { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryCalloc(1, 11); memcpy(val1.val, val.c_str(), val.size()); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 9f8fe8d84dbf1b9916992292212559ef6ca06c99..3e6663f4647eb059e1391953892eebbaf64c7a73 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -145,7 +145,7 @@ int walCheckAndRepairMeta(SWal* pWal) { } } - taosCloseDir(pDir); + taosCloseDir(&pDir); regfree(&logRegPattern); regfree(&idxRegPattern); @@ -351,7 +351,7 @@ static int walFindCurMetaVer(SWal* pWal) { break; } } - taosCloseDir(pDir); + taosCloseDir(&pDir); regfree(&walMetaRegexPattern); return metaVer; } diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index 261ec312ad968988106c782c290055c8f6327d81..d709bbbcc2bd12f649b37537dd8de6fee2adba21 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -6,7 +6,8 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/include" PUBLIC "${TD_SOURCE_DIR}/include/util" PUBLIC "${TD_SOURCE_DIR}/contrib/pthread" - PUBLIC "${TD_SOURCE_DIR}/contrib/gnuregex" + PUBLIC "${TD_SOURCE_DIR}/contrib/iconv" + PUBLIC "${TD_SOURCE_DIR}/contrib/msvcregex" ) # iconv find_path(IconvApiIncludes iconv.h PATHS) @@ -17,5 +18,14 @@ if(USE_TD_MEMORY) add_definitions(-DUSE_TD_MEMORY) endif () target_link_libraries( - os pthread dl rt m + os pthread ) +if(NOT TD_WINDOWS) + target_link_libraries( + os dl m rt + ) +else() + target_link_libraries( + os ws2_32 iconv msvcregex wcwidth winmm + ) +endif(NOT TD_WINDOWS) diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index 8b452947d393c307e3ddfe2883c906583e5a5677..0fe946bf68e669d4b5b152ec61e5b38c59883a34 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -16,78 +16,101 @@ #define ALLOW_FORBID_FUNC #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS // add -char interlocked_add_fetch_8(char volatile* ptr, char val) { +int8_t interlocked_add_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedExchangeAdd8(ptr, val) + val; } -short interlocked_add_fetch_16(short volatile* ptr, short val) { +int16_t interlocked_add_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedExchangeAdd16(ptr, val) + val; } -long interlocked_add_fetch_32(long volatile* ptr, long val) { +int32_t interlocked_add_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedExchangeAdd(ptr, val) + val; } -__int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) { +int64_t interlocked_add_fetch_64(int64_t volatile* ptr, int64_t val) { return InterlockedExchangeAdd64(ptr, val) + val; } -char interlocked_and_fetch_8(char volatile* ptr, char val) { +void* interlocked_add_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)(_InterlockedExchangeAdd((int32_t volatile*)(ptr), (int32_t)val) + (int32_t)val); +#else + return (void*)(InterlockedExchangeAdd64((int64_t volatile*)(ptr), (int64_t)val) + (int64_t)val); +#endif +} + +int8_t interlocked_and_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedAnd8(ptr, val) & val; } -short interlocked_and_fetch_16(short volatile* ptr, short val) { +int16_t interlocked_and_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedAnd16(ptr, val) & val; } -long interlocked_and_fetch_32(long volatile* ptr, long val) { +int32_t interlocked_and_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedAnd(ptr, val) & val; } - -__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val) { -#ifndef _M_IX86 - return _InterlockedAnd64(ptr, val) & val; -#else - __int64 old, res; +int64_t interlocked_and_fetch_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old, res; do { old = *ptr; res = old & val; } while (_InterlockedCompareExchange64(ptr, res, old) != old); return res; +#else + return _InterlockedAnd64(ptr, val) & val; #endif } -__int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old; +void* interlocked_and_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_and_fetch_32((int32_t volatile*)ptr, (int32_t)val); +#else + return (void*)interlocked_and_fetch_64((int64_t volatile*)ptr, (int64_t)val); +#endif +} + +int64_t interlocked_fetch_and_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old; do { old = *ptr; } while (_InterlockedCompareExchange64(ptr, old & val, old) != old); return old; #else - return _InterlockedAnd64((__int64 volatile*)(ptr), (__int64)(val)); + return _InterlockedAnd64((int64_t volatile*)(ptr), (int64_t)(val)); #endif } -char interlocked_or_fetch_8(char volatile* ptr, char val) { +void* interlocked_fetch_and_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)_InterlockedAnd((int32_t volatile*)(ptr), (int32_t)(val)); +#else + return (void*)_InterlockedAnd64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +int8_t interlocked_or_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedOr8(ptr, val) | val; } -short interlocked_or_fetch_16(short volatile* ptr, short val) { +int16_t interlocked_or_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedOr16(ptr, val) | val; } -long interlocked_or_fetch_32(long volatile* ptr, long val) { +int32_t interlocked_or_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedOr(ptr, val) | val; } -__int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old, res; +int64_t interlocked_or_fetch_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old, res; do { old = *ptr; res = old | val; @@ -98,33 +121,49 @@ __int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val) { #endif } -__int64 interlocked_fetch_or_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old; +void* interlocked_or_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_or_fetch_32((int32_t volatile*)ptr, (int32_t)val); +#else + return (void*)interlocked_or_fetch_64((int64_t volatile*)ptr, (int64_t)val); +#endif +} + +int64_t interlocked_fetch_or_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old; do { old = *ptr; } while(_InterlockedCompareExchange64(ptr, old | val, old) != old); return old; #else - return _InterlockedOr64((__int64 volatile*)(ptr), (__int64)(val)); + return _InterlockedOr64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +void* interlocked_fetch_or_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)_InterlockedOr((int32_t volatile*)(ptr), (int32_t)(val)); +#else + return (void*)interlocked_fetch_or_64((int64_t volatile*)(ptr), (int64_t)(val)); #endif } -char interlocked_xor_fetch_8(char volatile* ptr, char val) { +int8_t interlocked_xor_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedXor8(ptr, val) ^ val; } -short interlocked_xor_fetch_16(short volatile* ptr, short val) { +int16_t interlocked_xor_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedXor16(ptr, val) ^ val; } -long interlocked_xor_fetch_32(long volatile* ptr, long val) { +int32_t interlocked_xor_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedXor(ptr, val) ^ val; } -__int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old, res; +int64_t interlocked_xor_fetch_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old, res; do { old = *ptr; res = old ^ val; @@ -135,15 +174,62 @@ __int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val) { #endif } -__int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old; +void* interlocked_xor_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_xor_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); +#else + return (void*)interlocked_xor_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +int64_t interlocked_fetch_xor_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old; do { old = *ptr; } while (_InterlockedCompareExchange64(ptr, old ^ val, old) != old); return old; #else - return _InterlockedXor64((__int64 volatile*)(ptr), (__int64)(val)); + return _InterlockedXor64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +void* interlocked_fetch_xor_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)_InterlockedXor((int32_t volatile*)(ptr), (int32_t)(val)); +#else + return (void*)interlocked_fetch_xor_64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +int32_t interlocked_sub_fetch_32(int32_t volatile* ptr, int32_t val) { + return interlocked_add_fetch_32(ptr, -val); +} + +int64_t interlocked_sub_fetch_64(int64_t volatile* ptr, int64_t val) { + return interlocked_add_fetch_64(ptr, -val); +} + +void* interlocked_sub_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_sub_fetch_32((int32_t volatile*)ptr, (int32_t)val); +#else + return (void*)interlocked_add_fetch_64((int64_t volatile*)ptr, (int64_t)val); +#endif +} +int32_t interlocked_fetch_sub_32(int32_t volatile* ptr, int32_t val) { + return _InterlockedExchangeAdd(ptr, -val); +} + +int64_t interlocked_fetch_sub_64(int64_t volatile* ptr, int64_t val) { + return _InterlockedExchangeAdd64(ptr, -val); +} + +void* interlocked_fetch_sub_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_fetch_sub_32((int32_t volatile*)ptr, (int32_t)val); +#else + return (void*)interlocked_fetch_sub_64((int64_t volatile*)ptr, (int64_t)val); #endif } @@ -189,7 +275,7 @@ int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val ) { int8_t atomic_load_8(int8_t volatile *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(int8_t volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -199,7 +285,7 @@ int8_t atomic_load_8(int8_t volatile *ptr) { } int16_t atomic_load_16(int16_t volatile *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(int16_t volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -209,7 +295,7 @@ int16_t atomic_load_16(int16_t volatile *ptr) { } int32_t atomic_load_32(int32_t volatile *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(int32_t volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -219,7 +305,7 @@ int32_t atomic_load_32(int32_t volatile *ptr) { } int64_t atomic_load_64(int64_t volatile *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(int64_t volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -229,7 +315,7 @@ int64_t atomic_load_64(int64_t volatile *ptr) { } void* atomic_load_ptr(void *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(void* volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -239,7 +325,7 @@ void* atomic_load_ptr(void *ptr) { } void atomic_store_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(int8_t volatile*)(ptr)) = (int8_t)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -249,7 +335,7 @@ void atomic_store_8(int8_t volatile *ptr, int8_t val) { } void atomic_store_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(int16_t volatile*)(ptr)) = (int16_t)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -259,7 +345,7 @@ void atomic_store_16(int16_t volatile *ptr, int16_t val) { } void atomic_store_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(int32_t volatile*)(ptr)) = (int32_t)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -269,7 +355,7 @@ void atomic_store_32(int32_t volatile *ptr, int32_t val) { } void atomic_store_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(int64_t volatile*)(ptr)) = (int64_t)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -279,7 +365,7 @@ void atomic_store_64(int64_t volatile *ptr, int64_t val) { } void atomic_store_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(void* volatile*)(ptr)) = (void*)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -289,7 +375,7 @@ void atomic_store_ptr(void *ptr, void *val) { } int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchange8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return atomic_exchange_8_impl((int8_t*)ptr, (int8_t)val); @@ -299,7 +385,7 @@ int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchange16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return atomic_exchange_16_impl((int16_t*)ptr, (int16_t)val); @@ -309,7 +395,7 @@ int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchange((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return atomic_exchange_32_impl((int32_t*)ptr, (int32_t)val); @@ -319,7 +405,7 @@ int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchange64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return atomic_exchange_64_impl((int64_t*)ptr, (int64_t)val); @@ -329,7 +415,7 @@ int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val) { } void* atomic_exchange_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #ifdef _WIN64 return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); #else @@ -343,7 +429,7 @@ void* atomic_exchange_ptr(void *ptr, void *val) { } int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchange8((int8_t volatile*)(ptr), (int8_t)(newval), (int8_t)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -353,7 +439,7 @@ int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t } int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchange16((int16_t volatile*)(ptr), (int16_t)(newval), (int16_t)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -363,7 +449,7 @@ int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, in } int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchange((int32_t volatile*)(ptr), (int32_t)(newval), (int32_t)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -373,7 +459,7 @@ int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, in } int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchange64((int64_t volatile*)(ptr), (int64_t)(newval), (int64_t)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -383,7 +469,7 @@ int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, in } void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchangePointer((void* volatile*)(ptr), (void*)(newval), (void*)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -393,7 +479,7 @@ void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval) { } int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); @@ -403,7 +489,7 @@ int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); @@ -413,7 +499,7 @@ int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); @@ -423,7 +509,7 @@ int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); @@ -432,18 +518,18 @@ int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val) { #endif } -void* atomic_add_fetch_ptr(void *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +void* atomic_add_fetch_ptr(void *ptr, void *val) { +#ifdef WINDOWS return interlocked_add_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); #else - return __atomic_add_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST); + return __atomic_add_fetch((void **)(ptr), (size_t)(val), __ATOMIC_SEQ_CST); #endif } int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); @@ -453,7 +539,7 @@ int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); @@ -463,7 +549,7 @@ int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); @@ -473,7 +559,7 @@ int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); @@ -482,18 +568,18 @@ int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val) { #endif } -void* atomic_fetch_add_ptr(void *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return _InterlockedExchangeAddptr((void* volatile*)(ptr), (void*)(val)); +void* atomic_fetch_add_ptr(void *ptr, void *val) { +#ifdef WINDOWS + return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); #else - return __atomic_fetch_add((void **)(ptr), (val), __ATOMIC_SEQ_CST); + return __atomic_fetch_add((void **)(ptr), (size_t)(val), __ATOMIC_SEQ_CST); #endif } int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_8((int8_t volatile*)(ptr), -(int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); @@ -503,7 +589,7 @@ int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_16((int16_t volatile*)(ptr), -(int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); @@ -513,8 +599,8 @@ int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return interlocked_add_fetch_32((int32_t volatile*)(ptr), -(int32_t)(val)); +#ifdef WINDOWS + return interlocked_sub_fetch_32(ptr, val); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); #else @@ -523,8 +609,8 @@ int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return interlocked_add_fetch_64((int64_t volatile*)(ptr), -(int64_t)(val)); +#ifdef WINDOWS + return interlocked_sub_fetch_64(ptr, val); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); #else @@ -532,9 +618,9 @@ int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val) { #endif } -void* atomic_sub_fetch_ptr(void *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return interlocked_add_fetch_ptr((void* volatile*)(ptr), -(void*)(val)); +void* atomic_sub_fetch_ptr(void *ptr, void* val) { +#ifdef WINDOWS + return interlocked_sub_fetch_ptr(ptr, val); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); #else @@ -543,7 +629,7 @@ void* atomic_sub_fetch_ptr(void *ptr, int32_t val) { } int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd8((int8_t volatile*)(ptr), -(int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_sub((ptr), (val)); @@ -553,7 +639,7 @@ int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd16((int16_t volatile*)(ptr), -(int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_sub((ptr), (val)); @@ -563,17 +649,17 @@ int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return _InterlockedExchangeAdd((int32_t volatile*)(ptr), -(int32_t)(val)); +#ifdef WINDOWS + return interlocked_fetch_sub_32(ptr, val); #elif defined(_TD_NINGSI_60) - return __sync_fetch_and_sub((ptr), (val)); + return __sync_fetch_and_sub(ptr, val); #else return __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST); #endif } int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), -(int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_sub((ptr), (val)); @@ -582,9 +668,9 @@ int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val) { #endif } -void* atomic_fetch_sub_ptr(void *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return _InterlockedExchangeAddptr((void* volatile*)(ptr), -(void*)(val)); +void* atomic_fetch_sub_ptr(void *ptr, void* val) { +#ifdef WINDOWS + return interlocked_fetch_sub_ptr(ptr, val); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_sub((ptr), (val)); #else @@ -593,7 +679,7 @@ void* atomic_fetch_sub_ptr(void *ptr, int32_t val) { } int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -603,7 +689,7 @@ int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -613,7 +699,7 @@ int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -623,7 +709,7 @@ int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -633,7 +719,7 @@ int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val) { } void* atomic_and_fetch_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -643,7 +729,7 @@ void* atomic_and_fetch_ptr(void *ptr, void *val) { } int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedAnd8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -653,7 +739,7 @@ int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedAnd16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -663,7 +749,7 @@ int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedAnd((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -673,7 +759,7 @@ int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_and_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -683,7 +769,7 @@ int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val) { } void* atomic_fetch_and_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_and_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -693,7 +779,7 @@ void* atomic_fetch_and_ptr(void *ptr, void *val) { } int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -703,7 +789,7 @@ int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -713,7 +799,7 @@ int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -723,7 +809,7 @@ int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -733,7 +819,7 @@ int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val) { } void* atomic_or_fetch_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -743,7 +829,7 @@ void* atomic_or_fetch_ptr(void *ptr, void *val) { } int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedOr8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -753,7 +839,7 @@ int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedOr16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -763,7 +849,7 @@ int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedOr((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -773,7 +859,7 @@ int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_or_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -783,7 +869,7 @@ int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val) { } void* atomic_fetch_or_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_or_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -793,7 +879,7 @@ void* atomic_fetch_or_ptr(void *ptr, void *val) { } int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -803,7 +889,7 @@ int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -813,7 +899,7 @@ int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -823,7 +909,7 @@ int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -833,7 +919,7 @@ int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val) { } void* atomic_xor_fetch_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -843,7 +929,7 @@ void* atomic_xor_fetch_ptr(void *ptr, void *val) { } int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedXor8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); @@ -853,7 +939,7 @@ int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedXor16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); @@ -863,7 +949,7 @@ int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedXor((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); @@ -873,7 +959,7 @@ int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_xor_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); @@ -883,7 +969,7 @@ int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val) { } void* atomic_fetch_xor_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_xor_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index a955fb3b0a88d53beae43e49ed03df735388c0d2..46fa2ecd3aed5c5e99b1241f5736b04b8f50c12f 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -17,19 +17,40 @@ #define ALLOW_FORBID_FUNC #include "os.h" -#include "osString.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -/* - * windows implementation - */ +#ifdef WINDOWS + +#include + +typedef struct TdDirEntry { + WIN32_FIND_DATA findFileData; +} TdDirEntry; + + +typedef struct TdDir { + TdDirEntry dirEntry; + HANDLE hFind; +} TdDir; + +int wordexp(char *words, wordexp_t *pwordexp, int flags) { + pwordexp->we_offs = 0; + pwordexp->we_wordc = 1; + pwordexp->we_wordv[0] = pwordexp->wordPos; + + memset(pwordexp->wordPos, 0, 1025); + if (_fullpath(pwordexp->wordPos, words, 1024) == NULL) { + pwordexp->we_wordv[0] = words; + printf("failed to parse relative path:%s to abs path", words); + return -1; + } + + printf("parse relative path:%s to abs path:%s", words, pwordexp->wordPos); + return 0; +} -// todo +void wordfree(wordexp_t *pwordexp) {} #else -/* - * linux implementation - */ #include #include @@ -39,19 +60,21 @@ typedef struct dirent dirent; typedef struct DIR TdDir; -typedef struct dirent TdDirent; +typedef struct dirent TdDirEntry; + +#endif void taosRemoveDir(const char *dirname) { - DIR *dir = opendir(dirname); - if (dir == NULL) return; + TdDirPtr pDir = taosOpenDir(dirname); + if (pDir == NULL) return; - struct dirent *de = NULL; - while ((de = readdir(dir)) != NULL) { - if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; + TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir)) != NULL) { + if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; char filename[1024]; - snprintf(filename, sizeof(filename), "%s/%s", dirname, de->d_name); - if (de->d_type & DT_DIR) { + snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de)); + if (taosDirEntryIsDir(de)) { taosRemoveDir(filename); } else { (void)taosRemoveFile(filename); @@ -59,13 +82,14 @@ void taosRemoveDir(const char *dirname) { } } - closedir(dir); + taosCloseDir(&pDir); rmdir(dirname); //printf("dir:%s is removed\n", dirname); + return; } -bool taosDirExist(char *dirname) { return taosCheckExistFile(dirname); } +bool taosDirExist(const char *dirname) { return taosCheckExistFile(dirname); } int32_t taosMkDir(const char *dirname) { int32_t code = mkdir(dirname, 0755); @@ -118,18 +142,18 @@ int32_t taosMulMkDir(const char *dirname) { } void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { - DIR *dir = opendir(dirname); - if (dir == NULL) return; + TdDirPtr pDir = taosOpenDir(dirname); + if (pDir == NULL) return; - int64_t sec = taosGetTimestampSec(); - struct dirent *de = NULL; + int64_t sec = taosGetTimestampSec(); + TdDirEntryPtr de = NULL; - while ((de = readdir(dir)) != NULL) { - if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; + while ((de = taosReadDir(pDir)) != NULL) { + if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; char filename[1024]; - snprintf(filename, sizeof(filename), "%s/%s", dirname, de->d_name); - if (de->d_type & DT_DIR) { + snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de)); + if (taosDirEntryIsDir(de)) { continue; } else { int32_t len = (int32_t)strlen(filename); @@ -156,7 +180,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { } } - closedir(dir); + taosCloseDir(&pDir); rmdir(dirname); } @@ -177,65 +201,131 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { return 0; } -int32_t taosRealPath(char *dirname, int32_t maxlen) { +int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) { char tmp[PATH_MAX] = {0}; +#ifdef WINDOWS + if (_fullpath(dirname, tmp, maxlen) != NULL) { +#else if (realpath(dirname, tmp) != NULL) { - strncpy(dirname, tmp, maxlen); +#endif + if (realPath == NULL) { + strncpy(dirname, tmp, maxlen); + } else { + strncpy(realPath, tmp, maxlen); + } + return 0; } - return 0; + return -1; } bool taosIsDir(const char *dirname) { - DIR *dir = opendir(dirname); - if (dir != NULL) { - closedir(dir); + TdDirPtr pDir = taosOpenDir(dirname); + if (pDir != NULL) { + taosCloseDir(&pDir); return true; } return false; } char* taosDirName(char *name) { +#ifdef WINDOWS + char Drive1[MAX_PATH], Dir1[MAX_PATH]; + _splitpath(name, Drive1, Dir1, NULL, NULL); + size_t dirNameLen = strlen(Drive1) + strlen(Dir1); + if (dirNameLen > 0) { + name[dirNameLen] = 0; + } + return name; +#else return dirname(name); +#endif } char* taosDirEntryBaseName(char *name) { - return basename(name); +#ifdef WINDOWS + char Filename1[MAX_PATH], Ext1[MAX_PATH]; + _splitpath(name, NULL, NULL, Filename1, Ext1); + return name + (strlen(name) - strlen(Filename1) - strlen(Ext1)); +#else + return (char*)basename(name); +#endif } TdDirPtr taosOpenDir(const char *dirname) { if (dirname == NULL) { return NULL; } + +#ifdef WINDOWS + char szFind[MAX_PATH]; //这是要找的 + HANDLE hFind; + + TdDirPtr pDir = taosMemoryMalloc(sizeof(TdDir)); + + strcpy(szFind, dirname); + strcat(szFind, "\\*.*"); //利用通配符找这个目录下的所以文件，包括目录 + + pDir->hFind = FindFirstFile(szFind, &(pDir->dirEntry.findFileData)); + if (INVALID_HANDLE_VALUE == pDir->hFind) { + taosMemoryFree(pDir); + return NULL; + } + return pDir; +#else return (TdDirPtr)opendir(dirname); +#endif + } TdDirEntryPtr taosReadDir(TdDirPtr pDir) { if (pDir == NULL) { return NULL; } +#ifdef WINDOWS + if (!FindNextFile(pDir->hFind, &(pDir->dirEntry.findFileData))) { + return NULL; + } + return (TdDirEntryPtr)&(pDir->dirEntry.findFileData); +#else return (TdDirEntryPtr)readdir((DIR*)pDir); +#endif } bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry) { if (pDirEntry == NULL) { return false; } +#ifdef WINDOWS + return (pDirEntry->findFileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; +#else return (((dirent*)pDirEntry)->d_type & DT_DIR) != 0; +#endif } char* taosGetDirEntryName(TdDirEntryPtr pDirEntry) { if (pDirEntry == NULL) { return NULL; } +#ifdef WINDOWS + return pDirEntry->findFileData.cFileName; +#else return ((dirent*)pDirEntry)->d_name; +#endif } -int32_t taosCloseDir(TdDirPtr pDir) { - if (pDir == NULL) { +int32_t taosCloseDir(TdDirPtr *ppDir) { + if (ppDir == NULL || *ppDir == NULL) { return -1; } - return closedir((DIR*)pDir); -} - +#ifdef WINDOWS + FindClose((*ppDir)->hFind); + taosMemoryFree(*ppDir); + *ppDir = NULL; + return 0; +#else + closedir((DIR*)*ppDir); + *ppDir = NULL; + return 0; #endif +} diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 22884298efbcf7015a6426a2fa2827afa87914b1..6746025f78be619868e53267588f8f4defe1d5cb 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -16,8 +16,6 @@ #define _DEFAULT_SOURCE #include "osEnv.h" -extern void taosWinSocketInit(); - char configDir[PATH_MAX] = {0}; char tsDataDir[PATH_MAX] = {0}; char tsLogDir[PATH_MAX] = {0}; @@ -37,6 +35,7 @@ int64_t tsOpenMax = 0; int64_t tsStreamMax = 0; float tsNumOfCores = 0; int64_t tsTotalMemoryKB = 0; +char* tsProcPath = NULL; void osDefaultInit() { taosSeedRand(taosSafeRand()); @@ -50,7 +49,7 @@ void osDefaultInit() { tsNumOfCores = 2; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS taosWinSocketInit(); const char *tmpDir = getenv("tmp"); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index c2c4af18e5cafe14276ca7e6c65df2a6c982b213..786e58a6d261fb5eca7f4d0ecedaf9336e5f6bb1 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -16,8 +16,11 @@ #include "os.h" #include "osSemaphore.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #include +#define F_OK 0 +#define W_OK 2 +#define R_OK 4 #if defined(_MSDOS) #define open _open @@ -33,12 +36,14 @@ extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ #endif /* defined(_UTF8_SOURCE) */ #endif /* defined(_WIN32) */ +#define _SEND_FILE_STEP_ 1000 + #else #include #include #if !defined(_TD_DARWIN_64) - #include +#include #endif #include #include @@ -58,15 +63,15 @@ typedef int32_t FileFd; typedef struct TdFile { TdThreadRwlock rwlock; - int refId; - FileFd fd; - FILE *fp; + int refId; + FileFd fd; + FILE *fp; } * TdFilePtr, TdFile; #define FILE_WITH_LOCK 1 void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS const char *tdengineTmpFileNamePrefix = "tdengine-"; char tmpPath[PATH_MAX]; @@ -117,7 +122,7 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha } int64_t taosCopyFile(const char *from, const char *to) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else char buffer[4096]; @@ -160,7 +165,7 @@ _err: int32_t taosRemoveFile(const char *path) { return remove(path); } int32_t taosRenameFile(const char *oldName, const char *newName) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); if (code < 0) { // printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); @@ -178,11 +183,11 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { } int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else struct stat fileStat; - int32_t code = stat(path, &fileStat); + int32_t code = stat(path, &fileStat); if (code < 0) { return code; } @@ -199,11 +204,11 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { #endif } int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else struct stat fileStat; - int32_t code = stat(path, &fileStat); + int32_t code = stat(path, &fileStat); if (code < 0) { return code; } @@ -223,10 +228,10 @@ int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { void autoDelFileListAdd(const char *path) { return; } TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return NULL; #else - int fd = -1; + int fd = -1; FILE *fp = NULL; if (tdFileOptions & TD_FILE_STREAM) { char *mode = NULL; @@ -285,7 +290,7 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { } int64_t taosCloseFile(TdFilePtr *ppFile) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else if (ppFile == NULL || *ppFile == NULL) { @@ -325,7 +330,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t leftbytes = count; int64_t readbytes; char *tbuf = (char *)buf; @@ -366,7 +371,14 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) taosThreadRwlockRdlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); // Please check if you have closed the file. +#ifdef WINDOWS + size_t pos = lseek(pFile->fd, 0, SEEK_CUR); + lseek(pFile->fd, (long)offset, SEEK_SET); + int64_t ret = read(pFile->fd, buf, count); + lseek(pFile->fd, pos, SEEK_SET); +#else int64_t ret = pread(pFile->fd, buf, count, offset); +#endif #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); #endif @@ -380,7 +392,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockWrlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t nleft = count; int64_t nwritten = 0; @@ -414,7 +426,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t ret = lseek(pFile->fd, (long)offset, whence); #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); @@ -423,16 +435,16 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { } int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. struct stat fileStat; - int32_t code = fstat(pFile->fd, &fileStat); + int32_t code = fstat(pFile->fd, &fileStat); if (code < 0) { return code; } @@ -450,36 +462,36 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { } int32_t taosLockFile(TdFilePtr pFile) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); #endif } int32_t taosUnLockFile(TdFilePtr pFile) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); #endif } int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS if (pFile->fd < 0) { errno = EBADF; - uError("%s\n", "fd arg was negative"); + printf("%s\n", "fd arg was negative"); return -1; } @@ -489,7 +501,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { li_0.QuadPart = (int64_t)0; BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); if (!cur) { - uError("SetFilePointerEx Error getting current position in file.\n"); + printf("SetFilePointerEx Error getting current position in file.\n"); return -1; } @@ -498,7 +510,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); if (cur2 == 0) { int error = GetLastError(); - uError("SetFilePointerEx GetLastError is: %d\n", error); + printf("SetFilePointerEx GetLastError is: %d\n", error); switch (error) { case ERROR_INVALID_HANDLE: errno = EBADF; @@ -512,7 +524,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { if (!SetEndOfFile(h)) { int error = GetLastError(); - uError("SetEndOfFile GetLastError is:%d", error); + printf("SetEndOfFile GetLastError is:%d", error); switch (error) { case ERROR_INVALID_HANDLE: errno = EBADF; @@ -529,17 +541,17 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. return ftruncate(pFile->fd, l_size); #endif } int32_t taosFsyncFile(TdFilePtr pFile) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS if (pFile->fd < 0) { errno = EBADF; - uError("%s\n", "fd arg was negative"); + printf("%s\n", "fd arg was negative"); return -1; } @@ -558,86 +570,53 @@ int32_t taosFsyncFile(TdFilePtr pFile) { #endif } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - -#define _SEND_FILE_STEP_ 1000 - -int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { - fseek(in_file, (int32_t)(*offset), 0); - int64_t writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = {0}; - - for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { - size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); - if (rlen <= 0) { - return writeLen; - } else if (rlen < _SEND_FILE_STEP_) { - fwrite(buffer, 1, rlen, out_file); - return (int64_t)(writeLen + rlen); - } else { - fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); - writeLen += _SEND_FILE_STEP_; - } - } - - int64_t remain = count - writeLen; - if (remain > 0) { - size_t rlen = fread(buffer, 1, (size_t)remain, in_file); - if (rlen <= 0) { - return writeLen; - } else { - fwrite(buffer, 1, (size_t)remain, out_file); - writeLen += remain; - } +int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { + if (pFileOut == NULL || pFileIn == NULL) { + return 0; } + assert(pFileIn->fd >= 0 && pFileOut->fd >= 0); - return writeLen; -} - -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { - if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0); +#ifdef WINDOWS + lseek(pFileIn->fd, (int32_t)(*offset), 0); int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = {0}; - for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { - int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_); + for (int64_t len = 0; len < (size - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { + size_t rlen = read(pFileIn->fd, (void *)buffer, _SEND_FILE_STEP_); if (rlen <= 0) { return writeLen; } else if (rlen < _SEND_FILE_STEP_) { - taosWriteSocket(dfd, buffer, rlen); + write(pFileOut->fd, (void *)buffer, (uint32_t)rlen); return (int64_t)(writeLen + rlen); } else { - taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_); + write(pFileOut->fd, (void *)buffer, _SEND_FILE_STEP_); writeLen += _SEND_FILE_STEP_; } } - int64_t remain = count - writeLen; + int64_t remain = size - writeLen; if (remain > 0) { - int32_t rlen = read(sfd, buffer, (int32_t)remain); + size_t rlen = read(pFileIn->fd, (void *)buffer, (size_t)remain); if (rlen <= 0) { return writeLen; } else { - taosWriteSocket(sfd, buffer, (int32_t)remain); + write(pFileOut->fd, (void *)buffer, (uint32_t)remain); writeLen += remain; } } - return writeLen; -} #elif defined(_TD_DARWIN_64) -int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { int r = 0; if (offset) { r = fseek(in_file, *offset, SEEK_SET); if (r == -1) return -1; } - off_t len = count; + off_t len = size; while (len > 0) { - char buf[1024 * 16]; + char buf[1024 * 16]; off_t n = sizeof(buf); if (len < n) n = len; size_t m = fread(buf, 1, n, in_file); @@ -651,64 +630,10 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co } len -= m; } - return count - len; -} - -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { - int r = 0; - if (offset) { - r = lseek(sfd, *offset, SEEK_SET); - if (r == -1) return -1; - } - off_t len = count; - while (len > 0) { - char buf[1024 * 16]; - off_t n = sizeof(buf); - if (len < n) n = len; - size_t m = read(sfd, buf, n); - if (m == -1) return -1; - if (m == 0) break; - size_t l = write(dfd, buf, m); - if (l == -1) return -1; - len -= l; - } - return count - len; -} + return size - len; #else -// int64_t taosSendFile(int fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size) { -// if (pFileSrc == NULL) { -// return 0; -// } -// assert(pFileSrc->fd >= 0); - -// int64_t leftbytes = size; -// int64_t sentbytes; - -// while (leftbytes > 0) { -// sentbytes = sendfile(fdDst, pFileSrc->fd, offset, leftbytes); -// if (sentbytes == -1) { -// if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { -// continue; -// } else { -// return -1; -// } -// } else if (sentbytes == 0) { -// return (int64_t)(size - leftbytes); -// } - -// leftbytes -= sentbytes; -// } - -// return size; -// } - -int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { - if (pFileOut == NULL || pFileIn == NULL) { - return 0; - } - assert(pFileIn->fd >= 0 && pFileOut->fd >= 0); int64_t leftbytes = size; int64_t sentbytes; @@ -728,9 +653,8 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in } return size; -} - #endif +} void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { if (pFile == NULL) { @@ -745,22 +669,10 @@ void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { fflush(pFile->fp); } -#if !defined(WINDOWS) -void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) { - if (pFile == NULL) { - return NULL; - } - assert(pFile->fd >= 0); // Please check if you have closed the file. - - void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0); - return ptr; -} -#endif - bool taosValidFile(TdFilePtr pFile) { return pFile != NULL; } int32_t taosUmaskFile(int32_t maskVal) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else return umask(maskVal); @@ -769,16 +681,26 @@ int32_t taosUmaskFile(int32_t maskVal) { int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; } int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { - if (pFile == NULL) { + if (pFile == NULL || ptrBuf == NULL ) { return -1; } if (*ptrBuf != NULL) { taosMemoryFreeClear(*ptrBuf); } assert(pFile->fp != NULL); - +#ifdef WINDOWS + *ptrBuf = taosMemoryMalloc(1024); + if (*ptrBuf == NULL) return -1; + if (fgets(*ptrBuf, 1023, pFile->fp) == NULL) { + taosMemoryFreeClear(*ptrBuf); + return -1; + } + (*ptrBuf)[1023] = 0; + return strlen(*ptrBuf); +#else size_t len = 0; return getline(ptrBuf, &len, pFile->fp); +#endif } int32_t taosEOFFile(TdFilePtr pFile) { if (pFile == NULL) { @@ -789,8 +711,6 @@ int32_t taosEOFFile(TdFilePtr pFile) { return feof(pFile->fp); } -#if !defined(WINDOWS) - bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) { int flags = 0; @@ -805,10 +725,11 @@ bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) { if (tdFileAccessOptions & TD_FILE_ACCESS_WRITE_OK) { flags |= W_OK; } - +#ifdef WINDOWS + return _access(pathname, flags) == 0; +#else return access(pathname, flags) == 0; +#endif } bool taosCheckExistFile(const char *pathname) { return taosCheckAccessFile(pathname, TD_FILE_ACCESS_EXIST_OK); }; - -#endif // WINDOWS diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 2df61c08d418788ecadcdff7e869a948d8c6855e..d2369ea2a29e2e45979b041e1f2b586c6374c4e3 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -17,7 +17,7 @@ #define _DEFAULT_SOURCE #include "osLocale.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #if (_WIN64) #include #include @@ -87,7 +87,7 @@ void taosSetSystemLocale(const char *inLocale, const char *inCharSet) { } void taosGetSystemLocale(char *outLocale, char *outCharset) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS char *locale = setlocale(LC_CTYPE, "chs"); if (locale != NULL) { tstrncpy(outLocale, locale, TD_LOCALE_LEN); diff --git a/source/os/src/osLz4.c b/source/os/src/osLz4.c index 29d219f37b879abe675508af5b26acec0abadddd..f4e4951857de7fa30fa63843395ee02954357325 100644 --- a/source/os/src/osLz4.c +++ b/source/os/src/osLz4.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS /* * windows implementation diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index cd756b45e2ff9d6bb4ddb6a3bdefe62b269ccae2..5b733daec2f32a9e058115e6db51ff5ee192f8ba 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -35,7 +35,7 @@ typedef struct TdMemoryInfo { // static TdMemoryInfoPtr GlobalMemoryPtr = NULL; -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define tstrdup(str) _strdup(str) #else #define tstrdup(str) strdup(str) @@ -181,7 +181,11 @@ int32_t taosMemorySize(void *ptr) { assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); return pTdMemoryInfo->memorySize; +#else +#ifdef WINDOWS + return _msize(ptr); #else return malloc_usable_size(ptr); #endif +#endif } diff --git a/source/os/src/osProc.c b/source/os/src/osProc.c index b6de638ac2e4c9fe851d7c73515a9d509ef2ae95..f92a3b37836a787ac17f870ab0377ce54d369523 100644 --- a/source/os/src/osProc.c +++ b/source/os/src/osProc.c @@ -17,9 +17,10 @@ #define _DEFAULT_SOURCE #include "os.h" -static char *tsProcPath = NULL; - int32_t taosNewProc(char **args) { +#ifdef WINDOWS + return 0; +#else int32_t pid = fork(); if (pid == 0) { args[0] = tsProcPath; @@ -30,23 +31,36 @@ int32_t taosNewProc(char **args) { } else { return pid; } +#endif } void taosWaitProc(int32_t pid) { +#ifdef WINDOWS +#else int32_t status = -1; waitpid(pid, &status, 0); +#endif } -void taosKillProc(int32_t pid) { kill(pid, SIGINT); } +void taosKillProc(int32_t pid) { +#ifdef WINDOWS +#else + kill(pid, SIGINT); +#endif +} bool taosProcExist(int32_t pid) { +#ifdef WINDOWS + return false; +#else int32_t p = getpgid(pid); return p >= 0; +#endif } // the length of the new name must be less than the original name to take effect void taosSetProcName(int32_t argc, char **argv, const char *name) { - prctl(PR_SET_NAME, name); + setThreadName(name); for (int32_t i = 0; i < argc; ++i) { int32_t len = strlen(argv[i]); diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index 847f3a52a60da49f87be579cea8f9a4e008918a5..461a72e96297aa8f1b981d0b52c9fec6ce2e0916 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -15,7 +15,9 @@ #define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS +#include "windows.h" +#include "wincrypt.h" #else #include #include @@ -25,9 +27,25 @@ void taosSeedRand(uint32_t seed) { return srand(seed); } uint32_t taosRand(void) { return rand(); } -uint32_t taosRandR(uint32_t *pSeed) { return rand_r(pSeed); } +uint32_t taosRandR(uint32_t *pSeed) { +#ifdef WINDOWS + return rand_s(pSeed); +#else + return rand_r(pSeed); +#endif +} uint32_t taosSafeRand(void) { +#ifdef WINDOWS + uint32_t seed; + HCRYPTPROV hCryptProv; + if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, 0)) return seed; + if (hCryptProv != NULL) { + if (!CryptGenRandom(hCryptProv, 4, &seed)) return seed; + } + if (hCryptProv != NULL) CryptReleaseContext(hCryptProv, 0); + return seed; +#else TdFilePtr pFile; int seed; @@ -43,6 +61,7 @@ uint32_t taosSafeRand(void) { } return (uint32_t)seed; +#endif } void taosRandStr(char* str, int32_t size) { diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 1297fdbc276cc3a9a4611fa296ba495b452287e3..b2c7d185e65b820dcfd29fd3a9d64d03c45c4ed9 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS /* * windows implementation diff --git a/source/os/src/osShm.c b/source/os/src/osShm.c index b276b48d0e86193da888d545fa6dc4cf1cf6694d..1cd51f94a09f770914c67e46cdb65f6334d83b6c 100644 --- a/source/os/src/osShm.c +++ b/source/os/src/osShm.c @@ -22,15 +22,20 @@ static int32_t shmids[MAX_SHMIDS] = {0}; static void taosDeleteCreatedShms() { +#if defined(WINDOWS) +#else for (int32_t i = 0; i < MAX_SHMIDS; ++i) { int32_t shmid = shmids[i] - 1; if (shmid >= 0) { shmctl(shmid, IPC_RMID, NULL); } } +#endif } int32_t taosCreateShm(SShm* pShm, int32_t key, int32_t shmsize) { +#if defined(WINDOWS) +#else pShm->id = -1; #if 1 @@ -64,10 +69,13 @@ int32_t taosCreateShm(SShm* pShm, int32_t key, int32_t shmsize) { shmctl(pShm->id, IPC_RMID, NULL); #endif +#endif return 0; } void taosDropShm(SShm* pShm) { +#if defined(WINDOWS) +#else if (pShm->id >= 0) { if (pShm->ptr != NULL) { shmdt(pShm->ptr); @@ -77,14 +85,18 @@ void taosDropShm(SShm* pShm) { pShm->id = -1; pShm->size = 0; pShm->ptr = NULL; +#endif } int32_t taosAttachShm(SShm* pShm) { +#if defined(WINDOWS) +#else errno = 0; void* ptr = shmat(pShm->id, NULL, 0); if (errno == 0) { pShm->ptr = ptr; } +#endif return errno; } diff --git a/source/os/src/osSignal.c b/source/os/src/osSignal.c index 1d7fa517e58c873a4fbfb1d66792a971945370a6..d9b225868a74779ea400caaccd17460524094561 100644 --- a/source/os/src/osSignal.c +++ b/source/os/src/osSignal.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS /* * windows implementation @@ -47,6 +47,8 @@ void taosDflSignal(int32_t signum) { signal(signum, SIG_DFL); } +void taosKillChildOnParentStopped() { } + #else /* diff --git a/source/os/src/osSleep.c b/source/os/src/osSleep.c index 724347b0bc9fb4ff27de2d2d69d8be253cb10743..870467ceef398bbc78a162819032800b8d6474ac 100644 --- a/source/os/src/osSleep.c +++ b/source/os/src/osSleep.c @@ -23,7 +23,7 @@ #endif void taosSsleep(int32_t s) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS Sleep(1000 * s); #else sleep(s); @@ -31,7 +31,7 @@ void taosSsleep(int32_t s) { } void taosMsleep(int32_t ms) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS Sleep(ms); #else usleep(ms * 1000); @@ -39,8 +39,15 @@ void taosMsleep(int32_t ms) { } void taosUsleep(int32_t us) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - nanosleep(1000 * us); +#ifdef WINDOWS + HANDLE timer; + LARGE_INTEGER interval; + interval.QuadPart = (10 * us); + + timer = CreateWaitableTimer(NULL, TRUE, NULL); + SetWaitableTimer(timer, &interval, 0, NULL, NULL, 0); + WaitForSingleObject(timer, INFINITE); + CloseHandle(timer); #else usleep(us); #endif diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index d79d7c3d580b53f3de9a1829afee9fe4f40c6892..0430f68a268c251a257dc822a99d4b55b4d9599f 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -25,9 +25,6 @@ #include #include #include -#include -#include -#include "winsock2.h" #else #include #include @@ -68,7 +65,7 @@ int32_t taosSendto(TdSocketPtr pSocket, void *buf, int len, unsigned int flags, if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen); #else return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen); @@ -78,7 +75,7 @@ int32_t taosWriteSocket(TdSocketPtr pSocket, void *buf, int len) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return send(pSocket->fd, buf, len, 0); ; #else @@ -89,7 +86,7 @@ int32_t taosReadSocket(TdSocketPtr pSocket, void *buf, int len) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return recv(pSocket->fd, buf, len, 0); ; #else @@ -97,14 +94,14 @@ int32_t taosReadSocket(TdSocketPtr pSocket, void *buf, int len) { #endif } -int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen) { +int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, int *addrLen) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } return recvfrom(pSocket->fd, buf, len, flags, destAddr, addrLen); } int32_t taosCloseSocketNoCheck1(SocketFd fd) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return closesocket(fd); #else return close(fd); @@ -205,8 +202,8 @@ int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer) { #endif } -void taosWinSocketInit1() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +void taosWinSocketInit() { +#ifdef WINDOWS static char flag = 0; if (flag == 0) { WORD wVersionRequested; @@ -223,7 +220,7 @@ int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS u_long mode; if (on) { mode = 1; @@ -255,7 +252,7 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { return 0; } @@ -274,21 +271,21 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void return setsockopt(pSocket->fd, level, optname, optval, optlen); #else - return setsockopt(pSocket->fd, level, optname, optval, (socklen_t)optlen); + return setsockopt(pSocket->fd, level, optname, optval, (int)optlen); #endif } int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else - return getsockopt(pSocket->fd, level, optname, optval, (socklen_t *)optlen); + return getsockopt(pSocket->fd, level, optname, optval, (int *)optlen); #endif } uint32_t taosInetAddr(const char *ipAddr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS uint32_t value; int32_t ret = inet_pton(AF_INET, ipAddr, &value); if (ret <= 0) { @@ -301,7 +298,7 @@ uint32_t taosInetAddr(const char *ipAddr) { #endif } const char *taosInetNtoa(struct in_addr ipInt) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS // not thread safe, only for debug usage while print log static char tmpDstStr[16]; return inet_ntop(AF_INET, &ipInt, tmpDstStr, INET6_ADDRSTRLEN); @@ -693,7 +690,7 @@ TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { } TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, - socklen_t *addrLen) { + int *addrLen) { if (pServerSocket == NULL || pServerSocket->fd < 0) { return NULL; } @@ -753,12 +750,12 @@ int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len } void taosBlockSIGPIPE() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int32_t rc = taosThreadSigMask(SIG_BLOCK, &signal_mask, NULL); + int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); if (rc != 0) { // printf("failed to block SIGPIPE"); } @@ -864,26 +861,26 @@ void tinet_ntoa(char *ipstr, uint32_t ip) { } void taosIgnSIGPIPE() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else signal(SIGPIPE, SIG_IGN); #endif } void taosSetMaskSIGPIPE() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int32_t rc = taosThreadSigMask(SIG_SETMASK, &signal_mask, NULL); + int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); if (rc != 0) { // printf("failed to setmask SIGPIPE"); } #endif } -int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, socklen_t *addrLen) { +int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *addrLen) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } @@ -893,7 +890,7 @@ int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, sockle TdEpollPtr taosCreateEpoll(int32_t size) { EpollFd fd = -1; -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else fd = epoll_create(size); #endif @@ -915,7 +912,7 @@ int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocke if (pEpoll == NULL || pEpoll->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else code = epoll_ctl(pEpoll->fd, epollOperate, pSocket->fd, event); #endif @@ -926,7 +923,7 @@ int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxE if (pEpoll == NULL || pEpoll->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else code = epoll_wait(pEpoll->fd, event, maxEvents, timeout); #endif diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 5adb564e52f0d050fe7885f21d8dd11349518763..ed596a051d34b003a016211a48356f2da2b431de 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -24,6 +24,55 @@ extern int wcwidth(wchar_t c); extern int wcswidth(const wchar_t *s, size_t n); +#ifdef WINDOWS +char *strsep(char **stringp, const char *delim) { + char * s; + const char *spanp; + int32_t c, sc; + char *tok; + if ((s = *stringp) == NULL) + return (NULL); + for (tok = s;;) { + c = *s++; + spanp = delim; + do { + if ((sc = *spanp++) == c) { + if (c == 0) + s = NULL; + else + s[-1] = 0; + *stringp = s; + return (tok); + } + } while (sc != 0); + } + /* NOTREACHED */ +} +/* Duplicate a string, up to at most size characters */ +char *strndup(const char *s, size_t size) { + size_t l; + char *s2; + l = strlen(s); + if (l > size) l=size; + s2 = malloc(l+1); + if (s2) { + strncpy(s2, s, l); + s2[l] = '\0'; + } + return s2; +} +/* Copy no more than N characters of SRC to DEST, returning the address of + the terminating '\0' in DEST, if any, or else DEST + N. */ +char *stpncpy (char *dest, const char *src, size_t n) { + size_t size = strnlen (src, n); + memcpy (dest, src, size); + dest += size; + if (size == n) + return dest; + return memset (dest, '\0', n - size); +} +#endif + int64_t taosStr2int64(const char *str) { char *endptr = NULL; return strtoll(str, &endptr, 10); diff --git a/source/os/src/osStrptime.c b/source/os/src/osStrptime.c index 99549f02cb9bd78b23ca7e51f97113662f2f4432..8d878577ea2f30afbef3dd44cd3503b28e646ef7 100644 --- a/source/os/src/osStrptime.c +++ b/source/os/src/osStrptime.c @@ -38,7 +38,7 @@ // //#include "lukemftp.h" -// #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +// #ifdef WINDOWS // #include // #include @@ -100,7 +100,7 @@ // #endif // char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { -// #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +// #ifdef WINDOWS // char c; // const char *bp; // size_t len = 0; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 4ffbc13fb3b33aa9d4e44b55e53caf3aa6476713..ccc302e6467770f946e8a9031c63d330d185b73e 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -17,7 +17,23 @@ #include "os.h" #include "taoserror.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#define PROCESS_ITEM 12 + +typedef struct { + uint64_t user; + uint64_t nice; + uint64_t system; + uint64_t idle; +} SysCpuInfo; + +typedef struct { + uint64_t utime; // user time + uint64_t stime; // kernel time + uint64_t cutime; // all user time + uint64_t cstime; // all dead time +} ProcCpuInfo; + +#ifdef WINDOWS /* * windows implementation @@ -92,22 +108,6 @@ LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { #include #include -#define PROCESS_ITEM 12 - -typedef struct { - uint64_t user; - uint64_t nice; - uint64_t system; - uint64_t idle; -} SysCpuInfo; - -typedef struct { - uint64_t utime; // user time - uint64_t stime; // kernel time - uint64_t cutime; // all user time - uint64_t cstime; // all dead time -} ProcCpuInfo; - static pid_t tsProcId; static char tsSysNetFile[] = "/proc/net/dev"; static char tsSysCpuFile[] = "/proc/stat"; @@ -125,8 +125,12 @@ static void taosGetProcIOnfos() { snprintf(tsProcCpuFile, sizeof(tsProcCpuFile), "/proc/%d/stat", tsProcId); snprintf(tsProcIOFile, sizeof(tsProcIOFile), "/proc/%d/io", tsProcId); } +#endif static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { +#ifdef WINDOWS +#elif defined(_TD_DARWIN_64) +#else TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { return -1; @@ -145,10 +149,14 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); +#endif return 0; } static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { +#ifdef WINDOWS +#elif defined(_TD_DARWIN_64) +#else TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { return -1; @@ -172,10 +180,10 @@ static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); +#endif return 0; } -#endif bool taosCheckSystemIsSmallEnd() { union check { @@ -187,7 +195,7 @@ bool taosCheckSystemIsSmallEnd() { } void taosGetSystemInfo() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS taosGetCpuCores(&tsNumOfCores); taosGetTotalMemory(&tsTotalMemoryKB); @@ -210,7 +218,7 @@ void taosGetSystemInfo() { } int32_t taosGetEmail(char *email, int32_t maxLen) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #elif defined(_TD_DARWIN_64) const char *filepath = "/usr/local/taos/email"; @@ -241,7 +249,7 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { } int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #elif defined(_TD_DARWIN_64) char *line = NULL; size_t size = 0; @@ -296,7 +304,7 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { } int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #elif defined(_TD_DARWIN_64) char *line = NULL; size_t size = 0; @@ -355,7 +363,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { } int32_t taosGetCpuCores(float *numOfCores) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS SYSTEM_INFO info; GetSystemInfo(&info); *numOfCores = info.dwNumberOfProcessors; @@ -399,7 +407,7 @@ void taosGetCpuUsage(double *cpu_system, double *cpu_engine) { } int32_t taosGetTotalMemory(int64_t *totalKB) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS MEMORYSTATUSEX memsStat; memsStat.dwLength = sizeof(memsStat); if (!GlobalMemoryStatusEx(&memsStat)) { @@ -417,7 +425,7 @@ int32_t taosGetTotalMemory(int64_t *totalKB) { } int32_t taosGetProcMemory(int64_t *usedKB) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS unsigned bytes_used = 0; #if defined(_WIN64) && defined(_MSC_VER) @@ -469,7 +477,7 @@ int32_t taosGetProcMemory(int64_t *usedKB) { } int32_t taosGetSysMemory(int64_t *usedKB) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS MEMORYSTATUSEX memsStat; memsStat.dwLength = sizeof(memsStat); if (!GlobalMemoryStatusEx(&memsStat)) { @@ -534,7 +542,7 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { } int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS IO_COUNTERS io_counter; if (GetProcessIoCounters(GetCurrentProcess(), &io_counter)) { if (rchars) *rchars = io_counter.ReadTransferCount; @@ -620,7 +628,7 @@ void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, i } int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS *receive_bytes = 0; *transmit_bytes = 0; return 0; @@ -691,7 +699,7 @@ void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) { } void taosKillSystem() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS printf("function taosKillSystem, exit!"); exit(0); #elif defined(_TD_DARWIN_64) @@ -705,7 +713,7 @@ void taosKillSystem() { } int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS GUID guid; CoCreateGuid(&guid); @@ -741,7 +749,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { } char *taosGetCmdlineByPID(int pid) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return ""; #elif defined(_TD_DARWIN_64) static char cmdline[1024]; @@ -777,7 +785,7 @@ char *taosGetCmdlineByPID(int pid) { } void taosSetCoreDump(bool enable) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS SetUnhandledExceptionFilter(&FlCrashDump); #elif defined(_TD_DARWIN_64) #else @@ -857,7 +865,7 @@ void taosSetCoreDump(bool enable) { } SysNameInfo taosGetSysNameInfo() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #elif defined(_TD_DARWIN_64) SysNameInfo info = {0}; diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index 665f6370e1c35e17c3fdef1f66217ffd3095c3e1..cf3c95e6589c81268e7363581e5aa757ba969cc1 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -183,16 +183,33 @@ void resetTerminalMode() { TdCmdPtr taosOpenCmd(const char *cmd) { if (cmd == NULL) return NULL; +#ifdef WINDOWS + return (TdCmdPtr)_popen(cmd, "r"); +#else return (TdCmdPtr)popen(cmd, "r"); +#endif } int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf) { - if (pCmd == NULL) { + if (pCmd == NULL || ptrBuf == NULL ) { return -1; } - + if (*ptrBuf != NULL) { + taosMemoryFreeClear(*ptrBuf); + } +#ifdef WINDOWS + *ptrBuf = taosMemoryMalloc(1024); + if (*ptrBuf == NULL) return -1; + if (fgets(*ptrBuf, 1023, (FILE*)pCmd) == NULL) { + taosMemoryFreeClear(*ptrBuf); + return -1; + } + (*ptrBuf)[1023] = 0; + return strlen(*ptrBuf); +#else size_t len = 0; return getline(ptrBuf, &len, (FILE*)pCmd); +#endif } int32_t taosEOFCmd(TdCmdPtr pCmd) { @@ -206,7 +223,11 @@ int64_t taosCloseCmd(TdCmdPtr *ppCmd) { if (ppCmd == NULL || *ppCmd == NULL) { return 0; } +#ifdef WINDOWS + _pclose((FILE*)(*ppCmd)); +#else pclose((FILE*)(*ppCmd)); +#endif *ppCmd = NULL; return 0; } diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index 656871fd3ccef2e370527f314db242c6e61d2176..a2778e44c51a8b20a22039cc662aedc0567086ff 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -294,7 +294,7 @@ int32_t taosThreadSetSpecific(TdThreadKey key, const void *value) { } int32_t taosThreadSpinDestroy(TdThreadSpinlock * lock) { -#ifndef __USE_XOPEN2K +#ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_destroy((pthread_mutex_t*)lock); #else return pthread_spin_destroy((pthread_spinlock_t*)lock); @@ -302,15 +302,16 @@ int32_t taosThreadSpinDestroy(TdThreadSpinlock * lock) { } int32_t taosThreadSpinInit(TdThreadSpinlock * lock, int32_t pshared) { -#ifndef __USE_XOPEN2K - return pthread_mutex_init((pthread_mutex_t*)lock, pshared); +#ifdef TD_USE_SPINLOCK_AS_MUTEX + assert(pshared == NULL); + return pthread_mutex_init((pthread_mutex_t*)lock, NULL); #else return pthread_spin_init((pthread_spinlock_t*)lock, pshared); #endif } int32_t taosThreadSpinLock(TdThreadSpinlock * lock) { -#ifndef __USE_XOPEN2K +#ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_lock((pthread_mutex_t*)lock); #else return pthread_spin_lock((pthread_spinlock_t*)lock); @@ -318,7 +319,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock * lock) { } int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock) { -#ifndef __USE_XOPEN2K +#ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_trylock((pthread_mutex_t*)lock); #else return pthread_spin_trylock((pthread_spinlock_t*)lock); @@ -326,7 +327,7 @@ int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock) { } int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock) { -#ifndef __USE_XOPEN2K +#ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_unlock((pthread_mutex_t*)lock); #else return pthread_spin_unlock((pthread_spinlock_t*)lock); @@ -337,10 +338,6 @@ void taosThreadTestCancel(void) { return pthread_testcancel(); } -int32_t taosThreadSigMask(int32_t how, sigset_t const *set, sigset_t * oset) { - return pthread_sigmask(how, set, oset); -} - -int32_t taosThreadSigWait(const sigset_t * set, int32_t *sig) { - return sigwait(set, sig); +void taosThreadClear(TdThread *thread) { + memset(thread, 0, sizeof(TdThread)); } \ No newline at end of file diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 9ea49b364e902f48febe0f85f4cf90e8425fe6ad..5d6b4e7c2b840098324c1fea1067356fecfd32a9 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -26,12 +26,11 @@ #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #include #include #include -#include //#define TM_YEAR_BASE 1970 //origin #define TM_YEAR_BASE 1900 //slguan /* @@ -85,13 +84,43 @@ static const char *am_pm[2] = { "AM", "PM" }; +#define BILLION (1E9) + +static BOOL g_first_time = 1; +static LARGE_INTEGER g_counts_per_sec; + +int clock_gettime(int dummy, struct timespec *ct) +{ + LARGE_INTEGER count; + + if (g_first_time) + { + g_first_time = 0; + + if (0 == QueryPerformanceFrequency(&g_counts_per_sec)) + { + g_counts_per_sec.QuadPart = 0; + } + } + + if ((NULL == ct) || (g_counts_per_sec.QuadPart <= 0) || + (0 == QueryPerformanceCounter(&count))) + { + return -1; + } + + ct->tv_sec = count.QuadPart / g_counts_per_sec.QuadPart; + ct->tv_nsec = ((count.QuadPart % g_counts_per_sec.QuadPart) * BILLION) / g_counts_per_sec.QuadPart; + + return 0; +} #else #include #endif char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS char c; const char *bp; size_t len = 0; @@ -391,7 +420,7 @@ char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { } FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS time_t t; t = taosGetTimestampSec(); SYSTEMTIME st; @@ -418,7 +447,7 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result) { if (result == NULL) { return localtime(timep); } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS localtime_s(result, timep); #else localtime_r(timep, result); @@ -427,3 +456,5 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result) { } int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } + +int32_t taosClockGetTime(int clock_id, struct timespec *pTS) { return clock_gettime(clock_id, pTS); } \ No newline at end of file diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index e6e3ec7962192a3aca4da49f9bfe9394d7bdc73b..2992b2909848d4ac175e5557976ec8062a5e8e4d 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -17,7 +17,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #include #include #include @@ -143,7 +143,7 @@ static void * taosProcessAlarmSignal(void *tharg) { #endif int taosInitTimer(void (*callback)(int), int ms) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS DWORD_PTR param = *((int64_t *)&callback); timerId = timeSetEvent(ms, 1, (LPTIMECALLBACK)taosWinOnTimer, param, TIME_PERIODIC); @@ -190,7 +190,7 @@ int taosInitTimer(void (*callback)(int), int ms) { } void taosUninitTimer() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS timeKillEvent(timerId); #elif defined(_TD_DARWIN_64) int r = 0; diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index dc23eaae1a946f1a42a23ed9a28acce097b5b2c2..cbf20f02cd87a44e214e4fa5cfe0b69c8a440b4f 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -17,7 +17,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #if (_WIN64) #include #include @@ -59,11 +59,12 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8 buf[i] = inTimezoneStr[i]; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS char winStr[TD_LOCALE_LEN * 2]; sprintf(winStr, "TZ=%s", buf); putenv(winStr); tzset(); + /* * get CURRENT time zone. * system current time zone is affected by daylight saving time(DST) * @@ -116,7 +117,7 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8 } void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS char *tz = getenv("TZ"); if (tz == NULL || strlen(tz) == 0) { strcpy(outTimezoneStr, "not configured"); diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 8609c6eb217ef6a87046542769cd269450466bc6..8553d714fc832de5aec4de50d63b79702566a7c9 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( ) target_link_libraries( util - PRIVATE os + PUBLIC os PUBLIC lz4_static PUBLIC api cjson ) diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index 48a0f327c4111d89dfd99437fc7c763d2d65b101..8675670cfe1e61da83e79e4e3a0ea665cec597ca 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -229,22 +229,21 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size } void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, - __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot) { + __ext_compar_fn_t compar, char* buf, bool maxroot) { int32_t parent; int32_t child; - char *buf; + + char* tmp = NULL; + if (buf == NULL) { + tmp = taosMemoryMalloc(size); + } else { + tmp = buf; + } if (base && size > 0 && compar) { parent = start; child = 2 * parent + 1; - if (swap == NULL) { - buf = taosMemoryCalloc(1, size); - if (buf == NULL) { - return; - } - } - if (maxroot) { while (child <= end) { if (child + 1 <= end && @@ -256,11 +255,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const break; } - if (swap == NULL) { - doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, buf); - } else { - (*swap)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parswap); - } + doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, tmp); parent = child; child = 2 * parent + 1; @@ -276,33 +271,35 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const break; } - if (swap == NULL) { - doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, buf); - } else { - (*swap)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parswap); - } + doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, tmp); parent = child; child = 2 * parent + 1; } } + } - if (swap == NULL) { - taosMemoryFreeClear(buf); - } + if (buf == NULL) { + taosMemoryFree(tmp); } } void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, - const void *parswap, __ext_swap_fn_t swap, bool maxroot) { + bool maxroot) { int32_t i; + char* buf = taosMemoryCalloc(1, size); + if (buf == NULL) { + return; + } + if (base && size > 0) { for (i = len / 2 - 1; i >= 0; i--) { - taosheapadjust(base, size, i, len - 1, parcompar, compar, parswap, swap, maxroot); + taosheapadjust(base, size, i, len - 1, parcompar, compar, buf, maxroot); } } + taosMemoryFree(buf); /* char *buf = taosMemoryCalloc(1, size); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 4477a5cacdeb1675c16daaedd05b040bddf38203..1c655fc2bf765500f4e4ef5e413590c58511d615 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "tarray.h" +#include "tcoding.h" SArray* taosArrayInit(size_t size, size_t elemSize) { assert(elemSize > 0); @@ -75,7 +76,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { } void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles) { - if (pArray == NULL || pData == NULL) { + if (pData == NULL) { return NULL; } @@ -317,7 +318,6 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)) { pArray->size = 0; } - void* taosArrayDestroy(SArray* pArray) { if (pArray) { taosMemoryFree(pArray->pData); @@ -327,7 +327,14 @@ void* taosArrayDestroy(SArray* pArray) { return NULL; } -void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) { +void taosArrayDestroyP(SArray* pArray, FDelete fp) { + for (int32_t i = 0; i < pArray->size; i++) { + fp(*(void**)TARRAY_GET_ELEM(pArray, i)); + } + taosArrayDestroy(pArray); +} + +void taosArrayDestroyEx(SArray* pArray, FDelete fp) { if (pArray == NULL) { return; } @@ -436,6 +443,39 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void return; } +SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy) { + ASSERT(pSrc->elemSize == sizeof(void*)); + SArray* pArray = taosArrayInit(pSrc->size, sizeof(void*)); + for (int32_t i = 0; i < pSrc->size; i++) { + void* clone = deepCopy(taosArrayGetP(pSrc, i)); + taosArrayPush(pArray, &clone); + } + return pArray; +} + +int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode) { + int32_t tlen = 0; + int32_t sz = pArray->size; + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + void* data = taosArrayGetP(pArray, i); + tlen += encode(buf, data); + } + return tlen; +} + +void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz) { + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + *pArray = taosArrayInit(sz, sizeof(void*)); + for (int32_t i = 0; i < sz; i++) { + void* data = taosMemoryCalloc(1, dataSz); + buf = decode(buf, data); + taosArrayPush(*pArray, &data); + } + return (void*)buf; +} + // order array void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) { taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index a69f11f285053922e1625f602c46560c6ef1a1ec..5f838b9af6d684eacb22025d247c72f2f1cbed5d 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "tcache.h" #include "taoserror.h" +#include "osThread.h" #include "tlog.h" #include "tutil.h" @@ -24,7 +25,7 @@ static TdThread cacheRefreshWorker = {0}; static TdThreadOnce cacheThreadInit = PTHREAD_ONCE_INIT; -static TdThreadMutex guard = PTHREAD_MUTEX_INITIALIZER; +static TdThreadMutex guard = TD_PTHREAD_MUTEX_INITIALIZER; static SArray *pCacheArrayList = NULL; static bool stopRefreshWorker = false; static bool refreshWorkerNormalStopped = false; diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 93022de021e43cade5819c191564000876bbb65c..0ddc76e4156fa344a7662fffb35f97b03b4d12f0 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -222,6 +222,11 @@ int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) { return compareLenPrefixedWStr(pRight, pLeft); } +int32_t compareJsonContainsKey(const void* pLeft, const void* pRight) { + if(pLeft) return 0; + return 1; +} + /* * Compare two strings * TSDB_MATCH: Match diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 06a6b8317dba302c3528c7cae0aac26d315aab9f..eea5c9fecc893b67195aaff0df45870cc10c6959 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -139,7 +139,7 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { return -1; } - if (taosRealPath(fullDir, PATH_MAX) != 0) { + if (taosRealPath(fullDir, NULL, PATH_MAX) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to get realpath of dir:%s since %s", inputDir, terrstr()); return -1; @@ -472,6 +472,8 @@ const char *cfgStypeStr(ECfgSrcType type) { return "apollo_url"; case CFG_STYPE_ARG_LIST: return "arg_list"; + case CFG_STYPE_TAOS_OPTIONS: + return "taos_options"; default: return "invalid"; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ded42365b6e6d188b23cb13751ce3eca68798ccd..9332cb481ecac5ee6febb31da97f4986a17e74f3 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -138,6 +138,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON, "Invalid JSON format") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON_TYPE, "Invalid JSON data type") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_VALUE_OUT_OF_RANGE, "Value out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_INPUT, "Invalid tsc input") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_API_ERROR, "Stmt API usage error") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_TBNAME_ERROR, "Stmt table name not set") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt clause") // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error") diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 6d3a259079d120617dd90c391998f0885f64659e..d2b07875a36357d33309ea07d0b527de7bbd9ec4 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -135,7 +135,7 @@ static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEnt } static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { - int32_t len = MIN(length, HASH_MAX_CAPACITY); + int32_t len = (length < HASH_MAX_CAPACITY ? length : HASH_MAX_CAPACITY); int32_t i = 4; while (i < len) i = (i << 1u); diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 93a843fee8ccbece4ac3e690e1905237b7a34485..59dab76989bacd88e6b56781a08399894d949b88 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -276,3 +276,37 @@ int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void } SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); } + +bool tjsonValidateJson(const char *jIn) { + if (!jIn){ + return false; + } + + // set json real data + cJSON *root = cJSON_Parse(jIn); + if (root == NULL){ + return false; + } + + if(!cJSON_IsObject(root)){ + return false; + } + int size = cJSON_GetArraySize(root); + for(int i = 0; i < size; i++) { + cJSON* item = cJSON_GetArrayItem(root, i); + if (!item) { + return false; + } + + char* jsonKey = item->string; + if (!jsonKey) return false; + for (size_t j = 0; j < strlen(jsonKey); ++i) { + if (isprint(jsonKey[i]) == 0) return false; + } + + if (item->type == cJSON_Object || item->type == cJSON_Array) { + return false; + } + } + return true; +} \ No newline at end of file diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 3dce260b10cd438f30156fc0d6454333bdd85fe2..23fe2e8301ae15acb93a7df9534e0fdd11f7ad9e 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include "os.h" #include "tlog.h" #include "tutil.h" @@ -91,6 +92,7 @@ int32_t sDebugFlag = 135; int32_t tsdbDebugFlag = 131; int32_t tqDebugFlag = 135; int32_t fsDebugFlag = 135; +int32_t fnDebugFlag = 135; int64_t dbgEmptyW = 0; int64_t dbgWN = 0; @@ -221,7 +223,7 @@ static void *taosThreadToOpenNewFile(void *param) { tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; - taosSsleep(3); + taosSsleep(10); taosCloseLogByFd(pOldFile); uInfo(" new log file:%d is opened", tsLogObj.flag); @@ -752,6 +754,7 @@ void taosSetAllDebugFlag(int32_t flag) { tsdbDebugFlag = flag; tqDebugFlag = flag; fsDebugFlag = flag; + fnDebugFlag = flag; uInfo("all debug flag are set to %d", flag); } diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index ae7b3d6f1f9552d91c8ea50faba183260fa7d578..03c9c255b04a89d3130c4fef5fd842f711e4fea2 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -286,13 +286,13 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHea pQueue->head = headLen + bodyLen; } else if (remain < 8 + headLen) { memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, remain - 8); - memcpy(pHead + remain - 8, pQueue->pBuffer, headLen - (remain - 8)); + memcpy((char*)pHead + remain - 8, pQueue->pBuffer, headLen - (remain - 8)); memcpy(pBody, pQueue->pBuffer + headLen - (remain - 8), bodyLen); pQueue->head = headLen - (remain - 8) + bodyLen; } else if (remain < 8 + headLen + bodyLen) { memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); memcpy(pBody, pQueue->pBuffer + pQueue->head + 8 + headLen, remain - 8 - headLen); - memcpy(pBody + remain - 8 - headLen, pQueue->pBuffer, bodyLen - (remain - 8 - headLen)); + memcpy((char*)pBody + remain - 8 - headLen, pQueue->pBuffer, bodyLen - (remain - 8 - headLen)); pQueue->head = bodyLen - (remain - 8 - headLen); } else { memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); @@ -434,7 +434,7 @@ void taosProcStop(SProcObj *pProc) { } tsem_post(&pQueue->sem); taosThreadJoin(pProc->thread, NULL); - pProc->thread = 0; + taosThreadClear(&pProc->thread); } void taosProcCleanup(SProcObj *pProc) { diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 118fe58d2e9a58b3682aeb728b63e1d4f1dd77e1..13637b8fe4971f8ed315a62a01b93a665a4d84a1 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -559,7 +559,7 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) { const uint32_t factor = 4; int32_t n = 1; -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS while ((taosRand() % factor) == 0 && n <= pSkipList->maxLevel) { #else while ((taosRandR(&(pSkipList->seed)) % factor) == 0 && n <= pSkipList->maxLevel) { diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index fecc58c2367c90af8c94e98e029be9874749fd2a..34a59a7d4847cd96bae7368bd66324c865722d13 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -132,7 +132,7 @@ static timer_map_t timerMap; static uintptr_t getNextTimerId() { uintptr_t id; do { - id = (uintptr_t)atomic_add_fetch_ptr((void **)&nextTimerId, 1); + id = (uintptr_t)atomic_add_fetch_ptr((void **)&nextTimerId, (void*)1); } while (id == 0); return id; } diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 8133e4d237f50c525ef653dd064854619e847dcc..adb6a37ba737fbf1ccafe35231853f8a2fc4cfc8 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -47,65 +47,6 @@ int32_t strdequote(char *z) { return j + 1; // only one quote, do nothing } -int32_t strRmquote(char *z, int32_t len) { - // delete escape character: \\, \', \" - char delim = z[0]; - if (delim != '\'' && delim != '\"') { - return len; - } - - int32_t cnt = 0; - int32_t j = 0; - for (uint32_t k = 1; k < len - 1; ++k) { - if (z[k] == '\\' || (z[k] == delim && z[k + 1] == delim)) { - if (z[k] == '\\' && z[k + 1] == '_') { - // match '_' self - } else { - z[j] = z[k + 1]; - cnt++; - j++; - k++; - continue; - } - } - - z[j] = z[k]; - j++; - } - - z[j] = 0; - - return len - 2 - cnt; -} - -int32_t strndequote(char *dst, const char *z, int32_t len) { - assert(dst != NULL); - if (z == NULL || len == 0) { - return 0; - } - - int32_t quote = z[0]; - int32_t i = 1, j = 0; - - while (z[i] != 0) { - if (z[i] == quote) { - if (z[i + 1] == quote) { - dst[j++] = (char)quote; - i++; - } else { - dst[j++] = 0; - return (j - 1); - } - } else { - dst[j++] = z[i]; - } - - i++; - } - - return j + 1; // only one quote, do nothing -} - size_t strtrim(char *z) { int32_t i = 0; int32_t j = 0; diff --git a/source/util/test/freelistTest.cpp b/source/util/test/freelistTest.cpp index 440e997042a4b7f3395f32d5b0eccfe2960a5c0f..a445a16ad360fae3f8bede7bbf28c1c311e21656 100644 --- a/source/util/test/freelistTest.cpp +++ b/source/util/test/freelistTest.cpp @@ -14,4 +14,4 @@ TEST(TD_UTIL_FREELIST_TEST, simple_test) { } tFreeListClear(&fl); -} \ No newline at end of file +} diff --git a/source/util/test/procTest.cpp b/source/util/test/procTest.cpp index e05008e8e14598a3c9f3534811d1707f889f999a..3c014369fbfebaf731486ed22abcb125d8b6accf 100644 --- a/source/util/test/procTest.cpp +++ b/source/util/test/procTest.cpp @@ -63,19 +63,19 @@ TEST_F(UtilTesProc, 00_Init_Cleanup) { ASSERT_EQ(taosCreateShm(&shm, 1234, 1024 * 1024 * 2), 0); shm.size = 1023; - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)NULL, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .childFreeBodyFp = (ProcFreeFp)taosMemoryMalloc, - .parentConsumeFp = (ProcConsumeFp)NULL, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeBodyFp = (ProcFreeFp)taosMemoryMalloc, - .shm = shm, - .parent = &shm, - .name = "1234"}; + SProcCfg cfg = {(ProcConsumeFp)NULL, + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryMalloc, + (ProcConsumeFp)NULL, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryMalloc, + shm, + &shm, + "1234"}; SProcObj *proc = taosProcInit(&cfg); ASSERT_EQ(proc, nullptr); @@ -104,19 +104,19 @@ void ConsumeChild1(void *parent, void *pHead, int16_t headLen, void *pBody, int3 TEST_F(UtilTesProc, 01_Push_Pop_Child) { shm.size = 3000; ASSERT_EQ(taosCreateShm(&shm, 1235, shm.size), 0); - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)ConsumeChild1, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .childFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .parentConsumeFp = (ProcConsumeFp)NULL, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .shm = shm, - .parent = (void *)((int64_t)1235), - .name = "1235_c"}; + SProcCfg cfg = {(ProcConsumeFp)ConsumeChild1, + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcConsumeFp)NULL, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + shm, + (void *)((int64_t)1235), + "1235_c"}; SProcObj *cproc = taosProcInit(&cfg); ASSERT_NE(cproc, nullptr); @@ -162,19 +162,19 @@ void ConsumeParent1(void *parent, void *pHead, int16_t headLen, void *pBody, int TEST_F(UtilTesProc, 02_Push_Pop_Parent) { shm.size = 3000; ASSERT_EQ(taosCreateShm(&shm, 1236, shm.size), 0); - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)NULL, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .childFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .parentConsumeFp = (ProcConsumeFp)ConsumeParent1, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .shm = shm, - .parent = (void *)((int64_t)1236), - .name = "1236_c"}; + SProcCfg cfg = {(ProcConsumeFp)NULL, + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcConsumeFp)ConsumeParent1, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + shm, + (void *)((int64_t)1236), + "1236_c"}; SProcObj *cproc = taosProcInit(&cfg); ASSERT_NE(cproc, nullptr); @@ -216,19 +216,19 @@ TEST_F(UtilTesProc, 03_Handle) { // uDebugFlag = 207; shm.size = 3000; ASSERT_EQ(taosCreateShm(&shm, 1237, shm.size), 0); - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)ConsumeChild3, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .childFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .parentConsumeFp = (ProcConsumeFp)NULL, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .shm = shm, - .parent = (void *)((int64_t)1235), - .name = "1237_p"}; + SProcCfg cfg = {(ProcConsumeFp)ConsumeChild3, + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcConsumeFp)NULL, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + shm, + (void *)((int64_t)1235), + "1237_p"}; SProcObj *cproc = taosProcInit(&cfg); ASSERT_NE(cproc, nullptr); diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 3174d57aef9b50583fc0b28d7bf389572bebdd0c..89561e61fe02e4665828bc8336ec79f6df46fc68 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -36,7 +36,7 @@ void *addRef(void *param) { for (int i=0; i < pSpace->steps; ++i) { printf("a"); - id = random() % pSpace->refNum; + id = taosRand() % pSpace->refNum; if (pSpace->rid[id] <= 0) { pSpace->p[id] = taosMemoryMalloc(128); pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]); @@ -53,7 +53,7 @@ void *removeRef(void *param) { for (int i=0; i < pSpace->steps; ++i) { printf("d"); - id = random() % pSpace->refNum; + id = taosRand() % pSpace->refNum; if (pSpace->rid[id] > 0) { code = taosRemoveRef(pSpace->rsetId, pSpace->rid[id]); if (code == 0) pSpace->rid[id] = 0; @@ -72,7 +72,7 @@ void *acquireRelease(void *param) { for (int i=0; i < pSpace->steps; ++i) { printf("a"); - id = random() % pSpace->refNum; + id = taosRand() % pSpace->refNum; void *p = taosAcquireRef(pSpace->rsetId, (int64_t) pSpace->p[id]); if (p) { taosUsleep(id % 5 + 1); diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py index ffd1d6cb8cb716dabf0e5299d070cf8d7d73daf4..28621f777ba16fed985642551b4db3449204fb2e 100644 --- a/tests/pytest/insert/binary.py +++ b/tests/pytest/insert/binary.py @@ -35,8 +35,8 @@ class TDTestCase: tdSql.prepare() tdLog.info('=============== step1') - tdLog.info('create table tb (ts timestamp, speed binary(5))') - tdSql.execute('create table tb (ts timestamp, speed binary(5))') + tdLog.info('create table tb (ts timestamp, speed binary(10))') + tdSql.execute('create table tb (ts timestamp, speed binary(10))') tdLog.info("insert into tb values (now, ) -x step1") tdSql.error("insert into tb values (now, )") tdLog.info('=============== step2') @@ -49,27 +49,29 @@ class TDTestCase: tdLog.info("tdSql.checkData(0, 0, '1234')") tdSql.checkData(0, 0, '1234') tdLog.info('=============== step3') - tdLog.info("insert into tb values (now+2a, '23456')") - tdSql.execute("insert into tb values (now+2a, '23456')") + tdLog.info("insert into tb values (now+2a, '0123456789')") + tdSql.execute("insert into tb values (now+2a, '0123456789')") tdLog.info('select speed from tb order by ts desc') tdSql.query('select speed from tb order by ts desc') tdLog.info('tdSql.checkRow(2)') tdSql.checkRows(2) tdLog.info('==> $data00') - tdLog.info("tdSql.checkData(0, 0, '23456')") - tdSql.checkData(0, 0, '23456') + tdLog.info("tdSql.checkData(0, 0, '0123456789')") + tdSql.checkData(0, 0, '0123456789') tdLog.info('=============== step4') - tdLog.info("insert into tb values (now+3a, '345678')") - tdSql.error("insert into tb values (now+3a, '345678')") + tdLog.info("insert into tb values (now+3a, '01234567890')") + tdSql.error("insert into tb values (now+3a, '01234567890')") tdLog.info("insert into tb values (now+3a, '34567')") tdSql.execute("insert into tb values (now+3a, '34567')") + tdLog.info("insert into tb values (now+4a, NULL)") + tdSql.execute("insert into tb values (now+4a, NULL)") tdLog.info('select speed from tb order by ts desc') tdSql.query('select speed from tb order by ts desc') - tdLog.info('tdSql.checkRow(3)') - tdSql.checkRows(3) - tdLog.info('==> $data00') - tdLog.info("tdSql.checkData(0, 0, '34567')") - tdSql.checkData(0, 0, '34567') + tdSql.checkRows(4) + tdLog.info("tdSql.checkData(0, 0, '0123456789')") + tdSql.checkData(0, 0, '0123456789') + tdLog.info("tdSql.checkData(3, 0, None)") + tdSql.checkData(3, 0, None) tdLog.info("insert into tb values (now+4a, \"'';\")") if platform.system() == "Linux": diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index de1050850d63a51010167f95f0de850c716732d8..2e046bd3ff123cdc4b24a1bf11c8bbf121a6d5fd 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -5,1056 +5,935 @@ #include #include #include +#include #include #include "../../../include/client/taos.h" +int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; +int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_NCHAR}; + +#define tListLen(x) (sizeof(x) / sizeof((x)[0])) + typedef struct { - TAOS *taos; - int idx; -}T_par; + int64_t* tsData; + bool* boolData; + int8_t* tinyData; + uint8_t* utinyData; + int16_t* smallData; + uint16_t* usmallData; + int32_t* intData; + uint32_t* uintData; + int64_t* bigData; + uint64_t* ubigData; + float* floatData; + double* doubleData; + char* binaryData; + char* isNull; + int32_t* binaryLen; + TAOS_BIND_v2* pBind; + char* sql; + int32_t* colTypes; + int32_t colNum; +} BindData; + +int32_t gVarCharSize = 10; +int32_t gVarCharLen = 5; + +int32_t gExecLoopTimes = 1; // no change +int32_t gFullColNum = tListLen(fullColList); + +int insertMBSETest1(TAOS_STMT *stmt); +int insertMBSETest2(TAOS_STMT *stmt); +int insertMBMETest1(TAOS_STMT *stmt); +int insertMBMETest2(TAOS_STMT *stmt); +int insertMBMETest3(TAOS_STMT *stmt); +int insertMBMETest4(TAOS_STMT *stmt); +int insertMPMETest1(TAOS_STMT *stmt); -void taosMsleep(int mseconds); -unsigned long long getCurrentTime(){ - struct timeval tv; - if (taosGetTimeOfDay(&tv) != 0) { - perror("Failed to get current time in ms"); - exit(EXIT_FAILURE); - } - return (uint64_t)tv.tv_sec * 1000000ULL + (uint64_t)tv.tv_usec; + +typedef struct { + char caseDesc[128]; + int32_t colNum; + int32_t *colList; // full table column list + bool autoCreate; + bool fullCol; + int32_t (*runFn)(TAOS_STMT*); + int32_t tblNum; + int32_t rowNum; + int32_t bindRowNum; + int32_t bindColNum; // equal colNum in full column case + int32_t bindNullNum; + int32_t runTimes; +} CaseCfg; + +CaseCfg gCase[] = { + {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 1}, + {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 1}, + + {"insert:MBSE1-FULL", tListLen(fullColList), fullColList, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 1}, + {"insert:MBSE1-C012", tListLen(fullColList), fullColList, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 1}, + {"insert:MBSE1-C002", tListLen(fullColList), fullColList, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 1}, + + {"insert:MBSE2-FULL", tListLen(fullColList), fullColList, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 1}, + {"insert:MBSE2-C012", tListLen(fullColList), fullColList, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 1}, + {"insert:MBSE2-C002", tListLen(fullColList), fullColList, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 1}, + + {"insert:MBME1-FULL", tListLen(fullColList), fullColList, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 1}, + {"insert:MBME1-C012", tListLen(fullColList), fullColList, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 1}, + {"insert:MBME1-C002", tListLen(fullColList), fullColList, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 1}, + + {"insert:MBME2-FULL", tListLen(fullColList), fullColList, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 1}, + {"insert:MBME2-C012", tListLen(fullColList), fullColList, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 1}, + {"insert:MBME2-C002", tListLen(fullColList), fullColList, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 1}, + + {"insert:MBME3-FULL", tListLen(fullColList), fullColList, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 1}, + {"insert:MBME3-C012", tListLen(fullColList), fullColList, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 1}, + {"insert:MBME3-C002", tListLen(fullColList), fullColList, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 1}, + + {"insert:MBME4-FULL", tListLen(fullColList), fullColList, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 1}, + {"insert:MBME4-C012", tListLen(fullColList), fullColList, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 1}, + {"insert:MBME4-C002", tListLen(fullColList), fullColList, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 1}, + + + {"insert:MPME1-FULL", tListLen(fullColList), fullColList, false, true, insertMPMETest1, 10, 10, 2, 0, 0, 1}, + {"insert:MPME1-C012", tListLen(fullColList), fullColList, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 1}, + +}; + +CaseCfg *gCurCase = NULL; + +typedef struct { + int32_t bindNullNum; + bool autoCreate; + bool checkParamNum; + bool printRes; + bool printCreateTblSql; + bool printInsertSql; + int32_t rowNum; //row num for one table + int32_t bindColNum; + int32_t bindRowNum; //row num for once bind + int32_t bindColTypeNum; + int32_t* bindColTypeList; + int32_t runTimes; +} CaseCtrl; + +CaseCtrl gCaseCtrl = { + .bindNullNum = 0, + .autoCreate = false, + .printCreateTblSql = false, + .printInsertSql = true, + .rowNum = 0, + .bindColNum = 0, + .bindRowNum = 0, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .checkParamNum = false, + .printRes = true, + .runTimes = 0, +}; + +int32_t taosGetTimeOfDay(struct timeval *tv) { + return gettimeofday(tv, NULL); +} +void *taosMemoryMalloc(uint64_t size) { + return malloc(size); } +void *taosMemoryCalloc(int32_t num, int32_t size) { + return calloc(num, size); +} +void taosMemoryFree(const void *ptr) { + if (ptr == NULL) return; + return free((void*)ptr); +} +static int64_t taosGetTimestampMs() { + struct timeval systemTime; + taosGetTimeOfDay(&systemTime); + return (int64_t)systemTime.tv_sec * 1000L + (int64_t)systemTime.tv_usec/1000; +} +static int64_t taosGetTimestampUs() { + struct timeval systemTime; + taosGetTimeOfDay(&systemTime); + return (int64_t)systemTime.tv_sec * 1000000L + (int64_t)systemTime.tv_usec; +} -int stmt_scol_func1(TAOS_STMT *stmt) { - struct { - int64_t ts; - int8_t b; - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - char bin[40]; - char blob[80]; - } v = {0}; - - TAOS_BIND params[10]; - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts); - params[0].buffer = &v.ts; - params[0].length = ¶ms[0].buffer_length; - params[0].is_null = NULL; - - params[1].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[1].buffer_length = sizeof(v.v1); - params[1].buffer = &v.v1; - params[1].length = ¶ms[1].buffer_length; - params[1].is_null = NULL; - - params[2].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[2].buffer_length = sizeof(v.v2); - params[2].buffer = &v.v2; - params[2].length = ¶ms[2].buffer_length; - params[2].is_null = NULL; - - params[3].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[3].buffer_length = sizeof(v.f4); - params[3].buffer = &v.f4; - params[3].length = ¶ms[3].buffer_length; - params[3].is_null = NULL; - - params[4].buffer_type = TSDB_DATA_TYPE_BINARY; - params[4].buffer_length = sizeof(v.bin); - params[4].buffer = v.bin; - params[4].length = ¶ms[4].buffer_length; - params[4].is_null = NULL; - - params[5].buffer_type = TSDB_DATA_TYPE_BINARY; - params[5].buffer_length = sizeof(v.bin); - params[5].buffer = v.bin; - params[5].length = ¶ms[5].buffer_length; - params[5].is_null = NULL; - - char *sql = "insert into ? (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); +bool colExists(TAOS_BIND_v2* pBind, int32_t dataType) { + int32_t i = 0; + while (true) { + if (0 == pBind[i].buffer_type) { + return false; + } + + if (pBind[i].buffer_type == dataType) { + return true; + } + + ++i; } - - for (int zz = 0; zz < 10; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - exit(1); - } - v.ts = 1591060628000 + zz * 10; - for (int i = 0; i < 10; ++i) { - v.ts += 1; - - v.b = (int8_t)(i+zz*10) % 2; - v.v1 = (int8_t)(i+zz*10); - v.v2 = (int16_t)((i+zz*10) * 2); - v.v4 = (int32_t)((i+zz*10) * 4); - v.v8 = (int64_t)((i+zz*10) * 8); - v.f4 = (float)((i+zz*10) * 40); - v.f8 = (double)((i+zz*10) * 80); - for (int j = 0; j < sizeof(v.bin) - 1; ++j) { - v.bin[j] = (char)((i)%10 + '0'); +} + +void generateInsertSQL(BindData *data) { + int32_t len = sprintf(data->sql, "insert into %s ", (gCurCase->tblNum > 1 ? "? " : "t0 ")); + if (!gCurCase->fullCol) { + len += sprintf(data->sql + len, "("); + for (int c = 0; c < gCurCase->bindColNum; ++c) { + if (c) { + len += sprintf(data->sql + len, ","); } + switch (data->pBind[c].buffer_type) { + case TSDB_DATA_TYPE_BOOL: + len += sprintf(data->sql + len, "booldata"); + break; + case TSDB_DATA_TYPE_TINYINT: + len += sprintf(data->sql + len, "tinydata"); + break; + case TSDB_DATA_TYPE_SMALLINT: + len += sprintf(data->sql + len, "smalldata"); + break; + case TSDB_DATA_TYPE_INT: + len += sprintf(data->sql + len, "intdata"); + break; + case TSDB_DATA_TYPE_BIGINT: + len += sprintf(data->sql + len, "bigdata"); + break; + case TSDB_DATA_TYPE_FLOAT: + len += sprintf(data->sql + len, "floatdata"); + break; + case TSDB_DATA_TYPE_DOUBLE: + len += sprintf(data->sql + len, "doubledata"); + break; + case TSDB_DATA_TYPE_VARCHAR: + len += sprintf(data->sql + len, "binarydata"); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + len += sprintf(data->sql + len, "ts"); + break; + case TSDB_DATA_TYPE_NCHAR: + len += sprintf(data->sql + len, "nchardata"); + break; + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(data->sql + len, "utinydata"); + break; + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(data->sql + len, "usmalldata"); + break; + case TSDB_DATA_TYPE_UINT: + len += sprintf(data->sql + len, "uintdata"); + break; + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(data->sql + len, "ubigdata"); + break; + default: + printf("invalid col type:%d", data->pBind[c].buffer_type); + exit(1); + } + } + + len += sprintf(data->sql + len, ") "); + } - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); + len += sprintf(data->sql + len, "values ("); + for (int c = 0; c < gCurCase->bindColNum; ++c) { + if (c) { + len += sprintf(data->sql + len, ","); } + len += sprintf(data->sql + len, "?"); } + len += sprintf(data->sql + len, ")"); - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); + if (gCaseCtrl.printInsertSql) { + printf("SQL: %s\n", data->sql); + } +} + +void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { + if (bindIdx < gCurCase->bindColNum) { + if (gCaseCtrl.bindColTypeNum) { + *dataType = gCaseCtrl.bindColTypeList[colIdx]; + return; + } else if (gCurCase->fullCol) { + *dataType = gCurCase->colList[bindIdx]; + return; + } else if (0 == colIdx) { + *dataType = TSDB_DATA_TYPE_TIMESTAMP; + return; + } else { + while (true) { + *dataType = rand() % (TSDB_DATA_TYPE_MAX - 1) + 1; + if (*dataType == TSDB_DATA_TYPE_JSON || *dataType == TSDB_DATA_TYPE_DECIMAL + || *dataType == TSDB_DATA_TYPE_BLOB || *dataType == TSDB_DATA_TYPE_MEDIUMBLOB + || *dataType == TSDB_DATA_TYPE_VARBINARY) { + continue; + } + + if (colExists(data->pBind, *dataType)) { + continue; + } + + break; + } + } + } else { + *dataType = data->pBind[bindIdx%gCurCase->bindColNum].buffer_type; + } +} + +int32_t prepareColData(BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t colIdx) { + int32_t dataType = TSDB_DATA_TYPE_TIMESTAMP; + + generateDataType(data, bindIdx, colIdx, &dataType); + + switch (dataType) { + case TSDB_DATA_TYPE_BOOL: + data->pBind[bindIdx].buffer_length = sizeof(bool); + data->pBind[bindIdx].buffer = data->boolData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + break; + case TSDB_DATA_TYPE_TINYINT: + data->pBind[bindIdx].buffer_length = sizeof(int8_t); + data->pBind[bindIdx].buffer = data->tinyData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + break; + case TSDB_DATA_TYPE_SMALLINT: + data->pBind[bindIdx].buffer_length = sizeof(int16_t); + data->pBind[bindIdx].buffer = data->smallData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + break; + case TSDB_DATA_TYPE_INT: + data->pBind[bindIdx].buffer_length = sizeof(int32_t); + data->pBind[bindIdx].buffer = data->intData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + break; + case TSDB_DATA_TYPE_BIGINT: + data->pBind[bindIdx].buffer_length = sizeof(int64_t); + data->pBind[bindIdx].buffer = data->bigData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + break; + case TSDB_DATA_TYPE_FLOAT: + data->pBind[bindIdx].buffer_length = sizeof(float); + data->pBind[bindIdx].buffer = data->floatData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + break; + case TSDB_DATA_TYPE_DOUBLE: + data->pBind[bindIdx].buffer_length = sizeof(double); + data->pBind[bindIdx].buffer = data->doubleData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + break; + case TSDB_DATA_TYPE_VARCHAR: + data->pBind[bindIdx].buffer_length = gVarCharSize; + data->pBind[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; + data->pBind[bindIdx].length = data->binaryLen; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + break; + case TSDB_DATA_TYPE_TIMESTAMP: + data->pBind[bindIdx].buffer_length = sizeof(int64_t); + data->pBind[bindIdx].buffer = data->tsData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = NULL; + break; + case TSDB_DATA_TYPE_NCHAR: + data->pBind[bindIdx].buffer_length = gVarCharSize; + data->pBind[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; + data->pBind[bindIdx].length = data->binaryLen; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + break; + case TSDB_DATA_TYPE_UTINYINT: + data->pBind[bindIdx].buffer_length = sizeof(uint8_t); + data->pBind[bindIdx].buffer = data->utinyData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + break; + case TSDB_DATA_TYPE_USMALLINT: + data->pBind[bindIdx].buffer_length = sizeof(uint16_t); + data->pBind[bindIdx].buffer = data->usmallData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + break; + case TSDB_DATA_TYPE_UINT: + data->pBind[bindIdx].buffer_length = sizeof(uint32_t); + data->pBind[bindIdx].buffer = data->uintData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + break; + case TSDB_DATA_TYPE_UBIGINT: + data->pBind[bindIdx].buffer_length = sizeof(uint64_t); + data->pBind[bindIdx].buffer = data->ubigData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + break; + default: + printf("invalid col type:%d", dataType); + exit(1); } + data->pBind[bindIdx].buffer_type = dataType; + data->pBind[bindIdx].num = gCurCase->bindRowNum; + return 0; } - -int stmt_scol_func2(TAOS_STMT *stmt) { - struct { - int64_t ts; - int8_t b; - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - char bin[40]; - char blob[80]; - } v = {0}; +int32_t prepareData(BindData *data) { + static int64_t tsData = 1591060628000; + uint64_t allRowNum = gCurCase->rowNum * gCurCase->tblNum; - TAOS_BIND params[10]; - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts); - params[0].buffer = &v.ts; - params[0].length = ¶ms[0].buffer_length; - params[0].is_null = NULL; - - params[1].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[1].buffer_length = sizeof(v.v1); - params[1].buffer = &v.v1; - params[1].length = ¶ms[1].buffer_length; - params[1].is_null = NULL; - - params[2].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[2].buffer_length = sizeof(v.v2); - params[2].buffer = &v.v2; - params[2].length = ¶ms[2].buffer_length; - params[2].is_null = NULL; - - params[3].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[3].buffer_length = sizeof(v.f4); - params[3].buffer = &v.f4; - params[3].length = ¶ms[3].buffer_length; - params[3].is_null = NULL; - - params[4].buffer_type = TSDB_DATA_TYPE_BINARY; - params[4].buffer_length = sizeof(v.bin); - params[4].buffer = v.bin; - params[4].length = ¶ms[4].buffer_length; - params[4].is_null = NULL; - - params[5].buffer_type = TSDB_DATA_TYPE_BINARY; - params[5].buffer_length = sizeof(v.bin); - params[5].buffer = v.bin; - params[5].length = ¶ms[5].buffer_length; - params[5].is_null = NULL; - - char *sql = "insert into m0 (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + data->colNum = 0; + data->colTypes = taosMemoryCalloc(30, sizeof(int32_t)); + data->sql = taosMemoryCalloc(1, 1024); + data->pBind = taosMemoryCalloc((allRowNum/gCurCase->bindRowNum)*gCurCase->bindColNum, sizeof(TAOS_BIND_v2)); + data->tsData = taosMemoryMalloc(allRowNum * sizeof(int64_t)); + data->boolData = taosMemoryMalloc(allRowNum * sizeof(bool)); + data->tinyData = taosMemoryMalloc(allRowNum * sizeof(int8_t)); + data->utinyData = taosMemoryMalloc(allRowNum * sizeof(uint8_t)); + data->smallData = taosMemoryMalloc(allRowNum * sizeof(int16_t)); + data->usmallData = taosMemoryMalloc(allRowNum * sizeof(uint16_t)); + data->intData = taosMemoryMalloc(allRowNum * sizeof(int32_t)); + data->uintData = taosMemoryMalloc(allRowNum * sizeof(uint32_t)); + data->bigData = taosMemoryMalloc(allRowNum * sizeof(int64_t)); + data->ubigData = taosMemoryMalloc(allRowNum * sizeof(uint64_t)); + data->floatData = taosMemoryMalloc(allRowNum * sizeof(float)); + data->doubleData = taosMemoryMalloc(allRowNum * sizeof(double)); + data->binaryData = taosMemoryMalloc(allRowNum * gVarCharSize); + data->binaryLen = taosMemoryMalloc(allRowNum * sizeof(int32_t)); + if (gCurCase->bindNullNum) { + data->isNull = taosMemoryCalloc(allRowNum, sizeof(char)); } - for (int zz = 0; zz < 10; zz++) { - v.ts = 1591060628000 + zz * 10; - for (int i = 0; i < 10; ++i) { - v.ts += 1; - - v.b = (int8_t)(i+zz*10) % 2; - v.v1 = (int8_t)(i+zz*10); - v.v2 = (int16_t)((i+zz*10) * 2); - v.v4 = (int32_t)((i+zz*10) * 4); - v.v8 = (int64_t)((i+zz*10) * 8); - v.f4 = (float)((i+zz*10) * 40); - v.f8 = (double)((i+zz*10) * 80); - for (int j = 0; j < sizeof(v.bin) - 1; ++j) { - v.bin[j] = (char)((i)%10 + '0'); - } + for (int32_t i = 0; i < allRowNum; ++i) { + data->tsData[i] = tsData++; + data->boolData[i] = i % 2; + data->tinyData[i] = i; + data->utinyData[i] = i+1; + data->smallData[i] = i; + data->usmallData[i] = i+1; + data->intData[i] = i; + data->uintData[i] = i+1; + data->bigData[i] = i; + data->ubigData[i] = i+1; + data->floatData[i] = i; + data->doubleData[i] = i+1; + memset(data->binaryData + gVarCharSize * i, 'a'+i%26, gVarCharLen); + if (gCurCase->bindNullNum) { + data->isNull[i] = i % 2; + } + data->binaryLen[i] = gVarCharLen; + } - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); + for (int b = 0; b < (allRowNum/gCurCase->bindRowNum); b++) { + for (int c = 0; c < gCurCase->bindColNum; ++c) { + prepareColData(data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); } } - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); + generateInsertSQL(data); + + return 0; +} + +void destroyData(BindData *data) { + taosMemoryFree(data->tsData); + taosMemoryFree(data->boolData); + taosMemoryFree(data->tinyData); + taosMemoryFree(data->utinyData); + taosMemoryFree(data->smallData); + taosMemoryFree(data->usmallData); + taosMemoryFree(data->intData); + taosMemoryFree(data->uintData); + taosMemoryFree(data->bigData); + taosMemoryFree(data->ubigData); + taosMemoryFree(data->floatData); + taosMemoryFree(data->doubleData); + taosMemoryFree(data->binaryData); + taosMemoryFree(data->binaryLen); + taosMemoryFree(data->isNull); + taosMemoryFree(data->pBind); + taosMemoryFree(data->colTypes); +} + +int32_t bpBindParam(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { + static int32_t n = 0; + + if (gCurCase->bindRowNum > 1) { + if (0 == (n++%2)) { + if (taos_stmt_bind_param_batch(stmt, bind)) { + printf("taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } else { + for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { + if (taos_stmt_bind_single_param_batch(stmt, bind++, i)) { + printf("taos_stmt_bind_single_param_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + } + } else { + if (taos_stmt_bind_param(stmt, bind)) { + printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } } return 0; } +void bpCheckIsInsert(TAOS_STMT *stmt) { + int32_t isInsert = 0; + if (taos_stmt_is_insert(stmt, &isInsert)) { + printf("taos_stmt_is_insert error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + if (0 == isInsert) { + printf("is insert failed\n"); + exit(1); + } +} +void bpCheckParamNum(TAOS_STMT *stmt) { + int32_t num = 0; + if (taos_stmt_num_params(stmt, &num)) { + printf("taos_stmt_num_params error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } -//300 tables 60 records -int stmt_scol_func3(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; + if (gCurCase->bindColNum != num) { + printf("is insert failed\n"); + exit(1); + } +} - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); +void bpCheckAffectedRows(TAOS_STMT *stmt, int32_t times) { + int32_t rows = taos_stmt_affected_rows(stmt); + int32_t insertNum = gCurCase->rowNum * gCurCase->tblNum * times; + if (insertNum != rows) { + printf("affected rows %d mis-match with insert num %d\n", rows, insertNum); + exit(1); + } +} - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); + +/* prepare [settbname [bind add]] exec */ +int insertMBSETest1(TAOS_STMT *stmt) { + BindData data = {0}; + prepareData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; + + bpCheckIsInsert(stmt); + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } - params[i+1].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.v1; - params[i+1].length = NULL; - params[i+1].is_null = no_null; - params[i+1].num = 10; + for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + } - params[i+2].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+2].buffer_length = sizeof(int16_t); - params[i+2].buffer = v.v2; - params[i+2].length = NULL; - params[i+2].is_null = no_null; - params[i+2].num = 10; + if (taos_stmt_execute(stmt) != 0) { + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } - params[i+3].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+3].buffer_length = sizeof(float); - params[i+3].buffer = v.f4; - params[i+3].length = NULL; - params[i+3].is_null = no_null; - params[i+3].num = 10; + bpCheckIsInsert(stmt); + bpCheckAffectedRows(stmt, 1); - params[i+4].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+4].buffer_length = 40; - params[i+4].buffer = v.bin; - params[i+4].length = lb; - params[i+4].is_null = no_null; - params[i+4].num = 10; + destroyData(&data); - params[i+5].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+5].buffer_length = 40; - params[i+5].buffer = v.bin; - params[i+5].length = lb; - params[i+5].is_null = no_null; - params[i+5].num = 10; + return 0; +} + + +/* prepare [settbname bind add] exec */ +int insertMBSETest2(TAOS_STMT *stmt) { + BindData data = {0}; + prepareData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckIsInsert(stmt); + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + + for (int32_t b = 0; b tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + exit(1); + } + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + if (taos_stmt_add_batch(stmt)) { + printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } } - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; + if (taos_stmt_execute(stmt) != 0) { + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); } - unsigned long long starttime = getCurrentTime(); + bpCheckIsInsert(stmt); + bpCheckAffectedRows(stmt, 1); - char *sql = "insert into ? (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); + destroyData(&data); + + return 0; +} + +/* prepare [settbname [bind add] exec] */ +int insertMBMETest1(TAOS_STMT *stmt) { + BindData data = {0}; + prepareData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); } - int id = 0; - for (int l = 0; l < 2; l++) { - for (int zz = 0; zz < 300; zz++) { + bpCheckIsInsert(stmt); + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + if (gCurCase->tblNum > 1) { char buf[32]; - sprintf(buf, "m%d", zz); + sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); } + } - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); } - + + for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } - - ++id; } - unsigned long long endtime = getCurrentTime(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + bpCheckIsInsert(stmt); + bpCheckAffectedRows(stmt, 1); - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); + destroyData(&data); return 0; } +/* prepare [settbname [bind add exec]] */ +int insertMBMETest2(TAOS_STMT *stmt) { + BindData data = {0}; + prepareData(&data); + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } -//10 tables 10 records single column bind -int stmt_scol_func4(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 2; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = no_null; - params[i+1].num = 2; - - params[i+2].buffer_type = TSDB_DATA_TYPE_INT; - params[i+2].buffer_length = sizeof(int32_t); - params[i+2].buffer = v.v4; - params[i+2].length = NULL; - params[i+2].is_null = no_null; - params[i+2].num = 2; - - params[i+3].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+3].buffer_length = sizeof(int64_t); - params[i+3].buffer = v.v8; - params[i+3].length = NULL; - params[i+3].is_null = no_null; - params[i+3].num = 2; + bpCheckIsInsert(stmt); - params[i+4].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+4].buffer_length = sizeof(double); - params[i+4].buffer = v.f8; - params[i+4].length = NULL; - params[i+4].is_null = no_null; - params[i+4].num = 2; - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 60000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = getCurrentTime(); - - char *sql = "insert into ? (ts,b,v4,v8,f8) values(?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 10; l++) { - for (int zz = 0; zz < 10; zz++) { + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + if (gCurCase->tblNum > 1) { char buf[32]; - sprintf(buf, "m%d", zz); + sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); } - - for (int col=0; col < 10; ++col) { - taos_stmt_bind_single_param_batch(stmt, params + id++, col); - } - - taos_stmt_add_batch(stmt); } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - } - - unsigned long long endtime = getCurrentTime(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - - -int stmt_func1(TAOS_STMT *stmt) { - struct { - int64_t ts; - int8_t b; - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - char bin[40]; - char blob[80]; - } v = {0}; - - TAOS_BIND params[10]; - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts); - params[0].buffer = &v.ts; - params[0].length = ¶ms[0].buffer_length; - params[0].is_null = NULL; - - params[1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[1].buffer_length = sizeof(v.b); - params[1].buffer = &v.b; - params[1].length = ¶ms[1].buffer_length; - params[1].is_null = NULL; - - params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[2].buffer_length = sizeof(v.v1); - params[2].buffer = &v.v1; - params[2].length = ¶ms[2].buffer_length; - params[2].is_null = NULL; - - params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[3].buffer_length = sizeof(v.v2); - params[3].buffer = &v.v2; - params[3].length = ¶ms[3].buffer_length; - params[3].is_null = NULL; - - params[4].buffer_type = TSDB_DATA_TYPE_INT; - params[4].buffer_length = sizeof(v.v4); - params[4].buffer = &v.v4; - params[4].length = ¶ms[4].buffer_length; - params[4].is_null = NULL; - - params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[5].buffer_length = sizeof(v.v8); - params[5].buffer = &v.v8; - params[5].length = ¶ms[5].buffer_length; - params[5].is_null = NULL; - - params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[6].buffer_length = sizeof(v.f4); - params[6].buffer = &v.f4; - params[6].length = ¶ms[6].buffer_length; - params[6].is_null = NULL; - - params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[7].buffer_length = sizeof(v.f8); - params[7].buffer = &v.f8; - params[7].length = ¶ms[7].buffer_length; - params[7].is_null = NULL; - - params[8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[8].buffer_length = sizeof(v.bin); - params[8].buffer = v.bin; - params[8].length = ¶ms[8].buffer_length; - params[8].is_null = NULL; - - params[9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[9].buffer_length = sizeof(v.bin); - params[9].buffer = v.bin; - params[9].length = ¶ms[9].buffer_length; - params[9].is_null = NULL; - - int is_null = 1; + + for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + exit(1); + } - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - for (int zz = 0; zz < 10; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - v.ts = 1591060628000 + zz * 10; - for (int i = 0; i < 10; ++i) { - v.ts += 1; - for (int j = 1; j < 10; ++j) { - params[j].is_null = ((i == j) ? &is_null : 0); + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); } - v.b = (int8_t)(i+zz*10) % 2; - v.v1 = (int8_t)(i+zz*10); - v.v2 = (int16_t)((i+zz*10) * 2); - v.v4 = (int32_t)((i+zz*10) * 4); - v.v8 = (int64_t)((i+zz*10) * 8); - v.f4 = (float)((i+zz*10) * 40); - v.f8 = (double)((i+zz*10) * 80); - for (int j = 0; j < sizeof(v.bin) - 1; ++j) { - v.bin[j] = (char)((i+zz)%10 + '0'); + + if (taos_stmt_add_batch(stmt)) { + printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); } - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); + if (taos_stmt_execute(stmt) != 0) { + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } } } - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } + bpCheckIsInsert(stmt); + bpCheckAffectedRows(stmt, 1); + + destroyData(&data); return 0; } +/* prepare [settbname [settbname bind add exec]] */ +int insertMBMETest3(TAOS_STMT *stmt) { + BindData data = {0}; + prepareData(&data); -int stmt_func2(TAOS_STMT *stmt) { - struct { - int64_t ts; - int8_t b; - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - char bin[40]; - char blob[80]; - } v = {0}; - - TAOS_BIND params[10]; - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts); - params[0].buffer = &v.ts; - params[0].length = ¶ms[0].buffer_length; - params[0].is_null = NULL; - - params[1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[1].buffer_length = sizeof(v.b); - params[1].buffer = &v.b; - params[1].length = ¶ms[1].buffer_length; - params[1].is_null = NULL; - - params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[2].buffer_length = sizeof(v.v1); - params[2].buffer = &v.v1; - params[2].length = ¶ms[2].buffer_length; - params[2].is_null = NULL; - - params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[3].buffer_length = sizeof(v.v2); - params[3].buffer = &v.v2; - params[3].length = ¶ms[3].buffer_length; - params[3].is_null = NULL; - - params[4].buffer_type = TSDB_DATA_TYPE_INT; - params[4].buffer_length = sizeof(v.v4); - params[4].buffer = &v.v4; - params[4].length = ¶ms[4].buffer_length; - params[4].is_null = NULL; - - params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[5].buffer_length = sizeof(v.v8); - params[5].buffer = &v.v8; - params[5].length = ¶ms[5].buffer_length; - params[5].is_null = NULL; - - params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[6].buffer_length = sizeof(v.f4); - params[6].buffer = &v.f4; - params[6].length = ¶ms[6].buffer_length; - params[6].is_null = NULL; - - params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[7].buffer_length = sizeof(v.f8); - params[7].buffer = &v.f8; - params[7].length = ¶ms[7].buffer_length; - params[7].is_null = NULL; - - params[8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[8].buffer_length = sizeof(v.bin); - params[8].buffer = v.bin; - params[8].length = ¶ms[8].buffer_length; - params[8].is_null = NULL; - - params[9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[9].buffer_length = sizeof(v.bin); - params[9].buffer = v.bin; - params[9].length = ¶ms[9].buffer_length; - params[9].is_null = NULL; - - int is_null = 1; - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); + int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); } - for (int l = 0; l < 100; l++) { - for (int zz = 0; zz < 10; zz++) { + bpCheckIsInsert(stmt); + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + if (gCurCase->tblNum > 1) { char buf[32]; - sprintf(buf, "m%d", zz); + sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); } - v.ts = 1591060628000 + zz * 100 * l; - for (int i = 0; i < zz; ++i) { - v.ts += 1; - for (int j = 1; j < 10; ++j) { - params[j].is_null = ((i == j) ? &is_null : 0); - } - v.b = (int8_t)(i+zz*10) % 2; - v.v1 = (int8_t)(i+zz*10); - v.v2 = (int16_t)((i+zz*10) * 2); - v.v4 = (int32_t)((i+zz*10) * 4); - v.v8 = (int64_t)((i+zz*10) * 8); - v.f4 = (float)((i+zz*10) * 40); - v.f8 = (double)((i+zz*10) * 80); - for (int j = 0; j < sizeof(v.bin) - 1; ++j) { - v.bin[j] = (char)((i+zz)%10 + '0'); - } - - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); - } } - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); } + + for (int32_t b = 0; b tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } - } - - - return 0; -} - - - - -int stmt_func3(TAOS_STMT *stmt) { - struct { - int64_t ts; - int8_t b; - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - char bin[40]; - char blob[80]; - } v = {0}; - - TAOS_BIND params[10]; - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts); - params[0].buffer = &v.ts; - params[0].length = ¶ms[0].buffer_length; - params[0].is_null = NULL; - - params[1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[1].buffer_length = sizeof(v.b); - params[1].buffer = &v.b; - params[1].length = ¶ms[1].buffer_length; - params[1].is_null = NULL; - - params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[2].buffer_length = sizeof(v.v1); - params[2].buffer = &v.v1; - params[2].length = ¶ms[2].buffer_length; - params[2].is_null = NULL; - - params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[3].buffer_length = sizeof(v.v2); - params[3].buffer = &v.v2; - params[3].length = ¶ms[3].buffer_length; - params[3].is_null = NULL; - - params[4].buffer_type = TSDB_DATA_TYPE_INT; - params[4].buffer_length = sizeof(v.v4); - params[4].buffer = &v.v4; - params[4].length = ¶ms[4].buffer_length; - params[4].is_null = NULL; - - params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[5].buffer_length = sizeof(v.v8); - params[5].buffer = &v.v8; - params[5].length = ¶ms[5].buffer_length; - params[5].is_null = NULL; - - params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[6].buffer_length = sizeof(v.f4); - params[6].buffer = &v.f4; - params[6].length = ¶ms[6].buffer_length; - params[6].is_null = NULL; - - params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[7].buffer_length = sizeof(v.f8); - params[7].buffer = &v.f8; - params[7].length = ¶ms[7].buffer_length; - params[7].is_null = NULL; - - params[8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[8].buffer_length = sizeof(v.bin); - params[8].buffer = v.bin; - params[8].length = ¶ms[8].buffer_length; - params[8].is_null = NULL; - - params[9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[9].buffer_length = sizeof(v.bin); - params[9].buffer = v.bin; - params[9].length = ¶ms[9].buffer_length; - params[9].is_null = NULL; - - int is_null = 1; - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - for (int l = 0; l < 100; l++) { - for (int zz = 0; zz < 10; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - v.ts = 1591060628000 + zz * 100 * l; - for (int i = 0; i < zz; ++i) { - v.ts += 1; - for (int j = 1; j < 10; ++j) { - params[j].is_null = ((i == j) ? &is_null : 0); - } - v.b = (int8_t)(i+zz*10) % 2; - v.v1 = (int8_t)(i+zz*10); - v.v2 = (int16_t)((i+zz*10) * 2); - v.v4 = (int32_t)((i+zz*10) * 4); - v.v8 = (int64_t)((i+zz*10) * 8); - v.f4 = (float)((i+zz*10) * 40); - v.f8 = (double)((i+zz*10) * 80); - for (int j = 0; j < sizeof(v.bin) - 1; ++j) { - v.bin[j] = (char)((i+zz)%10 + '0'); - } - - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); + if (taos_stmt_execute(stmt) != 0) { + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); } } } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } + bpCheckIsInsert(stmt); + bpCheckAffectedRows(stmt, 1); + destroyData(&data); return 0; } +/* prepare [settbname bind add exec] */ +int insertMBMETest4(TAOS_STMT *stmt) { + BindData data = {0}; + prepareData(&data); -//1 tables 10 records -int stmt_funcb_autoctb1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); - -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; + bpCheckIsInsert(stmt); - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; + for (int32_t b = 0; b tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + exit(1); + } - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + + if (taos_stmt_add_batch(stmt)) { + printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; + if (taos_stmt_execute(stmt) != 0) { + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } } + bpCheckIsInsert(stmt); + bpCheckAffectedRows(stmt, 1); - unsigned long long starttime = getCurrentTime(); + destroyData(&data); - char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - exit(1); - } + return 0; +} - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, tags); +/* [prepare [settbname [bind add] exec]] */ +int insertMPMETest1(TAOS_STMT *stmt) { + int32_t loop = 0; + + while (gCurCase->bindColNum >= 2) { + BindData data = {0}; + prepareData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); - } + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } + bpCheckIsInsert(stmt); + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + + for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } - ++id; + if (taos_stmt_execute(stmt) != 0) { + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } - unsigned long long endtime = getCurrentTime(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); + bpCheckIsInsert(stmt); - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); + destroyData(&data); + gCurCase->bindColNum -= 2; + gCurCase->fullCol = false; + loop++; + } + + bpCheckAffectedRows(stmt, loop); + + gExecLoopTimes = loop; + return 0; } - +#if 0 //1 tables 10 records -int stmt_funcb_autoctb2(TAOS_STMT *stmt) { +int stmt_funcb_autoctb1(TAOS_STMT *stmt) { struct { int64_t *ts; int8_t b[10]; @@ -1072,7 +951,7 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -1221,9 +1100,9 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); - char *sql = "insert into ? using stb1 tags(1,true,2,3,4,5.0,6.0,'a','b') values(?,?,?,?,?,?,?,?,?,?)"; + char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); @@ -1250,7 +1129,7 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -1266,9 +1145,8 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { - //1 tables 10 records -int stmt_funcb_autoctb3(TAOS_STMT *stmt) { +int stmt_funcb_autoctb2(TAOS_STMT *stmt) { struct { int64_t *ts; int8_t b[10]; @@ -1286,7 +1164,7 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -1329,211 +1207,56 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { params[i+2].length = NULL; params[i+2].is_null = is_null; params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+0].buffer = v.b; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+1].buffer = v.v2; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+2].buffer = v.f4; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+3].buffer = v.bin; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = (uintptr_t *)lb; - } - - - unsigned long long starttime = getCurrentTime(); - - char *sql = "insert into ? using stb1 tags(1,?,2,?,4,?,6.0,?,'b') values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - exit(1); - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, tags); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - - unsigned long long endtime = getCurrentTime(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); - - return 0; -} - - - - - - - -//1 tables 10 records -int stmt_funcb_autoctb4(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*5); - -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 5; i+=5) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_INT; - params[i+2].buffer_length = sizeof(int32_t); - params[i+2].buffer = v.v4; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+3].buffer_length = sizeof(int64_t); - params[i+3].buffer = v.v8; + + params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[i+3].buffer_length = sizeof(int16_t); + params[i+3].buffer = v.v2; params[i+3].length = NULL; params[i+3].is_null = is_null; params[i+3].num = 10; - params[i+4].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+4].buffer_length = sizeof(double); - params[i+4].buffer = v.f8; + params[i+4].buffer_type = TSDB_DATA_TYPE_INT; + params[i+4].buffer_length = sizeof(int32_t); + params[i+4].buffer = v.v4; params[i+4].length = NULL; params[i+4].is_null = is_null; params[i+4].num = 10; + + params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+5].buffer_length = sizeof(int64_t); + params[i+5].buffer = v.v8; + params[i+5].length = NULL; + params[i+5].is_null = is_null; + params[i+5].num = 10; + + params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+6].buffer_length = sizeof(float); + params[i+6].buffer = v.f4; + params[i+6].length = NULL; + params[i+6].is_null = is_null; + params[i+6].num = 10; + + params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+7].buffer_length = sizeof(double); + params[i+7].buffer = v.f8; + params[i+7].length = NULL; + params[i+7].is_null = is_null; + params[i+7].num = 10; + + params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+8].buffer_length = 40; + params[i+8].buffer = v.bin; + params[i+8].length = lb; + params[i+8].is_null = is_null; + params[i+8].num = 10; + + params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+9].buffer_length = 40; + params[i+9].buffer = v.bin; + params[i+9].length = lb; + params[i+9].is_null = is_null; + params[i+9].num = 10; + } int64_t tts = 1591060628000; @@ -1543,31 +1266,56 @@ int stmt_funcb_autoctb4(TAOS_STMT *stmt) { for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+0].buffer = v.b; + tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; + tags[i+0].buffer = v.v4; tags[i+0].is_null = &one_not_null; tags[i+0].length = NULL; - tags[i+1].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+1].buffer = v.v2; + tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + tags[i+1].buffer = v.b; tags[i+1].is_null = &one_not_null; tags[i+1].length = NULL; - tags[i+2].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+2].buffer = v.f4; + tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + tags[i+2].buffer = v.v1; tags[i+2].is_null = &one_not_null; tags[i+2].length = NULL; - tags[i+3].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+3].buffer = v.bin; + tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + tags[i+3].buffer = v.v2; tags[i+3].is_null = &one_not_null; - tags[i+3].length = (uintptr_t *)lb; + tags[i+3].length = NULL; + + tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; + tags[i+4].buffer = v.v8; + tags[i+4].is_null = &one_not_null; + tags[i+4].length = NULL; + + tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; + tags[i+5].buffer = v.f4; + tags[i+5].is_null = &one_not_null; + tags[i+5].length = NULL; + + tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; + tags[i+6].buffer = v.f8; + tags[i+6].is_null = &one_not_null; + tags[i+6].length = NULL; + + tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[i+7].buffer = v.bin; + tags[i+7].is_null = &one_not_null; + tags[i+7].length = (uintptr_t *)lb; + + tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; + tags[i+8].buffer = v.bin; + tags[i+8].is_null = &one_not_null; + tags[i+8].length = (uintptr_t *)lb; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); - char *sql = "insert into ? using stb1 tags(1,?,2,?,4,?,6.0,?,'b') (ts,b,v4,v8,f8) values(?,?,?,?,?)"; + char *sql = "insert into ? using stb1 tags(1,true,2,3,4,5.0,6.0,'a','b') values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); @@ -1583,7 +1331,7 @@ int stmt_funcb_autoctb4(TAOS_STMT *stmt) { printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); } - taos_stmt_bind_param_batch(stmt, params + id * 5); + taos_stmt_bind_param_batch(stmt, params + id * 10); taos_stmt_add_batch(stmt); } @@ -1594,7 +1342,7 @@ int stmt_funcb_autoctb4(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -1612,7 +1360,7 @@ int stmt_funcb_autoctb4(TAOS_STMT *stmt) { //1 tables 10 records -int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { +int stmt_funcb_autoctb3(TAOS_STMT *stmt) { struct { int64_t *ts; int8_t b[10]; @@ -1630,7 +1378,7 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -1754,13 +1502,13 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); - char *sql = "insert into ? using stb1 (id1,id2,id3,id4,id5,id6,id7,id8,id9) tags(1,?,2,?,4,?,6.0,?,'b') values(?,?,?,?,?,?,?,?,?,?)"; + char *sql = "insert into ? using stb1 tags(1,?,2,?,4,?,6.0,?,'b') values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); - return -1; + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + exit(1); } int id = 0; @@ -1783,7 +1531,7 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -1800,8 +1548,10 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { + + //1 tables 10 records -int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { +int stmt_funcb_autoctb4(TAOS_STMT *stmt) { struct { int64_t *ts; int8_t b[10]; @@ -1819,7 +1569,7 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*5); // int one_null = 1; int one_not_null = 0; @@ -1841,7 +1591,7 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { memset(v.bin[i], '0'+i%10, 40); } - for (int i = 0; i < 10; i+=10) { + for (int i = 0; i < 5; i+=5) { params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[i+0].buffer_length = sizeof(int64_t); params[i+0].buffer = &v.ts[10*i/10]; @@ -1856,62 +1606,26 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { params[i+1].is_null = is_null; params[i+1].num = 10; - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; + params[i+2].buffer_type = TSDB_DATA_TYPE_INT; + params[i+2].buffer_length = sizeof(int32_t); + params[i+2].buffer = v.v4; params[i+2].length = NULL; params[i+2].is_null = is_null; params[i+2].num = 10; - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; + params[i+3].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[i+3].buffer_length = sizeof(int64_t); + params[i+3].buffer = v.v8; params[i+3].length = NULL; params[i+3].is_null = is_null; params[i+3].num = 10; - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; + params[i+4].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[i+4].buffer_length = sizeof(double); + params[i+4].buffer = v.f8; params[i+4].length = NULL; params[i+4].is_null = is_null; params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - } int64_t tts = 1591060628000; @@ -1921,56 +1635,31 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; + tags[i+0].buffer_type = TSDB_DATA_TYPE_BOOL; + tags[i+0].buffer = v.b; tags[i+0].is_null = &one_not_null; tags[i+0].length = NULL; - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; + tags[i+1].buffer_type = TSDB_DATA_TYPE_SMALLINT; + tags[i+1].buffer = v.v2; tags[i+1].is_null = &one_not_null; tags[i+1].length = NULL; - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; + tags[i+2].buffer_type = TSDB_DATA_TYPE_FLOAT; + tags[i+2].buffer = v.f4; tags[i+2].is_null = &one_not_null; tags[i+2].length = NULL; - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; + tags[i+3].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[i+3].buffer = v.bin; tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; + tags[i+3].length = (uintptr_t *)lb; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); - char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; + char *sql = "insert into ? using stb1 tags(1,?,2,?,4,?,6.0,?,'b') (ts,b,v4,v8,f8) values(?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); @@ -1981,13 +1670,12 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { for (int zz = 0; zz < 1; zz++) { char buf[32]; sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, NULL); + code = taos_stmt_set_tbname_tags(stmt, buf, tags); if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. code:%s\n", taos_stmt_errstr(stmt)); - return -1; + printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); } - taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_bind_param_batch(stmt, params + id * 5); taos_stmt_add_batch(stmt); } @@ -1998,7 +1686,7 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -2015,10 +1703,8 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { - - //1 tables 10 records -int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { +int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { struct { int64_t *ts; int8_t b[10]; @@ -2036,7 +1722,7 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -2138,71 +1824,44 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; + tags[i+0].buffer_type = TSDB_DATA_TYPE_BOOL; + tags[i+0].buffer = v.b; tags[i+0].is_null = &one_not_null; tags[i+0].length = NULL; - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; + tags[i+1].buffer_type = TSDB_DATA_TYPE_SMALLINT; + tags[i+1].buffer = v.v2; tags[i+1].is_null = &one_not_null; tags[i+1].length = NULL; - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; + tags[i+2].buffer_type = TSDB_DATA_TYPE_FLOAT; + tags[i+2].buffer = v.f4; + tags[i+2].is_null = &one_not_null; + tags[i+2].length = NULL; - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; + tags[i+3].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[i+3].buffer = v.bin; + tags[i+3].is_null = &one_not_null; + tags[i+3].length = (uintptr_t *)lb; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); - char *sql = "insert into ? using stb1 (id1,id2,id3,id4,id5,id6,id7,id8,id9) tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; + char *sql = "insert into ? using stb1 (id1,id2,id3,id4,id5,id6,id7,id8,id9) tags(1,?,2,?,4,?,6.0,?,'b') values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt)); + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); return -1; - //exit(1); } int id = 0; for (int zz = 0; zz < 1; zz++) { char buf[32]; sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, NULL); + code = taos_stmt_set_tbname_tags(stmt, buf, tags); if (code != 0){ printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); - return -1; } taos_stmt_bind_param_batch(stmt, params + id * 10); @@ -2216,7 +1875,7 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -2232,8 +1891,9 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { + //1 tables 10 records -int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { +int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { struct { int64_t *ts; int8_t b[10]; @@ -2251,7 +1911,7 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -2400,12 +2060,12 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt)); + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); exit(1); } @@ -2413,24 +2073,13 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { for (int zz = 0; zz < 1; zz++) { char buf[32]; sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, tags); + code = taos_stmt_set_tbname_tags(stmt, buf, NULL); if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. error:%s\n", taos_stmt_errstr(stmt)); - exit(1); + printf("failed to execute taos_stmt_set_tbname_tags. code:%s\n", taos_stmt_errstr(stmt)); + return -1; } - code = taos_stmt_bind_param_batch(stmt, params + id * 10); - if (code != 0) { - printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - - code = taos_stmt_bind_param_batch(stmt, params + id * 10); - if (code != 0) { - printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); - return -1; - } - + taos_stmt_bind_param_batch(stmt, params + id * 10); taos_stmt_add_batch(stmt); } @@ -2441,7 +2090,7 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -2459,8 +2108,9 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { + //1 tables 10 records -int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { +int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { struct { int64_t *ts; int8_t b[10]; @@ -2478,7 +2128,7 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -2627,37 +2277,27 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); - char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(NULL, sql, 0); + char *sql = "insert into ? using stb1 (id1,id2,id3,id4,id5,id6,id7,id8,id9) tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(NULL)); + printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt)); return -1; + //exit(1); } int id = 0; for (int zz = 0; zz < 1; zz++) { char buf[32]; sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, tags); + code = taos_stmt_set_tbname_tags(stmt, buf, NULL); if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. error:%s\n", taos_stmt_errstr(stmt)); - exit(1); + printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); + return -1; } - code = taos_stmt_bind_param_batch(stmt, params + id * 10); - if (code != 0) { - printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - - code = taos_stmt_bind_param_batch(stmt, params + id * 10); - if (code != 0) { - printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); - return -1; - } - + taos_stmt_bind_param_batch(stmt, params + id * 10); taos_stmt_add_batch(stmt); } @@ -2668,7 +2308,7 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -2683,186 +2323,35 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { -//300 tables 60 records -int stmt_funcb1(TAOS_STMT *stmt) { + +//1 tables 10 records +int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { struct { int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; } v = {0}; - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = getCurrentTime(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - } - - unsigned long long endtime = getCurrentTime(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - + int *lb = taosMemoryMalloc(10 * sizeof(int)); -//1table 18000 reocrds -int stmt_funcb2(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[18000]; - int8_t v1[18000]; - int16_t v2[18000]; - int32_t v4[18000]; - int64_t v8[18000]; - float f4[18000]; - double f8[18000]; - char bin[18000][40]; - } v = {0}; + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(18000 * sizeof(int)); +// int one_null = 1; + int one_not_null = 0; - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 18000); - char* no_null = taosMemoryMalloc(sizeof(char) * 18000); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); - for (int i = 0; i < 18000; ++i) { + for (int i = 0; i < 10; ++i) { lb[i] = 40; no_null[i] = 0; is_null[i] = (i % 10 == 2) ? 1 : 0; @@ -2876,151 +2365,220 @@ int stmt_funcb2(TAOS_STMT *stmt) { memset(v.bin[i], '0'+i%10, 40); } - for (int i = 0; i < 30000; i+=10) { + for (int i = 0; i < 10; i+=10) { params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[18000*i/10]; + params[i+0].buffer = &v.ts[10*i/10]; params[i+0].length = NULL; params[i+0].is_null = no_null; - params[i+0].num = 18000; + params[i+0].num = 10; params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; params[i+1].buffer_length = sizeof(int8_t); params[i+1].buffer = v.b; params[i+1].length = NULL; params[i+1].is_null = is_null; - params[i+1].num = 18000; + params[i+1].num = 10; params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; params[i+2].buffer_length = sizeof(int8_t); params[i+2].buffer = v.v1; params[i+2].length = NULL; params[i+2].is_null = is_null; - params[i+2].num = 18000; + params[i+2].num = 10; params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; params[i+3].buffer_length = sizeof(int16_t); params[i+3].buffer = v.v2; params[i+3].length = NULL; params[i+3].is_null = is_null; - params[i+3].num = 18000; + params[i+3].num = 10; params[i+4].buffer_type = TSDB_DATA_TYPE_INT; params[i+4].buffer_length = sizeof(int32_t); params[i+4].buffer = v.v4; params[i+4].length = NULL; params[i+4].is_null = is_null; - params[i+4].num = 18000; + params[i+4].num = 10; params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; params[i+5].buffer_length = sizeof(int64_t); params[i+5].buffer = v.v8; params[i+5].length = NULL; params[i+5].is_null = is_null; - params[i+5].num = 18000; + params[i+5].num = 10; params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; params[i+6].buffer_length = sizeof(float); params[i+6].buffer = v.f4; params[i+6].length = NULL; params[i+6].is_null = is_null; - params[i+6].num = 18000; + params[i+6].num = 10; params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; params[i+7].buffer_length = sizeof(double); params[i+7].buffer = v.f8; params[i+7].length = NULL; params[i+7].is_null = is_null; - params[i+7].num = 18000; + params[i+7].num = 10; params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; params[i+8].buffer_length = 40; params[i+8].buffer = v.bin; params[i+8].length = lb; params[i+8].is_null = is_null; - params[i+8].num = 18000; + params[i+8].num = 10; params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; params[i+9].buffer_length = 40; params[i+9].buffer = v.bin; params[i+9].length = lb; params[i+9].is_null = is_null; - params[i+9].num = 18000; + params[i+9].num = 10; } int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { + for (int i = 0; i < 10; ++i) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + for (int i = 0; i < 1; ++i) { + tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; + tags[i+0].buffer = v.v4; + tags[i+0].is_null = &one_not_null; + tags[i+0].length = NULL; + + tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + tags[i+1].buffer = v.b; + tags[i+1].is_null = &one_not_null; + tags[i+1].length = NULL; + + tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + tags[i+2].buffer = v.v1; + tags[i+2].is_null = &one_not_null; + tags[i+2].length = NULL; + + tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + tags[i+3].buffer = v.v2; + tags[i+3].is_null = &one_not_null; + tags[i+3].length = NULL; + + tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; + tags[i+4].buffer = v.v8; + tags[i+4].is_null = &one_not_null; + tags[i+4].length = NULL; + + tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; + tags[i+5].buffer = v.f4; + tags[i+5].is_null = &one_not_null; + tags[i+5].length = NULL; + + tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; + tags[i+6].buffer = v.f8; + tags[i+6].is_null = &one_not_null; + tags[i+6].length = NULL; + + tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[i+7].buffer = v.bin; + tags[i+7].is_null = &one_not_null; + tags[i+7].length = (uintptr_t *)lb; + + tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; + tags[i+8].buffer = v.bin; + tags[i+8].is_null = &one_not_null; + tags[i+8].length = (uintptr_t *)lb; + } + + + unsigned long long starttime = taosGetTimestampUs(); + + char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt)); + exit(1); } int id = 0; - for (int l = 0; l < 10; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - ++id; + for (int zz = 0; zz < 1; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname_tags(stmt, buf, tags); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname_tags. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + code = taos_stmt_bind_param_batch(stmt, params + id * 10); + if (code != 0) { + printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); } - + + code = taos_stmt_bind_param_batch(stmt, params + id * 10); + if (code != 0) { + printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); + return -1; + } + + taos_stmt_add_batch(stmt); } - unsigned long long endtime = getCurrentTime(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + ++id; + + unsigned long long endtime = taosGetTimestampUs(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); taosMemoryFree(lb); taosMemoryFree(params); taosMemoryFree(is_null); taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } -//disorder -int stmt_funcb3(TAOS_STMT *stmt) { + + + + +//1 tables 10 records +int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { struct { int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; } v = {0}; - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = taosMemoryMalloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); + + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); + +// int one_null = 1; + int one_not_null = 0; - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); - for (int i = 0; i < 60; ++i) { + for (int i = 0; i < 10; ++i) { lb[i] = 40; no_null[i] = 0; is_null[i] = (i % 10 == 2) ? 1 : 0; @@ -3034,135 +2592,187 @@ int stmt_funcb3(TAOS_STMT *stmt) { memset(v.bin[i], '0'+i%10, 40); } - for (int i = 0; i < 9000000; i+=10) { + for (int i = 0; i < 10; i+=10) { params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; + params[i+0].buffer = &v.ts[10*i/10]; params[i+0].length = NULL; params[i+0].is_null = no_null; - params[i+0].num = 60; + params[i+0].num = 10; params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; params[i+1].buffer_length = sizeof(int8_t); params[i+1].buffer = v.b; params[i+1].length = NULL; params[i+1].is_null = is_null; - params[i+1].num = 60; + params[i+1].num = 10; params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; params[i+2].buffer_length = sizeof(int8_t); params[i+2].buffer = v.v1; params[i+2].length = NULL; params[i+2].is_null = is_null; - params[i+2].num = 60; + params[i+2].num = 10; params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; params[i+3].buffer_length = sizeof(int16_t); params[i+3].buffer = v.v2; params[i+3].length = NULL; params[i+3].is_null = is_null; - params[i+3].num = 60; + params[i+3].num = 10; params[i+4].buffer_type = TSDB_DATA_TYPE_INT; params[i+4].buffer_length = sizeof(int32_t); params[i+4].buffer = v.v4; params[i+4].length = NULL; params[i+4].is_null = is_null; - params[i+4].num = 60; + params[i+4].num = 10; params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; params[i+5].buffer_length = sizeof(int64_t); params[i+5].buffer = v.v8; params[i+5].length = NULL; params[i+5].is_null = is_null; - params[i+5].num = 60; + params[i+5].num = 10; params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; params[i+6].buffer_length = sizeof(float); params[i+6].buffer = v.f4; params[i+6].length = NULL; params[i+6].is_null = is_null; - params[i+6].num = 60; + params[i+6].num = 10; params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; params[i+7].buffer_length = sizeof(double); params[i+7].buffer = v.f8; params[i+7].length = NULL; params[i+7].is_null = is_null; - params[i+7].num = 60; + params[i+7].num = 10; params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; params[i+8].buffer_length = 40; params[i+8].buffer = v.bin; params[i+8].length = lb; params[i+8].is_null = is_null; - params[i+8].num = 60; + params[i+8].num = 10; params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; params[i+9].buffer_length = 40; params[i+9].buffer = v.bin; params[i+9].length = lb; params[i+9].is_null = is_null; - params[i+9].num = 60; + params[i+9].num = 10; } int64_t tts = 1591060628000; - int64_t ttt = 0; - for (int i = 0; i < 54000000; ++i) { + for (int i = 0; i < 10; ++i) { v.ts[i] = tts + i; - if (i > 0 && i%60 == 0) { - ttt = v.ts[i-1]; - v.ts[i-1] = v.ts[i-60]; - v.ts[i-60] = ttt; - } } - unsigned long long starttime = getCurrentTime(); - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + for (int i = 0; i < 1; ++i) { + tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; + tags[i+0].buffer = v.v4; + tags[i+0].is_null = &one_not_null; + tags[i+0].length = NULL; + + tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + tags[i+1].buffer = v.b; + tags[i+1].is_null = &one_not_null; + tags[i+1].length = NULL; + + tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; + tags[i+2].buffer = v.v1; + tags[i+2].is_null = &one_not_null; + tags[i+2].length = NULL; + + tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + tags[i+3].buffer = v.v2; + tags[i+3].is_null = &one_not_null; + tags[i+3].length = NULL; + + tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; + tags[i+4].buffer = v.v8; + tags[i+4].is_null = &one_not_null; + tags[i+4].length = NULL; + + tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; + tags[i+5].buffer = v.f4; + tags[i+5].is_null = &one_not_null; + tags[i+5].length = NULL; + + tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; + tags[i+6].buffer = v.f8; + tags[i+6].is_null = &one_not_null; + tags[i+6].length = NULL; + + tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[i+7].buffer = v.bin; + tags[i+7].is_null = &one_not_null; + tags[i+7].length = (uintptr_t *)lb; + + tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; + tags[i+8].buffer = v.bin; + tags[i+8].is_null = &one_not_null; + tags[i+8].length = (uintptr_t *)lb; } - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); + unsigned long long starttime = taosGetTimestampUs(); + + char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(NULL, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(NULL)); + return -1; + } + + int id = 0; + for (int zz = 0; zz < 1; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname_tags(stmt, buf, tags); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname_tags. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + code = taos_stmt_bind_param_batch(stmt, params + id * 10); + if (code != 0) { + printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } + + code = taos_stmt_bind_param_batch(stmt, params + id * 10); + if (code != 0) { + printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); + return -1; + } + + taos_stmt_add_batch(stmt); + } - ++id; + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); } - unsigned long long endtime = getCurrentTime(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + ++id; + + unsigned long long endtime = taosGetTimestampUs(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); taosMemoryFree(lb); taosMemoryFree(params); taosMemoryFree(is_null); taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } - - - //samets int stmt_funcb4(TAOS_STMT *stmt) { struct { @@ -3181,7 +2791,7 @@ int stmt_funcb4(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -3277,7 +2887,7 @@ int stmt_funcb4(TAOS_STMT *stmt) { v.ts[i] = tts; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -3307,7 +2917,7 @@ int stmt_funcb4(TAOS_STMT *stmt) { ++id; } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -3340,7 +2950,7 @@ int stmt_funcb5(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(18000 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 3000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 18000); char* no_null = taosMemoryMalloc(sizeof(char) * 18000); @@ -3436,7 +3046,7 @@ int stmt_funcb5(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into m0 values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -3460,7 +3070,7 @@ int stmt_funcb5(TAOS_STMT *stmt) { } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -3484,7 +3094,7 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(30000 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 3000*10); char* no_null = taosMemoryMalloc(sizeof(int) * 200000); for (int i = 0; i < 30000; ++i) { @@ -3514,7 +3124,7 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -3545,7 +3155,7 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -3575,7 +3185,7 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -3671,7 +3281,7 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -3702,7 +3312,7 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -3737,7 +3347,7 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -3833,7 +3443,7 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -3864,7 +3474,7 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { } } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -3895,7 +3505,7 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -3991,7 +3601,7 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -4024,7 +3634,7 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -4055,7 +3665,7 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 60*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -4152,7 +3762,7 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -4181,7 +3791,7 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { exit(1); } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -4192,15 +3802,12 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { return 0; } +#endif - -void check_result(TAOS *taos, char *tname, int printr, int expected) { +void prepareCheckResultImpl(TAOS *taos, char *tname, bool printr, int expected) { char sql[255] = "SELECT * FROM "; TAOS_RES *result; - //FORCE NO PRINT - printr = 0; - strcat(sql, tname); result = taos_query(taos, sql); @@ -4229,9 +3836,9 @@ void check_result(TAOS *taos, char *tname, int printr, int expected) { } if (rows == expected) { - printf("%d rows are fetched as expectation\n", rows); + printf("%d rows are fetched as expected from %s\n", rows, tname); } else { - printf("!!!expect %d rows, but %d rows are fetched\n", expected, rows); + printf("!!!expect %d rows, but %d rows are fetched from %s\n", expected, rows, tname); exit(1); } @@ -4240,6 +3847,22 @@ void check_result(TAOS *taos, char *tname, int printr, int expected) { } +void prepareCheckResult(TAOS *taos) { + char buf[32]; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + if (gCurCase->tblNum > 1) { + sprintf(buf, "t%d", t); + } else { + sprintf(buf, "t%d", 0); + } + + prepareCheckResultImpl(taos, buf, gCaseCtrl.printRes, gCurCase->rowNum * gExecLoopTimes); + } + + gExecLoopTimes = 1; +} + + //120table 60 record each table int sql_perf1(TAOS *taos) { @@ -4263,7 +3886,7 @@ int sql_perf1(TAOS *taos) { } - unsigned long long starttime = getCurrentTime(); + int64_t starttime = taosGetTimestampUs(); for (int i = 0; i < 3000; ++i) { result = taos_query(taos, sql[i]); int code = taos_errno(result); @@ -4275,7 +3898,7 @@ int sql_perf1(TAOS *taos) { taos_free_result(result); } - unsigned long long endtime = getCurrentTime(); + int64_t endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 3000; i++) { @@ -4291,7 +3914,7 @@ int sql_perf1(TAOS *taos) { //one table 60 records one time int sql_perf_s1(TAOS *taos) { - char **sql = taosMemoryCalloc(1, sizeof(char*) * 360000); + char **sql = calloc(1, sizeof(char*) * 360000); TAOS_RES *result; for (int i = 0; i < 360000; i++) { @@ -4316,7 +3939,7 @@ int sql_perf_s1(TAOS *taos) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); for (int i = 0; i < 360000; ++i) { result = taos_query(taos, sql[i]); int code = taos_errno(result); @@ -4328,7 +3951,7 @@ int sql_perf_s1(TAOS *taos) { taos_free_result(result); } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 360000; i++) { @@ -4363,7 +3986,7 @@ int sql_s_perf1(TAOS *taos) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); for (int i = 0; i < 3000; ++i) { result = taos_query(taos, sql[i]); int code = taos_errno(result); @@ -4375,7 +3998,7 @@ int sql_s_perf1(TAOS *taos) { taos_free_result(result); } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 3000; i++) { @@ -4385,8 +4008,131 @@ int sql_s_perf1(TAOS *taos) { return 0; } +void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t *colList, bool stable) { + int32_t blen = 0; + blen = sprintf(buf, "create table %s%d ", (stable ? "st" : "t"), tblIdx); + if (stable) { + blen += sprintf(buf + blen, "tags ("); + for (int c = 0; c < colNum; ++c) { + if (c > 0) { + blen += sprintf(buf + blen, ","); + } + switch (colList[c]) { + case TSDB_DATA_TYPE_BOOL: + blen += sprintf(buf + blen, "tbooldata bool"); + break; + case TSDB_DATA_TYPE_TINYINT: + blen += sprintf(buf + blen, "ttinydata tinyint"); + break; + case TSDB_DATA_TYPE_SMALLINT: + blen += sprintf(buf + blen, "tsmalldata smallint"); + break; + case TSDB_DATA_TYPE_INT: + blen += sprintf(buf + blen, "tintdata int"); + break; + case TSDB_DATA_TYPE_BIGINT: + blen += sprintf(buf + blen, "tbigdata bigint"); + break; + case TSDB_DATA_TYPE_FLOAT: + blen += sprintf(buf + blen, "tfloatdata float"); + break; + case TSDB_DATA_TYPE_DOUBLE: + blen += sprintf(buf + blen, "tdoubledata double"); + break; + case TSDB_DATA_TYPE_VARCHAR: + blen += sprintf(buf + blen, "tbinarydata binary(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + blen += sprintf(buf + blen, "tts ts"); + break; + case TSDB_DATA_TYPE_NCHAR: + blen += sprintf(buf + blen, "tnchardata nchar(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_UTINYINT: + blen += sprintf(buf + blen, "tutinydata tinyint unsigned"); + break; + case TSDB_DATA_TYPE_USMALLINT: + blen += sprintf(buf + blen, "tusmalldata smallint unsigned"); + break; + case TSDB_DATA_TYPE_UINT: + blen += sprintf(buf + blen, "tuintdata int unsigned"); + break; + case TSDB_DATA_TYPE_UBIGINT: + blen += sprintf(buf + blen, "tubigdata bigint unsigned"); + break; + default: + printf("invalid col type:%d", colList[c]); + exit(1); + } + } + + blen += sprintf(buf + blen, ")"); + } + + blen += sprintf(buf + blen, " ("); + + for (int c = 0; c < colNum; ++c) { + if (c > 0) { + blen += sprintf(buf + blen, ","); + } + + switch (colList[c]) { + case TSDB_DATA_TYPE_BOOL: + blen += sprintf(buf + blen, "booldata bool"); + break; + case TSDB_DATA_TYPE_TINYINT: + blen += sprintf(buf + blen, "tinydata tinyint"); + break; + case TSDB_DATA_TYPE_SMALLINT: + blen += sprintf(buf + blen, "smalldata smallint"); + break; + case TSDB_DATA_TYPE_INT: + blen += sprintf(buf + blen, "intdata int"); + break; + case TSDB_DATA_TYPE_BIGINT: + blen += sprintf(buf + blen, "bigdata bigint"); + break; + case TSDB_DATA_TYPE_FLOAT: + blen += sprintf(buf + blen, "floatdata float"); + break; + case TSDB_DATA_TYPE_DOUBLE: + blen += sprintf(buf + blen, "doubledata double"); + break; + case TSDB_DATA_TYPE_VARCHAR: + blen += sprintf(buf + blen, "binarydata binary(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + blen += sprintf(buf + blen, "ts timestamp"); + break; + case TSDB_DATA_TYPE_NCHAR: + blen += sprintf(buf + blen, "nchardata nchar(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_UTINYINT: + blen += sprintf(buf + blen, "utinydata tinyint unsigned"); + break; + case TSDB_DATA_TYPE_USMALLINT: + blen += sprintf(buf + blen, "usmalldata smallint unsigned"); + break; + case TSDB_DATA_TYPE_UINT: + blen += sprintf(buf + blen, "uintdata int unsigned"); + break; + case TSDB_DATA_TYPE_UBIGINT: + blen += sprintf(buf + blen, "ubigdata bigint unsigned"); + break; + default: + printf("invalid col type:%d", colList[c]); + exit(1); + } + } + + blen += sprintf(buf + blen, ")"); + + if (gCaseCtrl.printCreateTblSql) { + printf("Create Table SQL:%s\n", buf); + } +} -void prepare(TAOS *taos, int bigsize, int createChildTable) { +void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int autoCreate) { TAOS_RES *result; int code; @@ -4405,15 +4151,11 @@ void prepare(TAOS *taos, int bigsize, int createChildTable) { result = taos_query(taos, "use demo"); taos_free_result(result); - if (createChildTable) { + if (!autoCreate) { // create table - for (int i = 0 ; i < 300; i++) { + for (int i = 0 ; i < 10; i++) { char buf[1024]; - if (bigsize) { - sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40))", i) ; - } else { - sprintf(buf, "create table m%d (ts timestamp, b int)", i) ; - } + generateCreateTableSQL(buf, i, colNum, colList, false); result = taos_query(taos, buf); code = taos_errno(result); if (code != 0) { @@ -4425,12 +4167,7 @@ void prepare(TAOS *taos, int bigsize, int createChildTable) { } } else { char buf[1024]; - if (bigsize) { - sprintf(buf, "create stable stb1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40))" - " tags(id1 int, id2 bool, id3 tinyint, id4 smallint, id5 bigint, id6 float, id7 double, id8 binary(40), id9 nchar(40))") ; - } else { - sprintf(buf, "create stable stb1 (ts timestamp, b int) tags(id1 int, id2 bool, id3 tinyint, id4 smallint, id5 bigint, id6 float, id7 double, id8 binary(40), id9 nchar(40))") ; - } + generateCreateTableSQL(buf, 1, colNum, colList, true); result = taos_query(taos, buf); code = taos_errno(result); @@ -4444,587 +4181,66 @@ void prepare(TAOS *taos, int bigsize, int createChildTable) { } +void* runcase(TAOS *taos) { + TAOS_STMT *stmt = NULL; + int32_t caseIdx = 0; + for (int32_t i = 0; i < sizeof(gCase)/sizeof(gCase[0]); ++i) { + CaseCfg cfg = gCase[i]; + gCurCase = &cfg; -void preparem(TAOS *taos, int bigsize, int idx) { - TAOS_RES *result; - int code; - char dbname[32],sql[255]; - - sprintf(dbname, "demo%d", idx); - sprintf(sql, "drop database %s", dbname); - - - result = taos_query(taos, sql); - taos_free_result(result); + if ((gCaseCtrl.bindColTypeNum || gCaseCtrl.bindColNum) && (gCurCase->colNum != gFullColNum)) { + continue; + } - sprintf(sql, "create database %s keep 36500", dbname); - result = taos_query(taos, sql); - code = taos_errno(result); - if (code != 0) { - printf("failed to create database, reason:%s\n", taos_errstr(result)); - taos_free_result(result); - exit(1); - } - taos_free_result(result); + printf("* Case %d - %s Begin *\n", caseIdx, gCurCase->caseDesc); - sprintf(sql, "use %s", dbname); - result = taos_query(taos, sql); - taos_free_result(result); - - // create table - for (int i = 0 ; i < 300; i++) { - char buf[1024]; - if (bigsize) { - sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40))", i) ; - } else { - sprintf(buf, "create table m%d (ts timestamp, b int)", i) ; + if (gCaseCtrl.runTimes) { + gCurCase->runTimes = gCaseCtrl.runTimes; } - result = taos_query(taos, buf); - code = taos_errno(result); - if (code != 0) { - printf("failed to create table, reason:%s\n", taos_errstr(result)); - taos_free_result(result); - exit(1); + + if (gCaseCtrl.rowNum) { + gCurCase->rowNum = gCaseCtrl.rowNum; + } + + if (gCurCase->fullCol) { + gCurCase->bindColNum = gCurCase->colNum; } - taos_free_result(result); - } - -} - - - -//void runcase(TAOS *taos, int idx) { -void* runcase(void *par) { - T_par* tpar = (T_par *)par; - TAOS *taos = tpar->taos; - int idx = tpar->idx; - - TAOS_STMT *stmt; - - (void)idx; - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("10t+10records+specifycol start\n"); - stmt_scol_func1(stmt); - printf("10t+10records+specifycol end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 10); - check_result(taos, "m1", 1, 10); - check_result(taos, "m2", 1, 10); - check_result(taos, "m3", 1, 10); - check_result(taos, "m4", 1, 10); - check_result(taos, "m5", 1, 10); - check_result(taos, "m6", 1, 10); - check_result(taos, "m7", 1, 10); - check_result(taos, "m8", 1, 10); - check_result(taos, "m9", 1, 10); - printf("check result end\n"); - taos_stmt_close(stmt); -#endif - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("1t+100records+specifycol start\n"); - stmt_scol_func2(stmt); - printf("1t+100records+specifycol end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 100); - printf("check result end\n"); - taos_stmt_close(stmt); -#endif - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("300t+10r+bm+specifycol start\n"); - stmt_scol_func3(stmt); - printf("300t+10r+bm+specifycol end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 20); - check_result(taos, "m1", 1, 20); - check_result(taos, "m111", 1, 20); - check_result(taos, "m223", 1, 20); - check_result(taos, "m299", 1, 20); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("10t+2r+bm+specifycol start\n"); - stmt_scol_func4(stmt); - printf("10t+2r+bm+specifycol end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 20); - check_result(taos, "m1", 1, 20); - check_result(taos, "m2", 1, 20); - check_result(taos, "m3", 1, 20); - check_result(taos, "m4", 1, 20); - check_result(taos, "m5", 1, 20); - check_result(taos, "m6", 1, 20); - check_result(taos, "m7", 1, 20); - check_result(taos, "m8", 1, 20); - check_result(taos, "m9", 1, 20); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("10t+10records start\n"); - stmt_func1(stmt); - printf("10t+10records end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 10); - check_result(taos, "m1", 1, 10); - check_result(taos, "m2", 1, 10); - check_result(taos, "m3", 1, 10); - check_result(taos, "m4", 1, 10); - check_result(taos, "m5", 1, 10); - check_result(taos, "m6", 1, 10); - check_result(taos, "m7", 1, 10); - check_result(taos, "m8", 1, 10); - check_result(taos, "m9", 1, 10); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("10t+[0,1,2...9]records start\n"); - stmt_func2(stmt); - printf("10t+[0,1,2...9]records end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 0); - check_result(taos, "m1", 0, 100); - check_result(taos, "m2", 0, 200); - check_result(taos, "m3", 0, 300); - check_result(taos, "m4", 0, 400); - check_result(taos, "m5", 0, 500); - check_result(taos, "m6", 0, 600); - check_result(taos, "m7", 0, 700); - check_result(taos, "m8", 0, 800); - check_result(taos, "m9", 0, 900); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("10t+[0,100,200...900]records start\n"); - stmt_func3(stmt); - printf("10t+[0,100,200...900]records end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 0); - check_result(taos, "m1", 0, 100); - check_result(taos, "m2", 0, 200); - check_result(taos, "m3", 0, 300); - check_result(taos, "m4", 0, 400); - check_result(taos, "m5", 0, 500); - check_result(taos, "m6", 0, 600); - check_result(taos, "m7", 0, 700); - check_result(taos, "m8", 0, 800); - check_result(taos, "m9", 0, 900); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("300t+60r+bm start\n"); - stmt_funcb1(stmt); - printf("300t+60r+bm end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m111", 0, 180000); - check_result(taos, "m223", 0, 180000); - check_result(taos, "m299", 0, 180000); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb1 start\n"); - stmt_funcb_autoctb1(stmt); - printf("1t+10r+bm+autoctb1 end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 10); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb2 start\n"); - stmt_funcb_autoctb2(stmt); - printf("1t+10r+bm+autoctb2 end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 10); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb3 start\n"); - stmt_funcb_autoctb3(stmt); - printf("1t+10r+bm+autoctb3 end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 10); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb4 start\n"); - stmt_funcb_autoctb4(stmt); - printf("1t+10r+bm+autoctb4 end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 10); - printf("check result end\n"); - taos_stmt_close(stmt); -#endif - - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb+e1 start\n"); - stmt_funcb_autoctb_e1(stmt); - printf("1t+10r+bm+autoctb+e1 end\n"); - printf("check result start\n"); - //check_result(taos, "m0", 1, 0); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb+e2 start\n"); - stmt_funcb_autoctb_e2(stmt); - printf("1t+10r+bm+autoctb+e2 end\n"); - printf("check result start\n"); - //check_result(taos, "m0", 1, 0); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb+e3 start\n"); - stmt_funcb_autoctb_e3(stmt); - printf("1t+10r+bm+autoctb+e3 end\n"); - printf("check result start\n"); - //check_result(taos, "m0", 1, 0); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb+e4 start\n"); - stmt_funcb_autoctb_e4(stmt); - printf("1t+10r+bm+autoctb+e4 end\n"); - printf("check result start\n"); - //check_result(taos, "m0", 1, 0); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb+e5 start\n"); - stmt_funcb_autoctb_e5(stmt); - printf("1t+10r+bm+autoctb+e5 end\n"); - printf("check result start\n"); - //check_result(taos, "m0", 1, 0); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("1t+18000r+bm start\n"); - stmt_funcb2(stmt); - printf("1t+18000r+bm end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m111", 0, 180000); - check_result(taos, "m223", 0, 180000); - check_result(taos, "m299", 0, 180000); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("300t+60r+disorder+bm start\n"); - stmt_funcb3(stmt); - printf("300t+60r+disorder+bm end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m111", 0, 180000); - check_result(taos, "m223", 0, 180000); - check_result(taos, "m299", 0, 180000); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("300t+60r+samets+bm start\n"); - stmt_funcb4(stmt); - printf("300t+60r+samets+bm end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 1); - check_result(taos, "m1", 0, 1); - check_result(taos, "m111", 0, 1); - check_result(taos, "m223", 0, 1); - check_result(taos, "m299", 0, 1); - printf("check result end\n"); - taos_stmt_close(stmt); -#endif - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("1t+18000r+nodyntable+bm start\n"); - stmt_funcb5(stmt); - printf("1t+18000r+nodyntable+bm end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("300t+60r+bm+sc start\n"); - stmt_funcb_sc1(stmt); - printf("300t+60r+bm+sc end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m111", 0, 180000); - check_result(taos, "m223", 0, 180000); - check_result(taos, "m299", 0, 180000); - printf("check result end\n"); - taos_stmt_close(stmt); -#endif - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("1t+60r+bm+sc start\n"); - stmt_funcb_sc2(stmt); - printf("1t+60r+bm+sc end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m111", 0, 180000); - check_result(taos, "m223", 0, 180000); - check_result(taos, "m299", 0, 180000); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("10t+[1...10]r+bm+sc start\n"); - stmt_funcb_sc3(stmt); - printf("10t+[1...10]r+bm+sc end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 1); - check_result(taos, "m1", 1, 2); - check_result(taos, "m2", 1, 3); - check_result(taos, "m3", 1, 4); - check_result(taos, "m4", 1, 5); - check_result(taos, "m5", 1, 6); - check_result(taos, "m6", 1, 7); - check_result(taos, "m7", 1, 8); - check_result(taos, "m8", 1, 9); - check_result(taos, "m9", 1, 10); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - printf("1t+60r+bm start\n"); - stmt_funcb_s1(stmt); - printf("1t+60r+bm end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m111", 0, 180000); - check_result(taos, "m223", 0, 180000); - check_result(taos, "m299", 0, 180000); - printf("check result end\n"); - taos_stmt_close(stmt); + gCurCase->bindNullNum = gCaseCtrl.bindNullNum; + gCurCase->autoCreate = gCaseCtrl.autoCreate; + if (gCaseCtrl.bindColNum) { + gCurCase->bindColNum = gCaseCtrl.bindColNum; + gCurCase->fullCol = false; + } + if (gCaseCtrl.bindRowNum) { + gCurCase->bindRowNum = gCaseCtrl.bindRowNum; + } + if (gCaseCtrl.bindColTypeNum) { + gCurCase->bindColNum = gCaseCtrl.bindColTypeNum; + gCurCase->fullCol = false; + } + + for (int32_t n = 0; n < gCurCase->runTimes; ++n) { + prepare(taos, gCurCase->colNum, gCurCase->colList, gCurCase->autoCreate); + + stmt = taos_stmt_init(taos); + if (NULL == stmt) { + printf("taos_stmt_init failed, error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } -#endif + (*gCurCase->runFn)(stmt); + prepareCheckResult(taos); -#if 1 - prepare(taos, 1, 1); - - (void)stmt; - printf("120t+60r+sql start\n"); - sql_perf1(taos); - printf("120t+60r+sql end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m34", 0, 180000); - check_result(taos, "m67", 0, 180000); - check_result(taos, "m99", 0, 180000); - printf("check result end\n"); -#endif - -#if 1 - prepare(taos, 1, 1); - - (void)stmt; - printf("1t+60r+sql start\n"); - sql_perf_s1(taos); - printf("1t+60r+sql end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m34", 0, 180000); - check_result(taos, "m67", 0, 180000); - check_result(taos, "m99", 0, 180000); - printf("check result end\n"); -#endif - - -#if 1 - preparem(taos, 0, idx); - - stmt = taos_stmt_init(taos); - - printf("1t+30000r+bm start\n"); - stmt_funcb_ssz1(stmt); - printf("1t+30000r+bm end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 300000); - check_result(taos, "m1", 0, 300000); - check_result(taos, "m111", 0, 300000); - check_result(taos, "m223", 0, 300000); - check_result(taos, "m299", 0, 300000); - printf("check result end\n"); - taos_stmt_close(stmt); + taos_stmt_close(stmt); + } + + printf("* Case %d - %s End *\n", caseIdx, gCurCase->caseDesc); -#endif + caseIdx++; + } printf("test end\n"); @@ -5032,64 +4248,68 @@ void* runcase(void *par) { } +void runAll(TAOS *taos) { + printf("Normal Test\n"); + runcase(taos); + + printf("Null Test\n"); + gCaseCtrl.bindNullNum = 1; + runcase(taos); + gCaseCtrl.bindNullNum = 0; + + printf("Bind Row Test\n"); + gCaseCtrl.bindRowNum = 1; + runcase(taos); + gCaseCtrl.bindRowNum = 0; + + printf("Row Num Test\n"); + gCaseCtrl.rowNum = 1000; + gCaseCtrl.printRes = false; + runcase(taos); + gCaseCtrl.rowNum = 0; + gCaseCtrl.printRes = true; + + printf("Runtimes Test\n"); + gCaseCtrl.runTimes = 2; + runcase(taos); + gCaseCtrl.runTimes = 0; + + printf("Check Param Test\n"); + gCaseCtrl.checkParamNum = true; + runcase(taos); + gCaseCtrl.checkParamNum = false; + + printf("Bind Col Num Test\n"); + gCaseCtrl.bindColNum = 6; + runcase(taos); + gCaseCtrl.bindColNum = 0; + + printf("Bind Col Type Test\n"); + gCaseCtrl.bindColTypeNum = tListLen(bindColTypeList); + gCaseCtrl.bindColTypeList = bindColTypeList; + runcase(taos); +} + int main(int argc, char *argv[]) { - TAOS *taos[4]; + TAOS *taos = NULL; + srand(time(NULL)); + // connect to server if (argc < 2) { printf("please input server ip \n"); return 0; } - taos[0] = taos_connect(argv[1], "root", "taosdata", NULL, 0); - if (taos == NULL) { - printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); - exit(1); - } - - taos[1] = taos_connect(argv[1], "root", "taosdata", NULL, 0); - if (taos == NULL) { - printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); - exit(1); - } - - taos[2] = taos_connect(argv[1], "root", "taosdata", NULL, 0); + taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); if (taos == NULL) { printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); exit(1); } - taos[3] = taos_connect(argv[1], "root", "taosdata", NULL, 0); - if (taos == NULL) { - printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); - exit(1); - } - - TdThread *pThreadList = (TdThread *) taosMemoryCalloc(sizeof(TdThread), 4); - - TdThreadAttr thattr; - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - T_par par[4]; - - par[0].taos = taos[0]; - par[0].idx = 0; - par[1].taos = taos[1]; - par[1].idx = 1; - par[2].taos = taos[2]; - par[2].idx = 2; - par[3].taos = taos[3]; - par[3].idx = 3; - - taosThreadCreate(&(pThreadList[0]), &thattr, runcase, (void *)&par[0]); - //taosThreadCreate(&(pThreadList[1]), &thattr, runcase, (void *)&par[1]); - //taosThreadCreate(&(pThreadList[2]), &thattr, runcase, (void *)&par[2]); - //taosThreadCreate(&(pThreadList[3]), &thattr, runcase, (void *)&par[3]); + runAll(taos); - while(1) { - taosSsleep(1); - } return 0; } diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 92d0a89b0fe0f4b31a43e8981a763922c03d5343..46a172cc3af4bb05a209b47f54b1c9624328670d 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -4,18 +4,14 @@ ROOT=./ TARGET=exe LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt -CFLAGS = -O0 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \ +CFLAGS = -O0 -g -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \ -Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX \ - -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 \ - -fsanitize=address + -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 -Wno-sign-conversion all: $(TARGET) exe: gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) - gcc $(CFLAGS) ./stmtBatchTest.c -o $(ROOT)stmtBatchTest $(LFLAGS) - gcc $(CFLAGS) ./stmtTest.c -o $(ROOT)stmtTest $(LFLAGS) - gcc $(CFLAGS) ./stmt_function.c -o $(ROOT)stmt_function $(LFLAGS) clean: rm $(ROOT)batchprepare diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim deleted file mode 100644 index 5e22e02297bba63ec023780e425019f713dfd8d8..0000000000000000000000000000000000000000 --- a/tests/script/basicSuite.sim +++ /dev/null @@ -1,19 +0,0 @@ -run general/cache/new_metrics.sim -run general/compute/interval.sim -run general/http/restful_full.sim -run general/import/commit.sim -run general/import/replica1.sim -run general/parser/auto_create_tb_drop_tb.sim -run general/parser/binary_escapeCharacter.sim -run general/parser/select_from_cache_disk.sim -run general/parser/null_char.sim -run general/parser/alter.sim -run general/stable/vnode3.sim -run general/tag/filter.sim -run general/table/vgroup.sim -run general/user/authority.sim -run general/vector/metrics_mix.sim -run general/vector/table_field.sim -run general/tag/set.sim -run general/table/delete_writing.sim -run general/stable/disk.sim diff --git a/tests/script/bug.sim b/tests/script/bug.sim deleted file mode 100644 index 625b9b0cadb78f8d75e8611dbf60664f9cf7e65a..0000000000000000000000000000000000000000 --- a/tests/script/bug.sim +++ /dev/null @@ -1,2 +0,0 @@ -run general/parser/projection_limit_offset.sim -run general/parser/limit2.sim \ No newline at end of file diff --git a/tests/script/fullGeneralSuite.sim b/tests/script/fullGeneralSuite.sim deleted file mode 100644 index 188ce1405541cbbb230ceb186c44cfd4230925fc..0000000000000000000000000000000000000000 --- a/tests/script/fullGeneralSuite.sim +++ /dev/null @@ -1,225 +0,0 @@ -#unsupport run general/alter/cached_schema_after_alter.sim -run general/alter/count.sim -run general/alter/import.sim -#unsupport run general/alter/insert1.sim -run general/alter/insert2.sim -run general/alter/metrics.sim -run general/alter/table.sim -run general/cache/new_metrics.sim -run general/cache/restart_metrics.sim -run general/cache/restart_table.sim -run general/connection/connection.sim -run general/column/commit.sim -run general/column/metrics.sim -run general/column/table.sim -run general/compress/commitlog.sim -run general/compress/compress.sim -run general/compress/compress2.sim -run general/compress/uncompress.sim -run general/compute/avg.sim -run general/compute/bottom.sim -run general/compute/count.sim -run general/compute/diff.sim -run general/compute/diff2.sim -run general/compute/first.sim -run general/compute/interval.sim -run general/compute/last.sim -run general/compute/last_row.sim -run general/compute/leastsquare.sim -run general/compute/max.sim -run general/compute/min.sim -run general/compute/null.sim -run general/compute/percentile.sim -run general/compute/stddev.sim -run general/compute/sum.sim -run general/compute/top.sim -run general/compute/block_dist.sim -run general/db/alter_option.sim -run general/db/alter_tables_d2.sim -run general/db/alter_tables_v1.sim -run general/db/alter_tables_v4.sim -run general/db/alter_vgroups.sim -run general/db/basic.sim -run general/db/basic1.sim -run general/db/basic2.sim -run general/db/basic3.sim -run general/db/basic4.sim -run general/db/basic5.sim -run general/db/delete_reuse1.sim -run general/db/delete_reuse2.sim -run general/db/delete_reusevnode.sim -run general/db/delete_reusevnode2.sim -run general/db/delete_writing1.sim -run general/db/delete_writing2.sim -run general/db/delete.sim -run general/db/len.sim -run general/db/repeat.sim -run general/db/tables.sim -run general/db/vnodes.sim -run general/field/2.sim -run general/field/3.sim -run general/field/4.sim -run general/field/5.sim -run general/field/6.sim -run general/field/bigint.sim -run general/field/binary.sim -run general/field/bool.sim -run general/field/single.sim -run general/field/smallint.sim -run general/field/tinyint.sim -run general/http/restful.sim -run general/http/restful_insert.sim -run general/http/restful_limit.sim -run general/http/restful_full.sim -run general/http/prepare.sim -run general/http/telegraf.sim -run general/http/grafana_bug.sim -run general/http/grafana.sim -run general/import/basic.sim -run general/import/commit.sim -run general/import/large.sim -run general/import/replica1.sim -run general/insert/basic.sim -run general/insert/insert_drop.sim -run general/insert/query_block1_memory.sim -run general/insert/query_block2_memory.sim -run general/insert/query_block1_file.sim -run general/insert/query_block2_file.sim -run general/insert/query_file_memory.sim -run general/insert/query_multi_file.sim -run general/insert/tcp.sim -#unsupport run general/parser/alter.sim -run general/parser/alter1.sim -run general/parser/alter_stable.sim -run general/parser/auto_create_tb.sim -run general/parser/auto_create_tb_drop_tb.sim -run general/parser/col_arithmetic_operation.sim -run general/parser/columnValue.sim -run general/parser/commit.sim -run general/parser/create_db.sim -run general/parser/create_mt.sim -run general/parser/create_tb.sim -run general/parser/dbtbnameValidate.sim -run general/parser/import_commit1.sim -run general/parser/import_commit2.sim -run general/parser/import_commit3.sim -run general/parser/insert_tb.sim -run general/parser/first_last.sim -run general/parser/line_insert.sim -#unsupport run general/parser/import_file.sim -run general/parser/lastrow.sim -run general/parser/nchar.sim -#unsupport run general/parser/null_char.sim -run general/parser/single_row_in_tb.sim -run general/parser/select_from_cache_disk.sim -run general/parser/limit.sim -run general/parser/limit1.sim -run general/parser/limit1_tblocks100.sim -run general/parser/mixed_blocks.sim -run general/parser/selectResNum.sim -run general/parser/select_across_vnodes.sim -run general/parser/slimit1.sim -run general/parser/tbnameIn.sim -run general/parser/binary_escapeCharacter.sim -run general/parser/projection_limit_offset.sim -run general/parser/limit2.sim -run general/parser/slimit.sim -run general/parser/fill.sim -run general/parser/fill_stb.sim -run general/parser/interp.sim -run general/parser/where.sim -run general/parser/join.sim -run general/parser/join_multivnode.sim -run general/parser/select_with_tags.sim -run general/parser/groupby.sim -run general/parser/top_groupby.sim -run general/parser/tags_dynamically_specifiy.sim -run general/parser/set_tag_vals.sim -#unsupport run general/parser/repeatAlter.sim -#unsupport run general/parser/slimit_alter_tags.sim -run general/parser/precision_ns.sim -run general/stable/disk.sim -run general/stable/dnode3.sim -run general/stable/metrics.sim -run general/stable/values.sim -run general/stable/vnode3.sim -run general/table/autocreate.sim -run general/table/basic1.sim -run general/table/basic2.sim -run general/table/basic3.sim -run general/table/bigint.sim -run general/table/binary.sim -run general/table/bool.sim -run general/table/column_name.sim -run general/table/column_num.sim -run general/table/column_value.sim -run general/table/column2.sim -run general/table/date.sim -run general/table/db.table.sim -run general/table/delete_reuse1.sim -run general/table/delete_reuse2.sim -run general/table/delete_writing.sim -run general/table/describe.sim -run general/table/double.sim -run general/table/fill.sim -run general/table/float.sim -run general/table/int.sim -run general/table/limit.sim -run general/table/smallint.sim -run general/table/table_len.sim -run general/table/table.sim -run general/table/tinyint.sim -run general/table/vgroup.sim -run general/tag/3.sim -run general/tag/4.sim -run general/tag/5.sim -run general/tag/6.sim -run general/tag/add.sim -run general/tag/bigint.sim -run general/tag/binary_binary.sim -run general/tag/binary.sim -run general/tag/bool_binary.sim -run general/tag/bool_int.sim -run general/tag/bool.sim -run general/tag/change.sim -run general/tag/column.sim -#unsupport run general/tag/commit.sim -run general/tag/create.sim -run general/tag/delete.sim -run general/tag/double.sim -run general/tag/filter.sim -run general/tag/float.sim -run general/tag/int_binary.sim -run general/tag/int_float.sim -run general/tag/int.sim -run general/tag/set.sim -run general/tag/smallint.sim -run general/tag/tinyint.sim -run general/user/authority.sim -run general/user/monitor.sim -run general/user/pass_alter.sim -run general/user/pass_len.sim -run general/user/user_create.sim -run general/user/user_len.sim -run general/vector/metrics_field.sim -run general/vector/metrics_mix.sim -run general/vector/metrics_query.sim -run general/vector/metrics_tag.sim -run general/vector/metrics_time.sim -run general/vector/multi.sim -run general/vector/single.sim -run general/vector/table_field.sim -run general/vector/table_mix.sim -run general/vector/table_query.sim -run general/vector/table_time.sim -run general/stream/restart_stream.sim -run general/stream/stream_3.sim -run general/stream/stream_restart.sim -run general/stream/table_del.sim -run general/stream/metrics_del.sim -run general/stream/table_replica1_vnoden.sim -run general/stream/metrics_replica1_vnoden.sim -run general/db/show_create_db.sim -run general/db/show_create_table.sim -run general/parser/like.sim -run general/parser/regex.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 8dc7fb920effa5554fd0fc77d11fdd27971e8e95..506abf3fc5c34a6530bff04bd83b01cccc9e39a2 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -55,7 +55,7 @@ # ---- tmq ./test.sh -f tsim/tmq/basic.sim -./test.sh -f tsim/tmq/basic1.sim +#./test.sh -f tsim/tmq/basic1.sim #./test.sh -f tsim/tmq/oneTopic.sim #./test.sh -f tsim/tmq/multiTopic.sim @@ -93,4 +93,7 @@ # --- sma ./test.sh -f tsim/sma/tsmaCreateInsertData.sim +# --- valgrind +./test.sh -f tsim/valgrind/checkError.sim -v + #======================b1-end=============== diff --git a/tests/script/regressionSuite.sim b/tests/script/regressionSuite.sim deleted file mode 100644 index bada2f655202ddc34ce6e67e718336a2afc41d50..0000000000000000000000000000000000000000 --- a/tests/script/regressionSuite.sim +++ /dev/null @@ -1,221 +0,0 @@ -##unsupport run general/alter/cached_schema_after_alter.sim -run general/alter/count.sim -run general/alter/import.sim -##unsupport run general/alter/insert1.sim -run general/alter/insert2.sim -run general/alter/metrics.sim -run general/alter/table.sim -run general/cache/new_metrics.sim -run general/cache/restart_metrics.sim -run general/cache/restart_table.sim -run general/connection/connection.sim -run general/column/commit.sim -run general/column/metrics.sim -run general/column/table.sim -run general/compress/commitlog.sim -run general/compress/compress.sim -run general/compress/compress2.sim -run general/compress/uncompress.sim -run general/compute/avg.sim -run general/compute/bottom.sim -run general/compute/count.sim -run general/compute/diff.sim -run general/compute/diff2.sim -run general/compute/first.sim -run general/compute/interval.sim -run general/compute/last.sim -run general/compute/leastsquare.sim -run general/compute/max.sim -run general/compute/min.sim -run general/compute/null.sim -run general/compute/percentile.sim -run general/compute/stddev.sim -run general/compute/sum.sim -run general/compute/top.sim -run general/compute/block_dist.sim -run general/db/alter_option.sim -run general/db/alter_tables_d2.sim -run general/db/alter_tables_v1.sim -run general/db/alter_tables_v4.sim -run general/db/alter_vgroups.sim -run general/db/basic.sim -run general/db/basic1.sim -run general/db/basic2.sim -run general/db/basic3.sim -run general/db/basic4.sim -run general/db/basic5.sim -run general/db/delete_reuse1.sim -run general/db/delete_reuse2.sim -run general/db/delete_reusevnode.sim -run general/db/delete_reusevnode2.sim -run general/db/delete_writing1.sim -run general/db/delete_writing2.sim -run general/db/delete.sim -run general/db/len.sim -run general/db/repeat.sim -run general/db/tables.sim -run general/db/vnodes.sim -run general/field/2.sim -run general/field/3.sim -run general/field/4.sim -run general/field/5.sim -run general/field/6.sim -run general/field/bigint.sim -run general/field/binary.sim -run general/field/bool.sim -run general/field/single.sim -run general/field/smallint.sim -run general/field/tinyint.sim -run general/http/restful.sim -run general/http/restful_insert.sim -run general/http/restful_limit.sim -run general/http/restful_full.sim -run general/http/prepare.sim -run general/http/telegraf.sim -run general/http/grafana_bug.sim -run general/http/grafana.sim -run general/import/basic.sim -run general/import/commit.sim -run general/import/large.sim -run general/import/replica1.sim -run general/insert/basic.sim -run general/insert/insert_drop.sim -run general/insert/query_block1_memory.sim -run general/insert/query_block2_memory.sim -run general/insert/query_block1_file.sim -run general/insert/query_block2_file.sim -run general/insert/query_file_memory.sim -run general/insert/query_multi_file.sim -run general/insert/tcp.sim -run general/parser/alter.sim -run general/parser/alter1.sim -run general/parser/alter_stable.sim -run general/parser/auto_create_tb.sim -run general/parser/auto_create_tb_drop_tb.sim -run general/parser/col_arithmetic_operation.sim -run general/parser/columnValue.sim -run general/parser/commit.sim -run general/parser/create_db.sim -run general/parser/create_mt.sim -run general/parser/create_tb.sim -run general/parser/dbtbnameValidate.sim -run general/parser/import_commit1.sim -run general/parser/import_commit2.sim -run general/parser/import_commit3.sim -run general/parser/insert_tb.sim -run general/parser/first_last.sim -##unsupport run general/parser/import_file.sim -run general/parser/lastrow.sim -run general/parser/nchar.sim -run general/parser/null_char.sim -run general/parser/single_row_in_tb.sim -run general/parser/select_from_cache_disk.sim -run general/parser/limit.sim -run general/parser/limit1.sim -run general/parser/limit1_tblocks100.sim -run general/parser/mixed_blocks.sim -run general/parser/selectResNum.sim -run general/parser/select_across_vnodes.sim -run general/parser/slimit1.sim -run general/parser/tbnameIn.sim -run general/parser/binary_escapeCharacter.sim -run general/parser/projection_limit_offset.sim -run general/parser/limit2.sim -run general/parser/slimit.sim -run general/parser/fill.sim -run general/parser/fill_stb.sim -run general/parser/interp.sim -run general/parser/where.sim -run general/parser/join.sim -run general/parser/join_multivnode.sim -run general/parser/select_with_tags.sim -run general/parser/groupby.sim -run general/parser/bug.sim -run general/parser/tags_dynamically_specifiy.sim -run general/parser/set_tag_vals.sim -run general/parser/repeatAlter.sim -run general/parser/precision_ns.sim -##unsupport run general/parser/slimit_alter_tags.sim -run general/stable/disk.sim -run general/stable/dnode3.sim -run general/stable/metrics.sim -run general/stable/values.sim -run general/stable/vnode3.sim -run general/stable/refcount.sim -run general/stable/show.sim -run general/table/autocreate.sim -run general/table/basic1.sim -run general/table/basic2.sim -run general/table/basic3.sim -run general/table/bigint.sim -run general/table/binary.sim -run general/table/bool.sim -run general/table/column_name.sim -run general/table/column_num.sim -run general/table/column_value.sim -run general/table/column2.sim -run general/table/date.sim -run general/table/db.table.sim -run general/table/delete_reuse1.sim -run general/table/delete_reuse2.sim -run general/table/delete_writing.sim -run general/table/describe.sim -run general/table/double.sim -run general/table/fill.sim -run general/table/float.sim -run general/table/int.sim -run general/table/limit.sim -run general/table/smallint.sim -run general/table/table_len.sim -run general/table/table.sim -run general/table/tinyint.sim -run general/table/vgroup.sim -run general/tag/3.sim -run general/tag/4.sim -run general/tag/5.sim -run general/tag/6.sim -run general/tag/add.sim -run general/tag/bigint.sim -run general/tag/binary_binary.sim -run general/tag/binary.sim -run general/tag/bool_binary.sim -run general/tag/bool_int.sim -run general/tag/bool.sim -run general/tag/change.sim -run general/tag/column.sim -##unsupport run general/tag/commit.sim -run general/tag/create.sim -run general/tag/delete.sim -run general/tag/double.sim -run general/tag/filter.sim -run general/tag/float.sim -run general/tag/int_binary.sim -run general/tag/int_float.sim -run general/tag/int.sim -run general/tag/set.sim -run general/tag/smallint.sim -run general/tag/tinyint.sim -run general/user/authority.sim -run general/user/monitor.sim -run general/user/pass_alter.sim -run general/user/pass_len.sim -run general/user/user_create.sim -run general/user/user_len.sim -run general/vector/metrics_field.sim -run general/vector/metrics_mix.sim -run general/vector/metrics_query.sim -run general/vector/metrics_tag.sim -run general/vector/metrics_time.sim -run general/vector/multi.sim -run general/vector/single.sim -run general/vector/table_field.sim -run general/vector/table_mix.sim -run general/vector/table_query.sim -run general/vector/table_time.sim -run general/stream/restart_stream.sim -run general/stream/stream_3.sim -run general/stream/stream_restart.sim -run general/stream/table_del.sim -run general/stream/metrics_del.sim -run general/stream/table_replica1_vnoden.sim -run general/stream/metrics_replica1_vnoden.sim diff --git a/tests/script/runAllSimCases.sh b/tests/script/runAllSimCases.sh index 279bc8363ea3607dca9541685c3d22fc1394fadf..97bebd9eb82f25b44ac8b2dddfb88dbe0ac9c13a 100755 --- a/tests/script/runAllSimCases.sh +++ b/tests/script/runAllSimCases.sh @@ -1,4 +1,4 @@ -!/bin/bash +#!/bin/bash ################################################## # diff --git a/tests/script/sh/checkValgrind.sh b/tests/script/sh/checkValgrind.sh new file mode 100755 index 0000000000000000000000000000000000000000..17d7ce1aff509c7e753f6306519c10c37e9bd299 --- /dev/null +++ b/tests/script/sh/checkValgrind.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set +e +#set -x + +NODE_NAME= + +while getopts "n:" arg +do + case $arg in + n) + NODE_NAME=$OPTARG + ;; + ?) + echo "unkown argument" + ;; + esac +done + +SCRIPT_DIR=`dirname $0` +cd $SCRIPT_DIR/../ +SCRIPT_DIR=`pwd` + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TAOS_DIR=`pwd` +LOG_DIR=$TAOS_DIR/sim/$NODE_NAME/log +#CFG_DIR=$TAOS_DIR/sim/$NODE_NAME/cfg + +#echo ---- $LOG_DIR + +errors=`grep "ERROR SUMMARY:" ${LOG_DIR}/valgrind-taosd-*.log | cut -d ' ' -f 2,3,4,5 | tr -d "\n"` +echo $errors diff --git a/tests/script/test.sh b/tests/script/test.sh index 14dc43beaf2c104165c954a8aa3c96d7252da03c..e4191da0a91a85584b9aefa21547bcef06e6fb92 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -131,8 +131,8 @@ if [ -n "$FILE_NAME" ]; then FLAG="-v" fi - echo valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG - valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG + echo valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG + valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG else if [[ $MULTIPROCESS -eq 1 ]];then echo "ExcuteCmd(multiprocess):" $PROGRAM -m -c $CFG_DIR -f $FILE_NAME diff --git a/tests/script/tmp/data.sim b/tests/script/tmp/data.sim index 92fc9dccc92903331ea304d45ea1f918a4318e61..4e714c01ee29fb0b29b679c1d123de429b52d785 100644 --- a/tests/script/tmp/data.sim +++ b/tests/script/tmp/data.sim @@ -1,9 +1,14 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c telemetryReporting -v 1 +system sh/cfg.sh -n dnode1 -c telemetryInterval -v 1 +system sh/cfg.sh -n dnode1 -c telemetryServer -v localhost +system sh/cfg.sh -n dnode1 -c telemetryPort -v 80 + +return system sh/exec.sh -n dnode1 -s start sql connect -return sql create database db sql create table db.tb (ts timestamp, i int) sql insert into db.tb values(now, 1) diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 525de9c5c47375b8fd5797450645c3bb3178aed3..2b230c3c452fbd6048ad53d1af5d5408b2153f5c 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -66,7 +66,7 @@ print ============= create database # | REPLICA value [1 | 3] # | WAL value [1 | 2] -sql create database db BLOCKS 7 CACHE 3 CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' QUORUM 1 REPLICA 3 TTL 7 WAL 2 VGROUPS 6 SINGLE_STABLE 1 STREAM_MODE 1 +sql create database db BLOCKS 7 CACHE 3 CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 3 TTL 7 WAL 2 VGROUPS 6 SINGLE_STABLE 1 STREAM_MODE 1 sql show databases print rows: $rows print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -89,7 +89,7 @@ endi if $data4_db != 3 then # replica return -1 endi -if $data5_db != 1 then # quorum +if $data5_db != nostrict then # strict return -1 endi if $data6_db != 345600 then # days @@ -156,25 +156,25 @@ sql_error alter database db replica 0 # return -1 #endi -print ============== modify quorum -sql alter database db quorum 2 -sql show databases -print quorum $data5_db -if $data5_db != 2 then - return -1 -endi -sql alter database db quorum 1 -sql show databases -print quorum $data5_db -if $data5_db != 1 then - return -1 -endi +#print ============== modify quorum +#sql alter database db quorum 2 +#sql show databases +#print quorum $data5_db +#if $data5_db != 2 then +# return -1 +#endi +#sql alter database db quorum 1 +#sql show databases +#print quorum $data5_db +#if $data5_db != 1 then +# return -1 +#endi -sql_error alter database db quorum -1 -sql_error alter database db quorum 0 -sql_error alter database db quorum 3 -sql_error alter database db quorum 4 -sql_error alter database db quorum 5 +#sql_error alter database db quorum -1 +#sql_error alter database db quorum 0 +#sql_error alter database db quorum 3 +#sql_error alter database db quorum 4 +#sql_error alter database db quorum 5 #print ============== modify days sql_error alter database db days 480 diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index 2baba8fb7466610099a8f4597f1fbaa8d6b3168c..cd6a7ee28b442fb7507be11cb68cd1cd308f2ca0 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -115,7 +115,7 @@ if $data4_db != 1 then # replica print expect 1, actual: $data4_db return -1 endi -if $data5_db != 1 then # quorum +if $data5_db != nostrict then # strict return -1 endi if $data6_db != 14400 then # days @@ -355,16 +355,16 @@ print ====> QUORUM value [1 | 2, default: 1] #endi #sql drop database db -sql create database db QUORUM 1 -sql show databases -print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data5_db != 1 then - return -1 -endi -sql drop database db -sql_error create database db QUORUM 3 -sql_error create database db QUORUM 0 -sql_error create database db QUORUM -1 +#sql create database db QUORUM 1 +#sql show databases +#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $data5_db != 1 then +# return -1 +#endi +#sql drop database db +#sql_error create database db QUORUM 3 +#sql_error create database db QUORUM 0 +#sql_error create database db QUORUM -1 print ====> REPLICA value [1 | 3, default: 1] sql create database db REPLICA 3 diff --git a/tests/script/tsim/query/explain.sim b/tests/script/tsim/query/explain.sim index 638109d5105dd95e07f93cf90308124483a5ed82..66d3c48f5d0169f01413805234eb8dfa91f97a22 100644 --- a/tests/script/tsim/query/explain.sim +++ b/tests/script/tsim/query/explain.sim @@ -45,7 +45,7 @@ sql explain select * from information_schema.user_stables; sql explain select count(*),sum(f1) from tb1; sql explain select count(*),sum(f1) from st1; sql explain select count(*),sum(f1) from st1 group by f1; -sql explain select count(f1) from tb1 interval(10s, 2s) sliding(3s) fill(prev); +#sql explain select count(f1) from tb1 interval(10s, 2s) sliding(3s) fill(prev); sql explain select min(f1) from st1 interval(1m, 2a) sliding(30s); print ======== step3 @@ -65,7 +65,7 @@ sql explain analyze select * from information_schema.user_stables; sql explain analyze select count(*),sum(f1) from tb1; sql explain analyze select count(*),sum(f1) from st1; sql explain analyze select count(*),sum(f1) from st1 group by f1; -sql explain analyze select count(f1) from tb1 interval(10s, 2s) sliding(3s) fill(prev); +#sql explain analyze select count(f1) from tb1 interval(10s, 2s) sliding(3s) fill(prev); sql explain analyze select min(f1) from st1 interval(3m, 2a) sliding(1m); print ======== step5 @@ -78,7 +78,7 @@ sql explain analyze verbose true select * from information_schema.user_stables; sql explain analyze verbose true select count(*),sum(f1) from tb1; sql explain analyze verbose true select count(*),sum(f1) from st1; sql explain analyze verbose true select count(*),sum(f1) from st1 group by f1; -sql explain analyze verbose true select count(f1) from tb1 interval(10s, 2s) sliding(3s) fill(prev); +#sql explain analyze verbose true select count(f1) from tb1 interval(10s, 2s) sliding(3s) fill(prev); sql explain analyze verbose true select ts from tb1 where f1 > 0; sql explain analyze verbose true select f1 from st1 where f1 > 0 and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00'; sql explain analyze verbose true select * from information_schema.user_stables where db_name='db2'; diff --git a/tests/script/tsim/query/session.sim b/tests/script/tsim/query/session.sim index d5e77c7b28471fe84ddf6fbe8310ba038f1ac358..a69b6249fc3f0389ca2982d11c9afc1435f7ced6 100644 --- a/tests/script/tsim/query/session.sim +++ b/tests/script/tsim/query/session.sim @@ -279,7 +279,8 @@ endi print ================> syntax error check not active ================> reactive sql_error select * from dev_001 session(ts,1w) -sql_error select count(*) from st session(ts,1w) +print disable this temporarily, session can not be directly applied to super table. +#sql_error select count(*) from st session(ts,1w) sql_error select count(*) from dev_001 group by tagtype session(ts,1w) sql_error sql select count(*) from dev_001 session(ts,1n) sql_error sql select count(*) from dev_001 session(ts,1y) diff --git a/tests/script/tsim/testCaseSuite.sim b/tests/script/tsim/testCaseSuite.sim deleted file mode 100644 index c1b8c0176720b36c16601a8a48b6117f76fa2aed..0000000000000000000000000000000000000000 --- a/tests/script/tsim/testCaseSuite.sim +++ /dev/null @@ -1,31 +0,0 @@ - -run tsim/user/basic1.sim - -run tsim/db/alter_option.sim -run tsim/db/basic1.sim -run tsim/db/basic2.sim -run tsim/db/basic3.sim -run tsim/db/basic6.sim -run tsim/db/basic7.sim -run tsim/db/error1.sim - -run tsim/dnode/basic1.sim - -run tsim/insert/basic0.sim -run tsim/insert/basic1.sim -run tsim/insert/backquote.sim -run tsim/insert/null.sim - -#run tsim/parser/groupby-basic.sim -#run tsim/parser/fourArithmetic-basic.sim - -run tsim/query/interval.sim -run tsim/query/interval-offset.sim -#run tsim/query/scalarFunction.sim - -run tsim/show/basic.sim - -run tsim/table/basic1.sim - -run tsim/tmq/basic.sim -run tsim/tmq/basic1.sim diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index f7bd273624ce1298252aa1653327d562566d7627..df6a553d1a0c2ebc97a0b08d5aab9ccd8fd56544 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -8,196 +8,245 @@ # # notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). # -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi +run tsim/tmq/prepareBasicEnv.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 100 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 3 +$ifcheckdata = 1 +$showMsg = 1 +$showRow = 0 sql connect +sql use $dbName -$dbNamme = d0 -print =============== create database , vgroup 1 -sql create database $dbNamme vgroups 1 -sql use $dbNamme - -print =============== create super table -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) - -sql show stables -if $rows != 1 then - return -1 -endi - -print =============== create child table -sql create table ct0 using stb tags(1000) -sql create table ct1 using stb tags(2000) -#sql create table ct3 using stb tags(3000) +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb -print =============== create normal table -sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 -print =============== create multi topics. notes: now only support: -print =============== 1. columns from stb; 2. * from ctb; 3. columns from ctb -print =============== will support: * from stb; function from stb/ctb +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 -sql create topic topic_stb_column as select ts, c1, c3 from stb -#sql create topic topic_stb_all as select * from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb +#sql show topics +#if $rows != 9 then +# return -1 +#endi -sql create topic topic_ctb_column as select ts, c1, c3 from ct0 -sql create topic topic_ctb_all as select * from ct0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . ' -sql create topic topic_ntb_column as select ts, c1, c3 from ntb -sql create topic topic_ntb_all as select * from ntb -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb +print ================ test consume from stb +$loop_cnt = 0 +loop_consume_diff_topic_from_stb: +if $loop_cnt == 0 then + print == scenario 1: topic_stb_column + $topicList = ' . topic_stb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_stb_all + $topicList = ' . topic_stb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_stb_function + $topicList = ' . topic_stb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_stb_end +endi -sql show tables -if $rows != 3 then +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +#$expectmsgcnt = $totalMsgOfStb + 1 +$expectmsgcnt = 110 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then return -1 endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_stb +loop_consume_diff_topic_from_stb_end: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) -print =============== insert data - -$tbPrefix = ct -$tbNum = 2 -$rowNum = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - - $x = 0 - while $x < $rowNum - $c = $x / 10 - $c = $c * 10 - $c = $x - $c - - $binary = ' . binary - $binary = $binary . $c - $binary = $binary . ' - - sql insert into $tb values ($tstart , $c , $x , $binary ) - sql insert into ntb values ($tstart , $c , $x , $binary ) - $tstart = $tstart + 1 - $x = $x + 1 - endw - - $i = $i + 1 - $tstart = 1640966400000 -endw - -#root@trd02 /home $ tmq_sim --help -# -c Configuration directory, default is -# -d The name of the database for cosumer, no default -# -t The topic string for cosumer, no default -# -k The key-value string for cosumer, no default -# -g showMsgFlag, default is 0 -# - -$totalMsgCnt = $rowNum * $tbNum -print inserted totalMsgCnt: $totalMsgCnt - -# supported key: -# group.id: -# enable.auto.commit: -# auto.offset.reset: -# td.connect.ip: -# td.connect.user:root -# td.connect.pass:taosdata -# td.connect.port:6030 -# td.connect.db:db - -system_content echo -n \$BUILD_DIR -$tmq_sim = $system_content . /build/bin/tmq_sim -system_content echo -n \$SIM_DIR -$tsim_cfg = $system_content . /tsim/cfg - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 20, 0}@ then +sql show tables +if $rows != 2 then return -1 endi +####################################################################################### -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -#print cmd result----> $system_content -#if $system_content != @{consume success: 20, 0}@ then -# return -1 -#endi -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 20, 0}@ then - print expect @{consume success: 20, 0}@, actual: $system_content - return -1 +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: +if $loop_cnt == 0 then + print == scenario 1: topic_ctb_column + $topicList = ' . topic_ctb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ctb_all + $topicList = ' . topic_ctb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ctb_function + $topicList = ' . topic_ctb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ctb_end endi -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 10, 0}@ then +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfCtb + 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then return -1 endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 10, 0}@ then +if $data[0][2] != $totalMsgOfCtb then return -1 endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 10, 0}@ then +if $data[0][3] != $totalMsgOfCtb then return -1 endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ctb +loop_consume_diff_topic_from_ctb_end: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 20, 0}@ then +sql show tables +if $rows != 2 then return -1 endi +####################################################################################### -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 20, 0}@ then - return -1 + +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: +if $loop_cnt == 0 then + print == scenario 1: topic_ntb_column + $topicList = ' . topic_ntb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ntb_all + $topicList = ' . topic_ntb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ntb_function + $topicList = ' . topic_ntb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ntb_end endi -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 20, 0}@ then +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfNtb + 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then return -1 endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ntb +loop_consume_diff_topic_from_ntb_end: -print =============== create database , vgroup 4 -$dbNamme = d1 -sql create database $dbNamme vgroups 4 - +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/clearConsume.sim b/tests/script/tsim/tmq/clearConsume.sim new file mode 100644 index 0000000000000000000000000000000000000000..8b55e0f8c3a8f29791c26f021d3ce8d432d5acfa --- /dev/null +++ b/tests/script/tsim/tmq/clearConsume.sim @@ -0,0 +1,28 @@ +sql connect + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 100 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +sql use $dbName + +print == create consume info table and consume result table +sql drop table consumeinfo +sql drop table consumeresult +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi + diff --git a/tests/script/tsim/tmq/consume.sh b/tests/script/tsim/tmq/consume.sh new file mode 100755 index 0000000000000000000000000000000000000000..28e03d8fb977ca7cabc1e9038175a1619b4f0ec8 --- /dev/null +++ b/tests/script/tsim/tmq/consume.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +################################################## +# +# Do tmq test +# +################################################## + +set +e + +# set default value for parameters +EXEC_OPTON=start +DB_NAME=db +CDB_NAME=db +POLL_DELAY=5 +VALGRIND=0 +SIGNAL=SIGINT +SHOW_MSG=0 +SHOW_ROW=0 + +while getopts "d:s:v:y:x:g:r:w:" arg +do + case $arg in + d) + DB_NAME=$OPTARG + ;; + g) + SHOW_MSG=$OPTARG + ;; + r) + SHOW_ROW=$OPTARG + ;; + s) + EXEC_OPTON=$OPTARG + ;; + v) + VALGRIND=1 + ;; + y) + POLL_DELAY=$OPTARG + ;; + x) + SIGNAL=$OPTARG + ;; + w) + CDB_NAME=$OPTARG + ;; + ?) + echo "unkown argument" + ;; + esac +done + +SCRIPT_DIR=`pwd` + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TOP_DIR=`pwd` + +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "tmq_sim"|grep bin|head -n1|cut -d '/' -f 2,3` +else + BIN_DIR=`find . -name "tmq_sim"|grep bin|head -n1|cut -d '/' -f 2` +fi + +declare -x BUILD_DIR=$TOP_DIR/$BIN_DIR + +declare -x SIM_DIR=$TOP_DIR/sim + +PROGRAM=$BUILD_DIR/build/bin/tmq_sim + +PRG_DIR=$SIM_DIR/tsim +CFG_DIR=$PRG_DIR/cfg +LOG_DIR=$PRG_DIR/log + +echo "------------------------------------------------------------------------" +echo "BUILD_DIR: $BUILD_DIR" +echo "SIM_DIR : $SIM_DIR" +echo "CFG_DIR : $CFG_DIR" + + +echo "PROGRAM: $PROGRAM +echo "CFG_DIR: $CFG_DIR +echo "POLL_DELAY: $POLL_DELAY +echo "DB_NAME: $DB_NAME + +echo "------------------------------------------------------------------------" +if [ "$EXEC_OPTON" = "start" ]; then + if [ $VALGRIND -eq 1 ]; then + echo nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW > /dev/null 2>&1 & + nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW > /dev/null 2>&1 & + else + echo "nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME > /dev/null 2>&1 &" + nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME > /dev/null 2>&1 & + fi +else + PID=`ps -ef|grep tmq_sim | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + if [ "$SIGNAL" = "SIGKILL" ]; then + echo try to kill by signal SIGKILL + kill -9 $PID + else + echo try to kill by signal SIGINT + kill -SIGINT $PID + fi + sleep 1 + PID=`ps -ef|grep tmq_sim | grep -v grep | awk '{print $2}'` + done +fi diff --git a/tests/script/tsim/tmq/prepareBasicEnv.sim b/tests/script/tsim/tmq/prepareBasicEnv.sim new file mode 100644 index 0000000000000000000000000000000000000000..066d7d4ab0bbef8a7954701dc16dd797c9fbd24a --- /dev/null +++ b/tests/script/tsim/tmq/prepareBasicEnv.sim @@ -0,0 +1,88 @@ +# stop all dnodes before start this case +system sh/stop_dnodes.sh + +# deploy dnode 1 +system sh/deploy.sh -n dnode1 -i 1 + +# add some config items for this case +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +# start dnode 1 +system sh/exec.sh -n dnode1 -s start + +sql connect + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 100 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +print == create database $dbName vgroups $vgroups +sql create database $dbName vgroups $vgroups + +#wait database ready +$loop_cnt = 0 +check_db_ready: +if $loop_cnt == 10 then + print ====> database not ready! + return -1 +endi +sql show databases +print ==> rows: $rows +print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $data(db)[20] != nostrict then + sleep 100 + $loop_cnt = $loop_cnt + 1 + goto check_db_ready +endi + +sql use $dbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi + +print == create super table +sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) +sql show stables +if $rows != 1 then + return -1 +endi + +print == create child table, normal table and insert data +$i = 0 +while $i < $ctbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + sql create table $ctb using $stbPrefix tags( $i ) + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) + + $x = 0 + while $x < $rowsPerCtb + $binary = ' . binary- + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , $i , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw diff --git a/tests/script/tsim/valgrind/checkError.sim b/tests/script/tsim/valgrind/checkError.sim new file mode 100644 index 0000000000000000000000000000000000000000..80700dcbce71e80a75771853e84c7008c084b01d --- /dev/null +++ b/tests/script/tsim/valgrind/checkError.sim @@ -0,0 +1,83 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +#system sh/deploy.sh -n dnode2 -i 2 +#system sh/deploy.sh -n dnode3 -i 3 +#system sh/deploy.sh -n dnode4 -i 4 +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +#system sh/exec.sh -n dnode2 -s start +#system sh/exec.sh -n dnode3 -s start +#system sh/exec.sh -n dnode4 -s start + +sleep 2000 + +#$loop_cnt = 0 +#check_dnode_ready: +# $loop_cnt = $loop_cnt + 1 +# sleep 200 +# if $loop_cnt == 10 then +# print ====> dnode not ready! +# return -1 +# endi +#sql show dnodes +#print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +#print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +#print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +#print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +#if $data[0][0] != 1 then +# return -1 +#endi +#if $data[0][4] != ready then +# goto check_dnode_ready +#endi +# +##sql connect +#sql create dnode $hostname port 7200 +#sql create dnode $hostname port 7300 +#sql create dnode $hostname port 7400 +# +#$loop_cnt = 0 +#check_dnode_ready_1: +#$loop_cnt = $loop_cnt + 1 +#sleep 200 +#if $loop_cnt == 10 then +# print ====> dnodes not ready! +# return -1 +#endi +#sql show dnodes +#print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +#print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +#print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +#print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +#if $data[0][4] != ready then +# goto check_dnode_ready_1 +#endi +#if $data[1][4] != ready then +# goto check_dnode_ready_1 +#endi +#if $data[2][4] != ready then +# goto check_dnode_ready_1 +#endi +#if $data[3][4] != ready then +# goto check_dnode_ready_1 +#endi + +#=========== please add any actions above ================= + +print ====> stop all dondes to output valgrind log file +system sh/exec.sh -n dnode1 -s stop -x SIGINT + +print ====> start to check if there are ERRORS in vagrind log file for each dnode +# -n : dnode[x] be check +system_content sh/checkValgrind.sh -n dnode1 +print cmd return result----> [ $system_content ] +if $system_content == @ERROR SUMMARY: 0 errors@ then + return 0 +endi + +$null= +if $system_content == $null then + return 0 +endi + +return -1 \ No newline at end of file diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py new file mode 100644 index 0000000000000000000000000000000000000000..f09e7d1f630c1f1427960b07160bcf07df947425 --- /dev/null +++ b/tests/system-test/2-query/cast.py @@ -0,0 +1,634 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + tdLog.printNoPrefix("==========step2:insert data") + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdSql.query("select c1 from ct4") + data_ct4_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + tdSql.query("select c1 from t1") + data_t1_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + + tdLog.printNoPrefix("==========step2: cast int to bigint, expect no changes") + + tdSql.query("select cast(c1 as bigint) as b from ct4") + for i in range(len(data_ct4_c1)): + tdSql.checkData( i, 0, data_ct4_c1[i]) + tdSql.query("select cast(c1 as bigint) as b from t1") + for i in range(len(data_t1_c1)): + tdSql.checkData( i, 0, data_t1_c1[i]) + + tdLog.printNoPrefix("==========step5: cast int to binary, expect changes to str(int) ") + + tdSql.query("select cast(c1 as binary(32)) as b from ct4") + for i in range(len(data_ct4_c1)): + tdSql.checkData( i, 0, str(data_ct4_c1[i]) ) + tdSql.query("select cast(c1 as binary(32)) as b from t1") + for i in range(len(data_t1_c1)): + tdSql.checkData( i, 0, str(data_t1_c1[i]) ) + + tdLog.printNoPrefix("==========step6: cast int to nchar, expect changes to str(int) ") + + tdSql.query("select cast(c1 as nchar(32)) as b from ct4") + for i in range(len(data_ct4_c1)): + tdSql.checkData( i, 0, str(data_ct4_c1[i]) ) + tdSql.query("select cast(c1 as nchar(32)) as b from t1") + for i in range(len(data_t1_c1)): + tdSql.checkData( i, 0, str(data_t1_c1[i]) ) + + tdLog.printNoPrefix("==========step7: cast int to timestamp, expect changes to timestamp ") + + tdSql.query("select cast(c1 as timestamp) as b from ct4") + for i in range(len(data_ct4_c1)): + if data_ct4_c1[i] is None: + tdSql.checkData( i, 0 , None ) + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c1[i]/1000) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) + + tdSql.query("select cast(c1 as timestamp) as b from t1") + for i in range(len(data_t1_c1)): + if data_ct4_c1[i] is None: + tdSql.checkData( i, 0 , None ) + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c1[i]/1000) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) + + + tdLog.printNoPrefix("==========step8: cast bigint to bigint, expect no changes") + tdSql.query("select c2 from ct4") + data_ct4_c2 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + tdSql.query("select c2 from t1") + data_t1_c2 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + + tdSql.query("select cast(c2 as bigint) as b from ct4") + for i in range(len(data_ct4_c2)): + tdSql.checkData( i, 0, data_ct4_c2[i]) + tdSql.query("select cast(c2 as bigint) as b from t1") + for i in range(len(data_t1_c2)): + tdSql.checkData( i, 0, data_t1_c2[i]) + + + tdLog.printNoPrefix("==========step9: cast bigint to binary, expect changes to str(int) ") + + tdSql.query("select cast(c2 as binary(32)) as b from ct4") + for i in range(len(data_ct4_c2)): + tdSql.checkData( i, 0, str(data_ct4_c2[i]) ) + tdSql.query("select cast(c2 as binary(32)) as b from t1") + for i in range(len(data_t1_c2)): + tdSql.checkData( i, 0, str(data_t1_c2[i]) ) + + tdLog.printNoPrefix("==========step10: cast bigint to nchar, expect changes to str(int) ") + + tdSql.query("select cast(c2 as nchar(32)) as b from ct4") + for i in range(len(data_ct4_c2)): + tdSql.checkData( i, 0, str(data_ct4_c2[i]) ) + tdSql.query("select cast(c2 as nchar(32)) as b from t1") + for i in range(len(data_t1_c2)): + tdSql.checkData( i, 0, str(data_t1_c2[i]) ) + + tdLog.printNoPrefix("==========step11: cast bigint to timestamp, expect changes to timestamp ") + + tdSql.query("select cast(c2 as timestamp) as b from ct4") + for i in range(len(data_ct4_c2)): + if data_ct4_c2[i] is None: + tdSql.checkData( i, 0 , None ) + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c2[i]/1000) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) + + + tdSql.query("select cast(c2 as timestamp) as b from t1") + for i in range(len(data_t1_c2)): + if data_t1_c2[i] is None: + tdSql.checkData( i, 0 , None ) + elif i == 10: + continue + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c2[i]/1000) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) + + + tdLog.printNoPrefix("==========step12: cast smallint to bigint, expect no changes") + tdSql.query("select c3 from ct4") + data_ct4_c3 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + tdSql.query("select c3 from t1") + data_t1_c3 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + + tdSql.query("select cast(c3 as bigint) as b from ct4") + for i in range(len(data_ct4_c3)): + tdSql.checkData( i, 0, data_ct4_c3[i]) + tdSql.query("select cast(c3 as bigint) as b from t1") + for i in range(len(data_t1_c3)): + tdSql.checkData( i, 0, data_t1_c3[i]) + + + tdLog.printNoPrefix("==========step13: cast smallint to binary, expect changes to str(int) ") + + tdSql.query("select cast(c3 as binary(32)) as b from ct4") + for i in range(len(data_ct4_c3)): + tdSql.checkData( i, 0, str(data_ct4_c3[i]) ) + tdSql.query("select cast(c3 as binary(32)) as b from t1") + for i in range(len(data_t1_c3)): + tdSql.checkData( i, 0, str(data_t1_c3[i]) ) + + tdLog.printNoPrefix("==========step14: cast smallint to nchar, expect changes to str(int) ") + + tdSql.query("select cast(c3 as nchar(32)) as b from ct4") + for i in range(len(data_ct4_c3)): + tdSql.checkData( i, 0, str(data_ct4_c3[i]) ) + tdSql.query("select cast(c3 as nchar(32)) as b from t1") + for i in range(len(data_t1_c3)): + tdSql.checkData( i, 0, str(data_t1_c3[i]) ) + + tdLog.printNoPrefix("==========step15: cast smallint to timestamp, expect changes to timestamp ") + + tdSql.query("select cast(c3 as timestamp) as b from ct4") + for i in range(len(data_ct4_c3)): + if data_ct4_c3[i] is None: + tdSql.checkData( i, 0 , None ) + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c3[i]/1000) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) + + tdSql.query("select cast(c3 as timestamp) as b from t1") + for i in range(len(data_t1_c3)): + if data_ct4_c3[i] is None: + tdSql.checkData( i, 0 , None ) + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c3[i]/1000) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) + + + tdLog.printNoPrefix("==========step16: cast smallint to bigint, expect no changes") + tdSql.query("select c4 from ct4") + data_ct4_c4 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + tdSql.query("select c4 from t1") + data_t1_c4 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + + tdSql.query("select cast(c4 as bigint) as b from ct4") + for i in range(len(data_ct4_c4)): + tdSql.checkData( i, 0, data_ct4_c4[i]) + tdSql.query("select cast(c4 as bigint) as b from t1") + for i in range(len(data_t1_c4)): + tdSql.checkData( i, 0, data_t1_c4[i]) + + + tdLog.printNoPrefix("==========step17: cast smallint to binary, expect changes to str(int) ") + + tdSql.query("select cast(c4 as binary(32)) as b from ct4") + for i in range(len(data_ct4_c4)): + tdSql.checkData( i, 0, str(data_ct4_c4[i]) ) + tdSql.query("select cast(c4 as binary(32)) as b from t1") + for i in range(len(data_t1_c4)): + tdSql.checkData( i, 0, str(data_t1_c4[i]) ) + + tdLog.printNoPrefix("==========step18: cast smallint to nchar, expect changes to str(int) ") + + tdSql.query("select cast(c4 as nchar(32)) as b from ct4") + for i in range(len(data_ct4_c4)): + tdSql.checkData( i, 0, str(data_ct4_c4[i]) ) + tdSql.query("select cast(c4 as nchar(32)) as b from t1") + for i in range(len(data_t1_c4)): + tdSql.checkData( i, 0, str(data_t1_c4[i]) ) + + tdLog.printNoPrefix("==========step19: cast smallint to timestamp, expect changes to timestamp ") + + tdSql.query("select cast(c4 as timestamp) as b from ct4") + for i in range(len(data_ct4_c4)): + if data_ct4_c4[i] is None: + tdSql.checkData( i, 0 , None ) + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c4[i]/1000) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) + + tdSql.query("select cast(c4 as timestamp) as b from t1") + for i in range(len(data_t1_c4)): + if data_ct4_c4[i] is None: + tdSql.checkData( i, 0 , None ) + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c4[i]/1000) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) + + + tdLog.printNoPrefix("==========step20: cast float to bigint, expect no changes") + tdSql.query("select c5 from ct4") + data_ct4_c5 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + tdSql.query("select c5 from t1") + data_t1_c5 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + + tdSql.query("select cast(c5 as bigint) as b from ct4") + for i in range(len(data_ct4_c5)): + tdSql.checkData( i, 0, None ) if data_ct4_c5[i] is None else tdSql.checkData( i, 0, int(data_ct4_c5[i]) ) + tdSql.query("select cast(c5 as bigint) as b from t1") + for i in range(len(data_t1_c5)): + tdSql.checkData( i, 0, None ) if data_t1_c5[i] is None else tdSql.checkData( i, 0, int(data_t1_c5[i]) ) + + tdLog.printNoPrefix("==========step21: cast float to binary, expect changes to str(int) ") + tdSql.query("select cast(c5 as binary(32)) as b from ct4") + for i in range(len(data_ct4_c5)): + tdSql.checkData( i, 0, str(data_ct4_c5[i]) ) if data_ct4_c5[i] is None else tdSql.checkData( i, 0, f'{data_ct4_c5[i]:.6f}' ) + tdSql.query("select cast(c5 as binary(32)) as b from t1") + for i in range(len(data_t1_c5)): + tdSql.checkData( i, 0, str(data_t1_c5[i]) ) if data_t1_c5[i] is None else tdSql.checkData( i, 0, f'{data_t1_c5[i]:.6f}' ) + + tdLog.printNoPrefix("==========step22: cast float to nchar, expect changes to str(int) ") + tdSql.query("select cast(c5 as nchar(32)) as b from ct4") + for i in range(len(data_ct4_c5)): + tdSql.checkData( i, 0, None ) if data_ct4_c5[i] is None else tdSql.checkData( i, 0, f'{data_ct4_c5[i]:.6f}' ) + tdSql.query("select cast(c5 as nchar(32)) as b from t1") + for i in range(len(data_t1_c5)): + tdSql.checkData( i, 0, None ) if data_t1_c5[i] is None else tdSql.checkData( i, 0, f'{data_t1_c5[i]:.6f}' ) + + tdLog.printNoPrefix("==========step23: cast float to timestamp, expect changes to timestamp ") + tdSql.query("select cast(c5 as timestamp) as b from ct4") + for i in range(len(data_ct4_c5)): + if data_ct4_c5[i] is None: + tdSql.checkData( i, 0 , None ) + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c5[i]/1000)) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) + tdSql.query("select cast(c5 as timestamp) as b from t1") + for i in range(len(data_t1_c5)): + if data_t1_c5[i] is None: + tdSql.checkData( i, 0 , None ) + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c5[i]/1000)) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) + + tdLog.printNoPrefix("==========step24: cast double to bigint, expect no changes") + tdSql.query("select c6 from ct4") + data_ct4_c6 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + tdSql.query("select c6 from t1") + data_t1_c6 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + + tdSql.query("select cast(c6 as bigint) as b from ct4") + for i in range(len(data_ct4_c6)): + tdSql.checkData( i, 0, None ) if data_ct4_c6[i] is None else tdSql.checkData( i, 0, int(data_ct4_c6[i]) ) + tdSql.query("select cast(c6 as bigint) as b from t1") + for i in range(len(data_t1_c6)): + if data_t1_c6[i] is None: + tdSql.checkData( i, 0, None ) + elif data_t1_c6[i] > 99999999 or data_t1_c6[i] < -999999: + continue + else: + tdSql.checkData( i, 0, int(data_t1_c6[i]) ) + + tdLog.printNoPrefix("==========step25: cast double to binary, expect changes to str(int) ") + tdSql.query("select cast(c6 as binary(32)) as b from ct4") + for i in range(len(data_ct4_c6)): + tdSql.checkData( i, 0, None ) if data_ct4_c6[i] is None else tdSql.checkData( i, 0, f'{data_ct4_c6[i]:.6f}' ) + tdSql.query("select cast(c6 as binary(32)) as b from t1") + for i in range(len(data_t1_c6)): + tdSql.checkData( i, 0, None ) if data_t1_c6[i] is None else tdSql.checkData( i, 0, f'{data_t1_c6[i]:.6f}' ) + + tdLog.printNoPrefix("==========step26: cast double to nchar, expect changes to str(int) ") + tdSql.query("select cast(c6 as nchar(32)) as b from ct4") + for i in range(len(data_ct4_c6)): + tdSql.checkData( i, 0, None ) if data_ct4_c6[i] is None else tdSql.checkData( i, 0, f'{data_ct4_c6[i]:.6f}' ) + tdSql.query("select cast(c6 as nchar(32)) as b from t1") + for i in range(len(data_t1_c6)): + tdSql.checkData( i, 0, None ) if data_t1_c6[i] is None else tdSql.checkData( i, 0, f'{data_t1_c6[i]:.6f}' ) + + tdLog.printNoPrefix("==========step27: cast double to timestamp, expect changes to timestamp ") + tdSql.query("select cast(c6 as timestamp) as b from ct4") + for i in range(len(data_ct4_c6)): + if data_ct4_c6[i] is None: + tdSql.checkData( i, 0 , None ) + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c6[i]/1000)) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) + + tdSql.query("select cast(c6 as timestamp) as b from t1") + for i in range(len(data_t1_c6)): + if data_t1_c6[i] is None: + tdSql.checkData( i, 0 , None ) + elif i == 10: + continue + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c6[i]/1000)) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) + + tdLog.printNoPrefix("==========step28: cast bool to bigint, expect no changes") + tdSql.query("select c7 from ct4") + data_ct4_c7 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + tdSql.query("select c7 from t1") + data_t1_c7 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + + tdSql.query("select cast(c7 as bigint) as b from ct4") + for i in range(len(data_ct4_c7)): + tdSql.checkData( i, 0, data_ct4_c7[i]) + tdSql.query("select cast(c7 as bigint) as b from t1") + for i in range(len(data_t1_c7)): + tdSql.checkData( i, 0, data_t1_c7[i]) + + tdLog.printNoPrefix("==========step29: cast bool to binary, expect changes to str(int) ") + tdSql.query("select cast(c7 as binary(32)) as b from ct4") + for i in range(len(data_ct4_c7)): + tdSql.checkData( i, 0, None ) if data_ct4_c7[i] is None else tdSql.checkData( i, 0, str(data_ct4_c7[i]).lower() ) + tdSql.query("select cast(c7 as binary(32)) as b from t1") + for i in range(len(data_t1_c7)): + tdSql.checkData( i, 0, None ) if data_t1_c7[i] is None else tdSql.checkData( i, 0, str(data_t1_c7[i]).lower() ) + + tdLog.printNoPrefix("==========step30: cast bool to nchar, expect changes to str(int) ") + tdSql.query("select cast(c7 as nchar(32)) as b from ct4") + for i in range(len(data_ct4_c7)): + tdSql.checkData( i, 0, None ) if data_ct4_c7[i] is None else tdSql.checkData( i, 0, str(data_ct4_c7[i]).lower() ) + tdSql.query("select cast(c7 as nchar(32)) as b from t1") + for i in range(len(data_t1_c7)): + tdSql.checkData( i, 0, None ) if data_t1_c7[i] is None else tdSql.checkData( i, 0, str(data_t1_c7[i]).lower() ) + + tdLog.printNoPrefix("==========step31: cast bool to timestamp, expect changes to timestamp ") + tdSql.query("select cast(c7 as timestamp) as b from ct4") + for i in range(len(data_ct4_c7)): + if data_ct4_c7[i] is None: + tdSql.checkData( i, 0 , None ) + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c7[i]/1000)) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) + tdSql.query("select cast(c7 as timestamp) as b from t1") + for i in range(len(data_t1_c7)): + if data_t1_c7[i] is None: + tdSql.checkData( i, 0 , None ) + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c7[i]/1000)) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) + + + tdSql.query("select c8 from ct4") + data_ct4_c8 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + tdSql.query("select c8 from t1") + data_t1_c8 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + + tdLog.printNoPrefix("==========step32: cast binary to binary, expect no changes ") + tdSql.query("select cast(c8 as binary(32)) as b from ct4") + for i in range(len(data_ct4_c8)): + tdSql.checkData( i, 0, None ) if data_ct4_c8[i] is None else tdSql.checkData(i,0,data_ct4_c8[i]) + + tdSql.query("select cast(c8 as binary(32)) as b from t1") + for i in range(len(data_t1_c8)): + tdSql.checkData( i, 0, None ) if data_t1_c8[i] is None else tdSql.checkData(i,0,data_t1_c8[i]) + + tdLog.printNoPrefix("==========step33: cast binary to binary, expect truncate ") + tdSql.query("select cast(c8 as binary(2)) as b from ct4") + for i in range(len(data_ct4_c8)): + if data_ct4_c8[i] is None: + tdSql.checkData( i, 0, None) + elif tdSql.getData(i,0) == data_ct4_c8[i][:2]: + tdLog.info( f"sql:{tdSql.sql}, row:{i} col:0 data:{tdSql.queryResult[i][0]} == expect:{data_ct4_c8[i][:2]}" ) + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{tdSql.sql} row:{i} col:0 data:{tdSql.queryResult[i][0]} != expect:{data_ct4_c8[i][:2]}") + tdSql.query("select cast(c8 as binary(2)) as b from t1") + for i in range(len(data_t1_c8)): + if data_t1_c8[i] is None: + tdSql.checkData( i, 0, None) + elif tdSql.getData(i,0) == data_t1_c8[i][:2]: + tdLog.info( f"sql:{tdSql.sql}, row:{i} col:0 data:{tdSql.queryResult[i][0]} == expect:{data_t1_c8[i][:2]}" ) + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{tdSql.sql} row:{i} col:0 data:{tdSql.queryResult[i][0]} != expect:{data_t1_c8[i][:2]}") + + tdLog.printNoPrefix("==========step34: cast binary to nchar, expect changes to str(int) ") + tdSql.query("select cast(c8 as nchar(32)) as b from ct4") + for i in range(len(data_ct4_c8)): + if data_ct4_c8[i] is None: + tdSql.checkData( i, 0, None) + elif tdSql.getData(i,0) == data_ct4_c8[i]: + tdLog.info( f"sql:{tdSql.sql}, row:{i} col:0 data:{tdSql.queryResult[i][0]} == expect:{data_ct4_c8[i]}" ) + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{tdSql.sql} row:{i} col:0 data:{tdSql.queryResult[i][0]} != expect:{data_ct4_c8[i]}") + tdSql.query("select cast(c8 as nchar(32)) as b from t1") + for i in range(len(data_t1_c8)): + if data_t1_c8[i] is None: + tdSql.checkData( i, 0, None) + elif tdSql.getData(i,0) == data_t1_c8[i]: + tdLog.info( f"sql:{tdSql.sql}, row:{i} col:0 data:{tdSql.queryResult[i][0]} == expect:{data_t1_c8[i]}" ) + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{tdSql.sql} row:{i} col:0 data:{tdSql.queryResult[i][0]} != expect:{data_t1_c8[i]}") + + + tdSql.query("select c9 from ct4") + data_ct4_c9 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + tdSql.query("select c9 from t1") + data_t1_c9 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + "c10 timestamp" + + tdLog.printNoPrefix("==========step35: cast nchar to nchar, expect no changes ") + tdSql.query("select cast(c9 as nchar(32)) as b from ct4") + for i in range(len(data_ct4_c9)): + if data_ct4_c9[i] is None: + tdSql.checkData( i, 0, None) + elif tdSql.getData(i,0) == data_ct4_c9[i]: + tdLog.info( f"sql:{tdSql.sql}, row:{i} col:0 data:{tdSql.queryResult[i][0]} == expect:{data_ct4_c9[i]}" ) + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{tdSql.sql} row:{i} col:0 data:{tdSql.queryResult[i][0]} != expect:{data_ct4_c9[i]}") + tdSql.query("select cast(c9 as nchar(32)) as b from t1") + for i in range(len(data_t1_c9)): + tdSql.checkData( i, 0, data_t1_c9[i] ) + if data_t1_c9[i] is None: + tdSql.checkData( i, 0, None) + elif tdSql.getData(i,0) == data_t1_c9[i]: + tdLog.info( f"sql:{tdSql.sql}, row:{i} col:0 data:{tdSql.queryResult[i][0]} == expect:{data_t1_c9[i]}" ) + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{tdSql.sql} row:{i} col:0 data:{tdSql.queryResult[i][0]} != expect:{data_t1_c9[i]}") + + tdLog.printNoPrefix("==========step36: cast nchar to nchar, expect truncate ") + tdSql.query("select cast(c9 as nchar(2)) as b from ct4") + for i in range(len(data_ct4_c9)): + if data_ct4_c9[i] is None: + tdSql.checkData( i, 0, None) + elif tdSql.getData(i,0) == data_ct4_c9[i][:2]: + tdLog.info( f"sql:{tdSql.sql}, row:{i} col:0 data:{tdSql.queryResult[i][0]} == expect:{data_ct4_c9[i][:2]}" ) + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{tdSql.sql} row:{i} col:0 data:{tdSql.queryResult[i][0]} != expect:{data_ct4_c9[i][:2]}") + tdSql.query("select cast(c9 as nchar(2)) as b from t1") + for i in range(len(data_t1_c9)): + if data_t1_c9[i] is None: + tdSql.checkData( i, 0, None) + elif tdSql.getData(i,0) == data_t1_c9[i][:2]: + tdLog.info( f"sql:{tdSql.sql}, row:{i} col:0 data:{tdSql.queryResult[i][0]} == expect:{data_t1_c9[i][:2]}" ) + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{tdSql.sql} row:{i} col:0 data:{tdSql.queryResult[i][0]} != expect:{data_t1_c9[i][:2]}") + + tdSql.query("select c10 from ct4") + data_ct4_c10 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + tdSql.query("select c10 from t1") + data_t1_c10 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + + tdLog.printNoPrefix("==========step37: cast timestamp to nchar, expect no changes ") + tdSql.query("select cast(c10 as nchar(32)) as b from ct4") + for i in range(len(data_ct4_c10)): + if data_ct4_c10[i] is None: + tdSql.checkData( i, 0, None ) + else: + time2str = str(int(datetime.datetime.timestamp(data_ct4_c10[i])*1000)) + tdSql.checkData( i, 0, time2str ) + tdSql.query("select cast(c10 as nchar(32)) as b from t1") + for i in range(len(data_t1_c10)): + if data_t1_c10[i] is None: + tdSql.checkData( i, 0, None ) + elif i == 10: + continue + else: + time2str = str(int(datetime.datetime.timestamp(data_t1_c10[i])*1000)) + tdSql.checkData( i, 0, time2str ) + + tdLog.printNoPrefix("==========step38: cast timestamp to binary, expect no changes ") + tdSql.query("select cast(c10 as binary(32)) as b from ct4") + for i in range(len(data_ct4_c10)): + if data_ct4_c10[i] is None: + tdSql.checkData( i, 0, None ) + else: + time2str = str(int(datetime.datetime.timestamp(data_ct4_c10[i])*1000)) + tdSql.checkData( i, 0, time2str ) + tdSql.query("select cast(c10 as binary(32)) as b from t1") + for i in range(len(data_t1_c10)): + if data_t1_c10[i] is None: + tdSql.checkData( i, 0, None ) + elif i == 10: + continue + else: + time2str = str(int(datetime.datetime.timestamp(data_t1_c10[i])*1000)) + tdSql.checkData( i, 0, time2str ) + + + tdSql.error("select cast(c1 as int) as b from ct4") + tdSql.error("select cast(c1 as bool) as b from ct4") + tdSql.error("select cast(c1 as tinyint) as b from ct4") + tdSql.error("select cast(c1 as smallint) as b from ct4") + tdSql.error("select cast(c1 as float) as b from ct4") + tdSql.error("select cast(c1 as double) as b from ct4") + tdSql.error("select cast(c1 as tinyint unsigned) as b from ct4") + tdSql.error("select cast(c1 as smallint unsigned) as b from ct4") + tdSql.error("select cast(c1 as int unsigned) as b from ct4") + + tdSql.error("select cast(c2 as int) as b from ct4") + tdSql.error("select cast(c3 as bool) as b from ct4") + tdSql.error("select cast(c4 as tinyint) as b from ct4") + tdSql.error("select cast(c5 as smallint) as b from ct4") + tdSql.error("select cast(c6 as float) as b from ct4") + tdSql.error("select cast(c7 as double) as b from ct4") + tdSql.error("select cast(c8 as tinyint unsigned) as b from ct4") + + tdSql.error("select cast(c8 as timestamp ) as b from ct4") + tdSql.error("select cast(c9 as timestamp ) as b from ct4") + tdSql.error("select cast(c9 as binary(64) ) as b from ct4") + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/distinct.py b/tests/system-test/2-query/distinct.py new file mode 100644 index 0000000000000000000000000000000000000000..a82f3a6f59f243ffc6eafe6f36b1bdcdb9b9e12f --- /dev/null +++ b/tests/system-test/2-query/distinct.py @@ -0,0 +1,262 @@ +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + tdSql.execute("create stable db.stb1 (ts timestamp, c1 int, c2 int) tags(t0 tinyint, t1 int, t2 int)") + tdSql.execute("create stable db.stb2 (ts timestamp, c2 int, c3 binary(16)) tags(t2 binary(16), t3 binary(16), t4 int)") + maxRemainderNum=7 + tbnum=101 + for i in range(tbnum-1): + sql = f"create table db.t{i} using db.stb1 tags({i%maxRemainderNum}, {(i-1)%maxRemainderNum}, {i%2})" + tdSql.execute(sql) + tdSql.execute(f"insert into db.t{i} values (now-10d, {i}, {i%3})") + tdSql.execute(f"insert into db.t{i} values (now-9d, {i}, {(i-1)%3})") + tdSql.execute(f"insert into db.t{i} values (now-8d, {i}, {(i-2)%3})") + tdSql.execute(f"insert into db.t{i} (ts )values (now-7d)") + + tdSql.execute(f"create table db.t0{i} using db.stb2 tags('{i%maxRemainderNum}', '{(i-1)%maxRemainderNum}', {i%3})") + tdSql.execute(f"insert into db.t0{i} values (now-10d, {i}, '{(i+1)%3}')") + tdSql.execute(f"insert into db.t0{i} values (now-9d, {i}, '{(i+2)%3}')") + tdSql.execute(f"insert into db.t0{i} values (now-8d, {i}, '{(i)%3}')") + tdSql.execute(f"insert into db.t0{i} (ts )values (now-7d)") + # tdSql.execute("create table db.t100num using db.stb1 tags(null, null, null)") + # tdSql.execute("create table db.t0100num using db.stb2 tags(null, null, null)") + # tdSql.execute(f"insert into db.t100num values (now-10d, {tbnum-1}, 1)") + # tdSql.execute(f"insert into db.t100num values (now-9d, {tbnum-1}, 0)") + # tdSql.execute(f"insert into db.t100num values (now-8d, {tbnum-1}, 2)") + # tdSql.execute(f"insert into db.t100num (ts )values (now-7d)") + # tdSql.execute(f"insert into db.t0100num values (now-10d, {tbnum-1}, 1)") + # tdSql.execute(f"insert into db.t0100num values (now-9d, {tbnum-1}, 0)") + # tdSql.execute(f"insert into db.t0100num values (now-8d, {tbnum-1}, 2)") + # tdSql.execute(f"insert into db.t0100num (ts )values (now-7d)") + + #========== distinct multi-data-coloumn ========== + # tdSql.query(f"select distinct c1 from stb1 where c1 <{tbnum}") + # tdSql.checkRows(tbnum) + # tdSql.query(f"select distinct c2 from stb1") + # tdSql.checkRows(4) + # tdSql.query(f"select distinct c1,c2 from stb1 where c1 <{tbnum}") + # tdSql.checkRows(tbnum*3) + # tdSql.query(f"select distinct c1,c1 from stb1 where c1 <{tbnum}") + # tdSql.checkRows(tbnum) + # tdSql.query(f"select distinct c1,c2 from stb1 where c1 <{tbnum} limit 3") + # tdSql.checkRows(3) + # tdSql.query(f"select distinct c1,c2 from stb1 where c1 <{tbnum} limit 3 offset {tbnum*3-2}") + # tdSql.checkRows(2) + + tdSql.query(f"select distinct c1 from t1 where c1 <{tbnum}") + tdSql.checkRows(1) + tdSql.query(f"select distinct c2 from t1") + tdSql.checkRows(4) + tdSql.query(f"select distinct c1,c2 from t1 where c1 <{tbnum}") + tdSql.checkRows(3) + tdSql.query(f"select distinct c1,c1 from t1 ") + tdSql.checkRows(2) + tdSql.query(f"select distinct c1,c1 from t1 where c1 <{tbnum}") + tdSql.checkRows(1) + tdSql.query(f"select distinct c1,c2 from t1 where c1 <{tbnum} limit 3") + tdSql.checkRows(3) + tdSql.query(f"select distinct c1,c2 from t1 where c1 <{tbnum} limit 3 offset 2") + tdSql.checkRows(1) + + # tdSql.query(f"select distinct c3 from stb2 where c2 <{tbnum} ") + # tdSql.checkRows(3) + # tdSql.query(f"select distinct c3, c2 from stb2 where c2 <{tbnum} limit 2") + # tdSql.checkRows(2) + + # tdSql.error("select distinct c5 from stb1") + tdSql.error("select distinct c5 from t1") + tdSql.error("select distinct c1 from db.*") + tdSql.error("select c2, distinct c1 from stb1") + tdSql.error("select c2, distinct c1 from t1") + tdSql.error("select distinct c2 from ") + tdSql.error("distinct c2 from stb1") + tdSql.error("distinct c2 from t1") + tdSql.error("select distinct c1, c2, c3 from stb1") + tdSql.error("select distinct c1, c2, c3 from t1") + tdSql.error("select distinct stb1.c1, stb1.c2, stb2.c2, stb2.c3 from stb1") + tdSql.error("select distinct stb1.c1, stb1.c2, stb2.c2, stb2.c3 from t1") + tdSql.error("select distinct t1.c1, t1.c2, t2.c1, t2.c2 from t1") + # tdSql.query(f"select distinct c1 c2, c2 c3 from stb1 where c1 <{tbnum}") + # tdSql.checkRows(tbnum*3) + tdSql.query(f"select distinct c1 c2, c2 c3 from t1 where c1 <{tbnum}") + tdSql.checkRows(3) + # tdSql.error("select distinct c1, c2 from stb1 order by ts") + tdSql.error("select distinct c1, c2 from t1 order by ts") + # tdSql.error("select distinct c1, ts from stb1 group by c2") + tdSql.error("select distinct c1, ts from t1 group by c2") + # tdSql.error("select distinct c1, max(c2) from stb1 ") + tdSql.error("select distinct c1, max(c2) from t1 ") + # tdSql.error("select max(c2), distinct c1 from stb1 ") + tdSql.error("select max(c2), distinct c1 from t1 ") + # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 group by t0") + tdSql.error("select distinct c1, c2 from t1 where c1 > 3 group by t0") + # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 interval(1d) ") + tdSql.error("select distinct c1, c2 from t1 where c1 > 3 interval(1d) ") + # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 interval(1d) fill(next)") + tdSql.error("select distinct c1, c2 from t1 where c1 > 3 interval(1d) fill(next)") + # tdSql.error("select distinct c1, c2 from stb1 where ts > now-10d and ts < now interval(1d) fill(next)") + tdSql.error("select distinct c1, c2 from t1 where ts > now-10d and ts < now interval(1d) fill(next)") + # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 slimit 1") + # tdSql.error("select distinct c1, c2 from t1 where c1 > 3 slimit 1") + # tdSql.query(f"select distinct c1, c2 from stb1 where c1 between {tbnum-2} and {tbnum} ") + # tdSql.checkRows(6) + tdSql.query(f"select distinct c1, c2 from t1 where c1 between {tbnum-2} and {tbnum} ") + # tdSql.checkRows(1) + # tdSql.query("select distinct c1, c2 from stb1 where c1 in (1,2,3,4,5)") + # tdSql.checkRows(15) + tdSql.query("select distinct c1, c2 from t1 where c1 in (1,2,3,4,5)") + # tdSql.checkRows(1) + # tdSql.query("select distinct c1, c2 from stb1 where c1 in (100,1000,10000)") + # tdSql.checkRows(3) + tdSql.query("select distinct c1, c2 from t1 where c1 in (100,1000,10000)") + # tdSql.checkRows(0) + + # tdSql.query(f"select distinct c1,c2 from (select * from stb1 where c1 > {tbnum-2}) ") + # tdSql.checkRows(3) + # tdSql.query(f"select distinct c1,c2 from (select * from t1 where c1 < {tbnum}) ") + # tdSql.checkRows(3) + # tdSql.query(f"select distinct c1,c2 from (select * from stb1 where t2 !=0 and t2 != 1) ") + # tdSql.checkRows(0) + # tdSql.error("select distinct c1, c2 from (select distinct c1, c2 from stb1 where t0 > 2 and t1 < 3) ") + # tdSql.error("select c1, c2 from (select distinct c1, c2 from stb1 where t0 > 2 and t1 < 3) ") + # tdSql.query("select distinct c1, c2 from (select c2, c1 from stb1 where c1 > 2 ) where c1 < 4") + # tdSql.checkRows(3) + # tdSql.error("select distinct c1, c2 from (select c1 from stb1 where t0 > 2 ) where t1 < 3") + # tdSql.error("select distinct c1, c2 from (select c2, c1 from stb1 where c1 > 2 order by ts)") + # tdSql.error("select distinct c1, c2 from (select c2, c1 from t1 where c1 > 2 order by ts)") + # tdSql.error("select distinct c1, c2 from (select c2, c1 from stb1 where c1 > 2 group by c1)") + # tdSql.error("select distinct c1, c2 from (select max(c1) c1, max(c2) c2 from stb1 group by c1)") + # tdSql.error("select distinct c1, c2 from (select max(c1) c1, max(c2) c2 from t1 group by c1)") + # tdSql.query("select distinct c1, c2 from (select max(c1) c1, max(c2) c2 from stb1 )") + # tdSql.checkRows(1) + # tdSql.query("select distinct c1, c2 from (select max(c1) c1, max(c2) c2 from t1 )") + # tdSql.checkRows(1) + # tdSql.error("select distinct stb1.c1, stb1.c2 from stb1 , stb2 where stb1.ts=stb2.ts and stb1.t2=stb2.t4") + # tdSql.error("select distinct t1.c1, t1.c2 from t1 , t2 where t1.ts=t2.ts ") + + # tdSql.error("select distinct c1, c2 from (select count(c1) c1, count(c2) c2 from stb1 group by ts)") + # tdSql.error("select distinct c1, c2 from (select count(c1) c1, count(c2) c2 from t1 group by ts)") + + + + # #========== suport distinct multi-tags-coloumn ========== + # tdSql.query("select distinct t1 from stb1") + # tdSql.checkRows(maxRemainderNum+1) + # tdSql.query("select distinct t0, t1 from stb1") + # tdSql.checkRows(maxRemainderNum+1) + # tdSql.query("select distinct t1, t0 from stb1") + # tdSql.checkRows(maxRemainderNum+1) + # tdSql.query("select distinct t1, t2 from stb1") + # tdSql.checkRows(maxRemainderNum*2+1) + # tdSql.query("select distinct t0, t1, t2 from stb1") + # tdSql.checkRows(maxRemainderNum*2+1) + # tdSql.query("select distinct t0 t1, t1 t2 from stb1") + # tdSql.checkRows(maxRemainderNum+1) + # tdSql.query("select distinct t0, t0, t0 from stb1") + # tdSql.checkRows(maxRemainderNum+1) + # tdSql.query("select distinct t0, t1 from t1") + # tdSql.checkRows(1) + # tdSql.query("select distinct t0, t1 from t100num") + # tdSql.checkRows(1) + + # tdSql.query("select distinct t3 from stb2") + # tdSql.checkRows(maxRemainderNum+1) + # tdSql.query("select distinct t2, t3 from stb2") + # tdSql.checkRows(maxRemainderNum+1) + # tdSql.query("select distinct t3, t2 from stb2") + # tdSql.checkRows(maxRemainderNum+1) + # tdSql.query("select distinct t4, t2 from stb2") + # tdSql.checkRows(maxRemainderNum*3+1) + # tdSql.query("select distinct t2, t3, t4 from stb2") + # tdSql.checkRows(maxRemainderNum*3+1) + # tdSql.query("select distinct t2 t1, t3 t2 from stb2") + # tdSql.checkRows(maxRemainderNum+1) + # tdSql.query("select distinct t3, t3, t3 from stb2") + # tdSql.checkRows(maxRemainderNum+1) + # tdSql.query("select distinct t2, t3 from t01") + # tdSql.checkRows(1) + # tdSql.query("select distinct t3, t4 from t0100num") + # tdSql.checkRows(1) + + + # ########## should be error ######### + # tdSql.error("select distinct from stb1") + # tdSql.error("select distinct t3 from stb1") + # tdSql.error("select distinct t1 from db.*") + # tdSql.error("select distinct t2 from ") + # tdSql.error("distinct t2 from stb1") + # tdSql.error("select distinct stb1") + # tdSql.error("select distinct t0, t1, t2, t3 from stb1") + # tdSql.error("select distinct stb1.t0, stb1.t1, stb2.t2, stb2.t3 from stb1") + + # tdSql.error("select dist t0 from stb1") + # tdSql.error("select distinct stb2.t2, stb2.t3 from stb1") + # tdSql.error("select distinct stb2.t2 t1, stb2.t3 t2 from stb1") + + # tdSql.error("select distinct t0, t1 from t1 where t0 < 7") + + # ########## add where condition ########## + # tdSql.query("select distinct t0, t1 from stb1 where t1 > 3") + # tdSql.checkRows(3) + # tdSql.query("select distinct t0, t1 from stb1 where t1 > 3 limit 2") + # tdSql.checkRows(2) + # tdSql.query("select distinct t0, t1 from stb1 where t1 > 3 limit 2 offset 2") + # tdSql.checkRows(1) + # tdSql.query("select distinct t0, t1 from stb1 where t1 > 3 slimit 2") + # tdSql.checkRows(3) + # tdSql.error("select distinct t0, t1 from stb1 where c1 > 2") + # tdSql.query("select distinct t0, t1 from stb1 where t1 > 3 and t1 < 5") + # tdSql.checkRows(1) + # tdSql.error("select distinct stb1.t0, stb1.t1 from stb1, stb2 where stb1.t2=stb2.t4") + # tdSql.error("select distinct t0, t1 from stb1 where stb2.t4 > 2") + # tdSql.error("select distinct t0, t1 from stb1 where t1 > 3 group by t0") + # tdSql.error("select distinct t0, t1 from stb1 where t1 > 3 interval(1d) ") + # tdSql.error("select distinct t0, t1 from stb1 where t1 > 3 interval(1d) fill(next)") + # tdSql.error("select distinct t0, t1 from stb1 where ts > now-10d and ts < now interval(1d) fill(next)") + + # tdSql.error("select max(c1), distinct t0 from stb1 where t0 > 2") + # tdSql.error("select distinct t0, max(c1) from stb1 where t0 > 2") + # tdSql.error("select distinct t0 from stb1 where t0 in (select t0 from stb1 where t0 > 2)") + # tdSql.query("select distinct t0, t1 from stb1 where t0 in (1,2,3,4,5)") + # tdSql.checkRows(5) + # tdSql.query("select distinct t1 from (select t0, t1 from stb1 where t0 > 2) ") + # tdSql.checkRows(4) + # tdSql.error("select distinct t1 from (select distinct t0, t1 from stb1 where t0 > 2 and t1 < 3) ") + # tdSql.error("select distinct t1 from (select distinct t0, t1 from stb1 where t0 > 2 ) where t1 < 3") + # tdSql.query("select distinct t1 from (select t0, t1 from stb1 where t0 > 2 ) where t1 < 3") + # tdSql.checkRows(1) + # tdSql.error("select distinct t1, t0 from (select t1 from stb1 where t0 > 2 ) where t1 < 3") + # tdSql.error("select distinct t1, t0 from (select max(t1) t1, max(t0) t0 from stb1 group by t1)") + # tdSql.error("select distinct t1, t0 from (select max(t1) t1, max(t0) t0 from stb1)") + # tdSql.query("select distinct t1, t0 from (select t1,t0 from stb1 where t0 > 2 ) where t1 < 3") + # tdSql.checkRows(1) + # tdSql.error(" select distinct t1, t0 from (select t1,t0 from stb1 where t0 > 2 order by ts) where t1 < 3") + # tdSql.error("select t1, t0 from (select distinct t1,t0 from stb1 where t0 > 2 ) where t1 < 3") + # tdSql.error(" select distinct t1, t0 from (select t1,t0 from stb1 where t0 > 2 group by ts) where t1 < 3") + # tdSql.error("select distinct stb1.t1, stb1.t2 from stb1 , stb2 where stb1.ts=stb2.ts and stb1.t2=stb2.t4") + # tdSql.error("select distinct t1.t1, t1.t2 from t1 , t2 where t1.ts=t2.ts ") + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/varchar.py b/tests/system-test/2-query/varchar.py new file mode 100644 index 0000000000000000000000000000000000000000..5cc6c8e39965453c646cb267774564af1a66f42d --- /dev/null +++ b/tests/system-test/2-query/varchar.py @@ -0,0 +1,128 @@ +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 varchar(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 varchar(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + tdLog.printNoPrefix("==========step2:insert data") + for i in range(9): + tdSql.execute( + + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'varchar{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'varchar{i}', 'nchar{i}', now()+{1*i}a )" + ) + + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'varchar0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'varchar9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "varchar1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "varchar2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "varchar3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "varchar4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "varchar5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "varchar6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "varchar7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "varchar8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "varchar9 ", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + data_ct1_c8 = ["varchar8","varchar7","varchar6","varchar5","varchar0","varchar4","varchar3","varchar2","varchar1","varchar0","varchar9"] + + tdLog.printNoPrefix("==========step3: cast on varchar") + + tdSql.query("select c8 from ct1") + for i in range(tdSql.queryRows): + tdSql.checkData(i,0, data_ct1_c8[i]) + + + # tdSql.query("select c8 from ct4") + # data_ct4 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + # tdSql.query("select c8 from t1") + # data_t1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + + # tdLog.printNoPrefix("==========step4: cast on varchar") + + # tdSql.query("select cast(c8 as nchar(16)) as b from ct4") + # for i in range(len(data_ct4)): + # tdSql.checkData( i, 0, data_ct4[i]) + # tdSql.query("select cast(c8 as nchar(16)) as b from t1") + # for i in range(len(data_t1)): + # tdSql.checkData( i, 0, data_t1[i]) + + # tdSql.query("select cast(c8 as nchar(2)) as b from t1") + # for i in range(len(data_t1)): + # tdSql.checkData( i, 0, data_t1[i][:2]) + + + # tdSql.error("select cast(c1 as int) as b from ct4") + # tdSql.error("select cast(c1 as bool) as b from ct4") + # tdSql.error("select cast(c1 as tinyint) as b from ct4") + # tdSql.error("select cast(c1 as smallint) as b from ct4") + # tdSql.error("select cast(c1 as float) as b from ct4") + # tdSql.error("select cast(c1 as double) as b from ct4") + # tdSql.error("select cast(c1 as tinyint unsigned) as b from ct4") + # tdSql.error("select cast(c1 as smallint unsigned) as b from ct4") + # tdSql.error("select cast(c1 as int unsigned) as b from ct4") + + # tdSql.error("select cast(c2 as int) as b from ct4") + # tdSql.error("select cast(c3 as bool) as b from ct4") + # tdSql.error("select cast(c4 as tinyint) as b from ct4") + # tdSql.error("select cast(c5 as smallint) as b from ct4") + # tdSql.error("select cast(c6 as float) as b from ct4") + # tdSql.error("select cast(c7 as double) as b from ct4") + # tdSql.error("select cast(c8 as tinyint unsigned) as b from ct4") + + # tdSql.error("select cast(c8 as timestamp ) as b from ct4") + + # tdSql.error("select cast(c9 as binary(64) ) as b from ct4") + # tdSql.error("select cast(c9 as timestamp ) as b from ct4") + # tdSql.error("select cast(c10 as binary(64) ) as b from ct4") + # tdSql.error("select cast(c10 as nchar(64) ) as b from ct4") + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 0b8bbd24b2fa0a15424aa4d5f97719a17f852141..65e4785b5d10cbe1baf6ec5e3f3816fd9b7c21d9 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -1,17 +1,7 @@ -python3 ./test.py -f 2-query/between.py - - - - - - - - - - - - - - - +#!/bin/bash +set -e +#python3 ./test.py -f 2-query/between.py +python3 ./test.py -f 2-query/distinct.py +python3 ./test.py -f 2-query/varchar.py +python3 ./test.py -f 2-query/cast.py diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 2b3e9963f08bf21f2a8b527c31a7bfde5d016d4b..dc1a77c23155ba6bc29aed77362c58823295e063 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -14,14 +14,13 @@ */ #include -#include #include #include #include #include #include #include -#include +// #include #include "taos.h" #include "taoserror.h" @@ -227,7 +226,7 @@ int64_t getDirectorySize(char* dir) { } } - taosCloseDir(pDir); + taosCloseDir(&pDir); return totalSize; } @@ -340,7 +339,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); - tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); assert(tmq); tmq_conf_destroy(conf); return tmq; @@ -367,7 +366,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1); if (tmqmessage) { /*msg_process(tmqmessage);*/ - tmq_message_destroy(tmqmessage); + taos_free_result(tmqmessage); if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0); } @@ -400,7 +399,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog if (0 != g_stConfInfo.showMsgFlag) { /*msg_process(tmqmessage);*/ } - tmq_message_destroy(tmqmessage); + taos_free_result(tmqmessage); } else { break; } @@ -595,8 +594,8 @@ void printParaIntoFile() { taosOpenFile(g_stConfInfo.resultFileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM); if (NULL == pFile) { fprintf(stderr, "Failed to open %s for save result\n", g_stConfInfo.resultFileName); - exit - 1; - }; + exit(-1); + } g_fp = pFile; time_t tTime = taosGetTimestampSec(); diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index eaca8f151e9311fa113de29b6f1be838716f1239..f9ba0e418cd2b4be6258f0d7001bf279741d4a5e 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -14,14 +14,13 @@ */ #include -#include #include #include #include #include #include #include -#include +// #include #include "taos.h" #include "taoserror.h" @@ -31,44 +30,49 @@ #define NC "\033[0m" #define min(a, b) (((a) < (b)) ? (a) : (b)) -#define MAX_SQL_STR_LEN (1024 * 1024) -#define MAX_ROW_STR_LEN (16 * 1024) +#define MAX_SQL_STR_LEN (1024 * 1024) +#define MAX_ROW_STR_LEN (16 * 1024) +#define MAX_CONSUMER_THREAD_CNT (16) typedef struct { - int32_t expectMsgCnt; - int32_t consumeMsgCnt; - TdThread thread; + TdThread thread; + int32_t consumerId; + + int32_t ifCheckData; + int64_t expectMsgCnt; + + int64_t consumeMsgCnt; + int64_t consumeRowCnt; + int32_t checkresult; + + char topicString[1024]; + char keyString[1024]; + + int32_t numOfTopic; + char topics[32][64]; + + int32_t numOfKey; + char key[32][64]; + char value[32][64]; + + tmq_t* tmq; + tmq_list_t* topicList; + } SThreadInfo; typedef struct { // input from argvs - char dbName[32]; - char topicString[256]; - char keyString[1024]; - char topicString1[256]; - char keyString1[1024]; - int32_t showMsgFlag; - int32_t consumeDelay; // unit s - int32_t consumeMsgCnt; - int32_t checkMode; - - // save result after parse agrvs - int32_t numOfTopic; - char topics[32][64]; - - int32_t numOfKey; - char key[32][64]; - char value[32][64]; - - int32_t numOfTopic1; - char topics1[32][64]; - - int32_t numOfKey1; - char key1[32][64]; - char value1[32][64]; + char cdbName[32]; + char dbName[32]; + int32_t showMsgFlag; + int32_t showRowFlag; + int32_t consumeDelay; // unit s + int32_t numOfThread; + SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; } SConfInfo; static SConfInfo g_stConfInfo; +TdFilePtr g_fp = NULL; // char* g_pRowValue = NULL; // TdFilePtr g_fp = NULL; @@ -81,30 +85,65 @@ static void printHelp() { printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); printf("%s%s\n", indent, "-d"); printf("%s%s%s\n", indent, indent, "The name of the database for cosumer, no default "); - printf("%s%s\n", indent, "-t"); - printf("%s%s%s\n", indent, indent, "The topic string for cosumer, no default "); - printf("%s%s\n", indent, "-k"); - printf("%s%s%s\n", indent, indent, "The key-value string for cosumer, no default "); - printf("%s%s\n", indent, "-t1"); - printf("%s%s%s\n", indent, indent, "The topic1 string for cosumer, no default "); - printf("%s%s\n", indent, "-k1"); - printf("%s%s%s\n", indent, indent, "The key1-value1 string for cosumer, no default "); printf("%s%s\n", indent, "-g"); printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "showRowFlag, default is ", g_stConfInfo.showRowFlag); printf("%s%s\n", indent, "-y"); printf("%s%s%s%d\n", indent, indent, "consume delay, default is s", g_stConfInfo.consumeDelay); - printf("%s%s\n", indent, "-m"); - printf("%s%s%s%d\n", indent, indent, "consume msg count, default is s", g_stConfInfo.consumeMsgCnt); - printf("%s%s\n", indent, "-j"); - printf("%s%s%s%d\n", indent, indent, "check mode, default is s", g_stConfInfo.checkMode); exit(EXIT_SUCCESS); } + +void initLogFile() { + // FILE *fp = fopen(g_stConfInfo.resultFileName, "a"); + char file[256]; + sprintf(file, "%s/../log/tmqlog.txt", configDir); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM); + if (NULL == pFile) { + fprintf(stderr, "Failed to open %s for save result\n", "./tmqlog.txt"); + exit(-1); + } + g_fp = pFile; +} + + +void saveConfigToLogFile() { + time_t tTime = taosGetTimestampSec(); + struct tm tm = *taosLocalTime(&tTime, NULL); + + taosFprintfFile(g_fp, "###################################################################\n"); + taosFprintfFile(g_fp, "# configDir: %s\n", configDir); + taosFprintfFile(g_fp, "# dbName: %s\n", g_stConfInfo.dbName); + taosFprintfFile(g_fp, "# cdbName: %s\n", g_stConfInfo.cdbName); + taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); + taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); + taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); + taosFprintfFile(g_fp, " Topics: "); + for (int i = 0 ; i < g_stConfInfo.stThreads[i].numOfTopic; i++) { + taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[i]); + } + taosFprintfFile(g_fp, "\n"); + taosFprintfFile(g_fp, " Key: "); + for (int i = 0 ; i < g_stConfInfo.stThreads[i].numOfKey; i++) { + taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[i], g_stConfInfo.stThreads[i].value[i]); + } + taosFprintfFile(g_fp, "\n"); + } + + taosFprintfFile(g_fp, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + taosFprintfFile(g_fp, "###################################################################\n"); +} + void parseArgument(int32_t argc, char* argv[]) { memset(&g_stConfInfo, 0, sizeof(SConfInfo)); g_stConfInfo.showMsgFlag = 0; - g_stConfInfo.consumeDelay = 8000; - g_stConfInfo.consumeMsgCnt = 0; + g_stConfInfo.showRowFlag = 0; + g_stConfInfo.consumeDelay = 5; for (int32_t i = 1; i < argc; i++) { if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { @@ -112,40 +151,33 @@ void parseArgument(int32_t argc, char* argv[]) { exit(0); } else if (strcmp(argv[i], "-d") == 0) { strcpy(g_stConfInfo.dbName, argv[++i]); + } else if (strcmp(argv[i], "-w") == 0) { + strcpy(g_stConfInfo.cdbName, argv[++i]); } else if (strcmp(argv[i], "-c") == 0) { strcpy(configDir, argv[++i]); - } else if (strcmp(argv[i], "-t") == 0) { - strcpy(g_stConfInfo.topicString, argv[++i]); - } else if (strcmp(argv[i], "-k") == 0) { - strcpy(g_stConfInfo.keyString, argv[++i]); - } else if (strcmp(argv[i], "-t1") == 0) { - strcpy(g_stConfInfo.topicString1, argv[++i]); - } else if (strcmp(argv[i], "-k1") == 0) { - strcpy(g_stConfInfo.keyString1, argv[++i]); } else if (strcmp(argv[i], "-g") == 0) { g_stConfInfo.showMsgFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + g_stConfInfo.showRowFlag = atol(argv[++i]); } else if (strcmp(argv[i], "-y") == 0) { g_stConfInfo.consumeDelay = atol(argv[++i]); - } else if (strcmp(argv[i], "-m") == 0) { - g_stConfInfo.consumeMsgCnt = atol(argv[++i]); - } else if (strcmp(argv[i], "-j") == 0) { - g_stConfInfo.checkMode = atol(argv[++i]); } else { printf("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); } } - if (0 == g_stConfInfo.consumeMsgCnt) { - g_stConfInfo.consumeMsgCnt = 0x7fffffff; - } + initLogFile(); + + taosFprintfFile(g_fp, "====parseArgument() success\n"); -#if 0 +#if 1 pPrint("%s configDir:%s %s", GREEN, configDir, NC); pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC); - pPrint("%s topicString:%s %s", GREEN, g_stConfInfo.topicString, NC); - pPrint("%s keyString:%s %s", GREEN, g_stConfInfo.keyString, NC); + pPrint("%s cdbName:%s %s", GREEN, g_stConfInfo.cdbName, NC); + pPrint("%s consumeDelay:%d %s", GREEN, g_stConfInfo.consumeDelay, NC); pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); + pPrint("%s showRowFlag:%d %s", GREEN, g_stConfInfo.showRowFlag, NC); #endif } @@ -171,74 +203,31 @@ void ltrim(char* str) { // return str; } -void parseInputString() { - // printf("topicString: %s\n", g_stConfInfo.topicString); - // printf("keyString: %s\n\n", g_stConfInfo.keyString); - - char* token; - const char delim[2] = ","; - const char ch = ':'; - - token = strtok(g_stConfInfo.topicString, delim); - while (token != NULL) { - // printf("%s\n", token ); - strcpy(g_stConfInfo.topics[g_stConfInfo.numOfTopic], token); - ltrim(g_stConfInfo.topics[g_stConfInfo.numOfTopic]); - // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); - g_stConfInfo.numOfTopic++; - - token = strtok(NULL, delim); - } - - token = strtok(g_stConfInfo.topicString1, delim); - while (token != NULL) { - // printf("%s\n", token ); - strcpy(g_stConfInfo.topics1[g_stConfInfo.numOfTopic1], token); - ltrim(g_stConfInfo.topics1[g_stConfInfo.numOfTopic1]); - // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); - g_stConfInfo.numOfTopic1++; - - token = strtok(NULL, delim); - } - - token = strtok(g_stConfInfo.keyString, delim); - while (token != NULL) { - // printf("%s\n", token ); - { - char* pstr = token; - ltrim(pstr); - char* ret = strchr(pstr, ch); - memcpy(g_stConfInfo.key[g_stConfInfo.numOfKey], pstr, ret - pstr); - strcpy(g_stConfInfo.value[g_stConfInfo.numOfKey], ret + 1); - // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], - // g_stConfInfo.value[g_stConfInfo.numOfKey]); - g_stConfInfo.numOfKey++; - } +static int running = 1; +static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) { + char buf[1024]; + int32_t totalRows = 0; - token = strtok(NULL, delim); + //printf("topic: %s\n", tmq_get_topic_name(msg)); + //printf("vg:%d\n", tmq_get_vgroup_id(msg)); + taosFprintfFile(g_fp, "msg index:%" PRId64 ", threadLable: %d\n", msgIndex, threadLable); + taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg)); + + while (1) { + TAOS_ROW row = taos_fetch_row(msg); + if (row == NULL) break; + if (0 != g_stConfInfo.showRowFlag) { + TAOS_FIELD* fields = taos_fetch_fields(msg); + int32_t numOfFields = taos_field_count(msg); + taos_print_row(buf, row, fields, numOfFields); + taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); + } + totalRows++; } - token = strtok(g_stConfInfo.keyString1, delim); - while (token != NULL) { - // printf("%s\n", token ); - { - char* pstr = token; - ltrim(pstr); - char* ret = strchr(pstr, ch); - memcpy(g_stConfInfo.key1[g_stConfInfo.numOfKey1], pstr, ret - pstr); - strcpy(g_stConfInfo.value1[g_stConfInfo.numOfKey1], ret + 1); - // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], - // g_stConfInfo.value[g_stConfInfo.numOfKey]); - g_stConfInfo.numOfKey1++; - } - - token = strtok(NULL, delim); - } + return totalRows; } -static int running = 1; -/*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/ - int queryDB(TAOS* taos, char* command) { TAOS_RES* pRes = taos_query(taos, command); int code = taos_errno(pRes); @@ -252,288 +241,261 @@ int queryDB(TAOS* taos, char* command) { return 0; } -tmq_t* build_consumer() { -#if 0 - char sqlStr[1024] = {0}; - - TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - sprintf(sqlStr, "use %s", g_stConfInfo.dbName); - TAOS_RES* pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - exit(-1); - } - taos_free_result(pRes); -#endif +static void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets) { + printf("tmq_commit_cb_print() commit %d\n", resp); +} +void build_consumer(SThreadInfo *pInfo) { tmq_conf_t* conf = tmq_conf_new(); - // tmq_conf_set(conf, "group.id", "tg2"); - for (int32_t i = 0; i < g_stConfInfo.numOfKey; i++) { - tmq_conf_set(conf, g_stConfInfo.key[i], g_stConfInfo.value[i]); - } + + //tmq_conf_set(conf, "td.connect.ip", "localhost"); + //tmq_conf_set(conf, "td.connect.port", "6030"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); - tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0); - assert(tmq); - tmq_conf_destroy(conf); - return tmq; + + tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); + + // tmq_conf_set(conf, "group.id", "cgrp1"); + for (int32_t i = 0; i < pInfo->numOfKey; i++) { + tmq_conf_set(conf, pInfo->key[i], pInfo->value[i]); + } + + //tmq_conf_set(conf, "client.id", "c-001"); + + //tmq_conf_set(conf, "enable.auto.commit", "true"); + //tmq_conf_set(conf, "enable.auto.commit", "false"); + + //tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + + //tmq_conf_set(conf, "auto.offset.reset", "none"); + //tmq_conf_set(conf, "auto.offset.reset", "earliest"); + //tmq_conf_set(conf, "auto.offset.reset", "latest"); + + pInfo->tmq = tmq_consumer_new(conf, NULL, 0); + return; } -tmq_list_t* build_topic_list() { - tmq_list_t* topic_list = tmq_list_new(); +void build_topic_list(SThreadInfo *pInfo) { + pInfo->topicList = tmq_list_new(); // tmq_list_append(topic_list, "test_stb_topic_1"); - for (int32_t i = 0; i < g_stConfInfo.numOfTopic; i++) { - tmq_list_append(topic_list, g_stConfInfo.topics[i]); + for (int32_t i = 0; i < pInfo->numOfTopic; i++) { + tmq_list_append(pInfo->topicList, pInfo->topics[i]); } - return topic_list; + return; } -tmq_t* build_consumer_x() { -#if 0 +int32_t saveConsumeResult(SThreadInfo *pInfo) { char sqlStr[1024] = {0}; - + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); assert(pConn != NULL); - - sprintf(sqlStr, "use %s", g_stConfInfo.dbName); + + // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int + sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", + g_stConfInfo.cdbName, + pInfo->consumerId, + pInfo->consumeMsgCnt, + pInfo->consumeRowCnt, + pInfo->checkresult); + TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); + printf("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); exit(-1); } + taos_free_result(pRes); -#endif - - tmq_conf_t* conf = tmq_conf_new(); - // tmq_conf_set(conf, "group.id", "tg2"); - for (int32_t i = 0; i < g_stConfInfo.numOfKey1; i++) { - tmq_conf_set(conf, g_stConfInfo.key1[i], g_stConfInfo.value1[i]); - } - tmq_conf_set(conf, "td.connect.user", "root"); - tmq_conf_set(conf, "td.connect.pass", "taosdata"); - tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); - tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0); - assert(tmq); - tmq_conf_destroy(conf); - return tmq; -} -tmq_list_t* build_topic_list_x() { - tmq_list_t* topic_list = tmq_list_new(); - // tmq_list_append(topic_list, "test_stb_topic_1"); - for (int32_t i = 0; i < g_stConfInfo.numOfTopic1; i++) { - tmq_list_append(topic_list, g_stConfInfo.topics1[i]); - } - return topic_list; + return 0; } -void loop_consume(tmq_t* tmq) { +void loop_consume(SThreadInfo *pInfo) { tmq_resp_err_t err; + + int64_t totalMsgs = 0; + int64_t totalRows = 0; - int32_t totalMsgs = 0; - int32_t totalRows = 0; - int32_t skipLogNum = 0; while (running) { - TAOS_RES* tmqMsg = tmq_consumer_poll(tmq, 8000); - if (tmqMsg) { - totalMsgs++; - -#if 0 - TAOS_ROW row; - while (NULL != (row = tmq_get_row(tmqMsg))) { - totalRows++; - } -#endif - - /*skipLogNum += tmqGetSkipLogNum(tmqMsg);*/ + TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000); + if (tmqMsg) { if (0 != g_stConfInfo.showMsgFlag) { - /*msg_process(tmqMsg);*/ + totalRows += msg_process(tmqMsg, totalMsgs, pInfo->consumerId); } - tmq_message_destroy(tmqMsg); - } else { - break; - } - } - - err = tmq_consumer_close(tmq); - if (err) { - printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); - exit(-1); - } - - printf("{consume success: %d, %d}", totalMsgs, totalRows); -} -int32_t parallel_consume(tmq_t* tmq, int threadLable) { - tmq_resp_err_t err; + taos_free_result(tmqMsg); - int32_t totalMsgs = 0; - int32_t totalRows = 0; - int32_t skipLogNum = 0; - while (running) { - TAOS_RES* tmqMsg = tmq_consumer_poll(tmq, g_stConfInfo.consumeDelay * 1000); - if (tmqMsg) { totalMsgs++; - - // printf("threadFlag: %d, totalMsgs: %d\n", threadLable, totalMsgs); - -#if 0 - TAOS_ROW row; - while (NULL != (row = tmq_get_row(tmqMsg))) { - totalRows++; - } -#endif - - /*skipLogNum += tmqGetSkipLogNum(tmqMsg);*/ - if (0 != g_stConfInfo.showMsgFlag) { - /*msg_process(tmqMsg);*/ - } - tmq_message_destroy(tmqMsg); - - if (totalMsgs >= g_stConfInfo.consumeMsgCnt) { + + if (totalMsgs >= pInfo->expectMsgCnt) { break; } } else { break; } } - - err = tmq_consumer_close(tmq); + + err = tmq_consumer_close(pInfo->tmq); if (err) { printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } - // printf("%d", totalMsgs); // output to sim for check result - return totalMsgs; + pInfo->consumeMsgCnt = totalMsgs; + pInfo->consumeRowCnt = totalRows; + + taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %"PRId64", consumeRowCnt: %"PRId64"\n", pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt); + } -void* threadFunc(void* param) { +void *consumeThreadFunc(void *param) { int32_t totalMsgs = 0; - SThreadInfo* pInfo = (SThreadInfo*)param; + SThreadInfo *pInfo = (SThreadInfo *)param; - tmq_t* tmq = build_consumer_x(); - tmq_list_t* topic_list = build_topic_list_x(); - if ((NULL == tmq) || (NULL == topic_list)) { + build_consumer(pInfo); + build_topic_list(pInfo); + if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)){ return NULL; } - - tmq_resp_err_t err = tmq_subscribe(tmq, topic_list); + + tmq_resp_err_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); if (err) { printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } + + loop_consume(pInfo); - // if (0 == g_stConfInfo.consumeMsgCnt) { - // loop_consume(tmq); - // } else { - pInfo->consumeMsgCnt = parallel_consume(tmq, 1); - //} - - err = tmq_unsubscribe(tmq); + err = tmq_unsubscribe(pInfo->tmq); if (err) { printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); - pInfo->consumeMsgCnt = -1; + pInfo->consumeMsgCnt = -1; return NULL; - } + } + + // save consume result into consumeresult table + saveConsumeResult(pInfo); return NULL; } -int main(int32_t argc, char* argv[]) { - parseArgument(argc, argv); - parseInputString(); +void parseConsumeInfo() { + char* token; + const char delim[2] = ","; + const char ch = ':'; - int32_t numOfThreads = 1; - TdThreadAttr thattr; - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - SThreadInfo* pInfo = (SThreadInfo*)taosMemoryCalloc(numOfThreads, sizeof(SThreadInfo)); - - if (g_stConfInfo.numOfTopic1) { - // pthread_create one thread to consume - for (int32_t i = 0; i < numOfThreads; ++i) { - pInfo[i].expectMsgCnt = 0; - pInfo[i].consumeMsgCnt = 0; - taosThreadCreate(&(pInfo[i].thread), &thattr, threadFunc, (void*)(pInfo + i)); + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + token = strtok(g_stConfInfo.stThreads[i].topicString, delim); + while (token != NULL) { + // printf("%s\n", token ); + strcpy(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic], token); + ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]); + // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); + g_stConfInfo.stThreads[i].numOfTopic++; + + token = strtok(NULL, delim); + } + + token = strtok(g_stConfInfo.stThreads[i].keyString, delim); + while (token != NULL) { + // printf("%s\n", token ); + { + char* pstr = token; + ltrim(pstr); + char* ret = strchr(pstr, ch); + memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr); + strcpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1); + // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], + // g_stConfInfo.value[g_stConfInfo.numOfKey]); + g_stConfInfo.stThreads[i].numOfKey++; + } + + token = strtok(NULL, delim); } } +} - int32_t totalMsgs = 0; - tmq_t* tmq = build_consumer(); - tmq_list_t* topic_list = build_topic_list(); - if ((NULL == tmq) || (NULL == topic_list)) { - return -1; - } - - tmq_resp_err_t err = tmq_subscribe(tmq, topic_list); - if (err) { - printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); +int32_t getConsumeInfo() { + char sqlStr[1024] = {0}; + + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName); + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); + taosFprintfFile(g_fp, "error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); + taosCloseFile(&g_fp); + taos_free_result(pRes); exit(-1); + } + + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(pRes); + TAOS_FIELD* fields = taos_fetch_fields(pRes); + + // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int + + int32_t numOfThread = 0; + while ((row = taos_fetch_row(pRes))) { + int32_t* lengths = taos_fetch_lengths(pRes); + + for (int i = 0; i < num_fields; ++i) { + if (row[i] == NULL || 0 == i) { + continue; + } + + if ((1 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { + g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t *)row[i]); + } else if ((2 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { + memcpy(g_stConfInfo.stThreads[numOfThread].topicString, row[i], lengths[i]); + } else if ((3 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { + memcpy(g_stConfInfo.stThreads[numOfThread].keyString, row[i], lengths[i]); + } else if ((4 == i) && (fields[i].type == TSDB_DATA_TYPE_BIGINT)) { + g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t *)row[i]); + } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { + g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t *)row[i]); + } + } + numOfThread ++; } + g_stConfInfo.numOfThread = numOfThread; - if (0 == g_stConfInfo.numOfTopic1) { - loop_consume(tmq); - } else { - totalMsgs = parallel_consume(tmq, 0); - } + taos_free_result(pRes); - err = tmq_unsubscribe(tmq); - if (err) { - printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); - exit(-1); - } + parseConsumeInfo(); - if (g_stConfInfo.numOfTopic1) { - for (int32_t i = 0; i < numOfThreads; i++) { - taosThreadJoin(pInfo[i].thread, NULL); - } + return 0; +} - // printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); - if (0 == g_stConfInfo.checkMode) { - if ((totalMsgs + pInfo->consumeMsgCnt) == g_stConfInfo.consumeMsgCnt) { - printf("success"); - } else { - printf("fail, consumer msg cnt: %d, %d", totalMsgs, pInfo->consumeMsgCnt); - } - } else if (1 == g_stConfInfo.checkMode) { - if ((totalMsgs == g_stConfInfo.consumeMsgCnt) && (pInfo->consumeMsgCnt == g_stConfInfo.consumeMsgCnt)) { - printf("success"); - } else { - printf("fail, consumer msg cnt: %d, %d", totalMsgs, pInfo->consumeMsgCnt); - } - } else if (2 == g_stConfInfo.checkMode) { - if ((totalMsgs + pInfo->consumeMsgCnt) == 3 * g_stConfInfo.consumeMsgCnt) { - printf("success"); - } else { - printf("fail, consumer msg cnt: %d, %d", totalMsgs, pInfo->consumeMsgCnt); - } - } else if (3 == g_stConfInfo.checkMode) { - if ((totalMsgs == 2 * g_stConfInfo.consumeMsgCnt) && (pInfo->consumeMsgCnt == 2 * g_stConfInfo.consumeMsgCnt)) { - printf("success"); - } else { - printf("fail, consumer msg cnt: %d, %d", totalMsgs, pInfo->consumeMsgCnt); - } - } else if (4 == g_stConfInfo.checkMode) { - if (((totalMsgs == 0) && (pInfo->consumeMsgCnt == 3 * g_stConfInfo.consumeMsgCnt)) || - ((pInfo->consumeMsgCnt == 0) && (totalMsgs == 3 * g_stConfInfo.consumeMsgCnt)) || - ((pInfo->consumeMsgCnt == g_stConfInfo.consumeMsgCnt) && (totalMsgs == 2 * g_stConfInfo.consumeMsgCnt)) || - ((pInfo->consumeMsgCnt == 2 * g_stConfInfo.consumeMsgCnt) && (totalMsgs == g_stConfInfo.consumeMsgCnt))) { - printf("success"); - } else { - printf("fail, consumer msg cnt: %d, %d", totalMsgs, pInfo->consumeMsgCnt); - } - } else { - printf("fail, check mode unknow. consumer msg cnt: %d, %d", totalMsgs, pInfo->consumeMsgCnt); - } + +int main(int32_t argc, char* argv[]) { + parseArgument(argc, argv); + getConsumeInfo(); + saveConfigToLogFile(); + + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + + // pthread_create one thread to consume + for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { + taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, (void *)(&(g_stConfInfo.stThreads[i]))); + } + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); } + //printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); + + taosFprintfFile(g_fp, "\n"); + taosCloseFile(&g_fp); + return 0; } diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 8b3edb6035e7cde02ef6ff348fec68f791712d0d..dbc3c2c4602f1424d997a07fb28ac92e1ea1ff0e 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -441,6 +441,12 @@ void simStoreSystemContentResult(SScript *script, char *filename) { // if ((fd = fopen(filename, "r")) != NULL) { if ((pFile = taosOpenFile(filename, TD_FILE_READ)) != NULL) { taosReadFile(pFile, script->system_ret_content, MAX_SYSTEM_RESULT_LEN - 1); + int32_t len = strlen(script->system_ret_content); + for (int32_t i = 0; i < len; ++i) { + if (script->system_ret_content[i] == '\n' || script->system_ret_content[i] == '\r') { + script->system_ret_content[i] = 0; + } + } taosCloseFile(&pFile); char rmCmd[MAX_FILE_NAME_LEN] = {0}; sprintf(rmCmd, "rm -f %s", filename); diff --git a/tools/shell/inc/shell.h b/tools/shell/inc/shell.h index 7a16ee9d2cd8ca1a46471dc7efe59e8a4a4baa90..866cd63bdbfdfe3337d7075b798a56dc043b3fa3 100644 --- a/tools/shell/inc/shell.h +++ b/tools/shell/inc/shell.h @@ -50,6 +50,8 @@ typedef struct SShellArguments { char dir[TSDB_FILENAME_LEN]; int threadNum; int check; + bool status; + bool verbose; char* commands; int abort; int port; @@ -72,7 +74,6 @@ void read_history(); void write_history(); void source_file(TAOS* con, char* fptr); void source_dir(TAOS* con, SShellArguments* args); -void shellCheck(TAOS* con, SShellArguments* args); void get_history_path(char* history); void shellCheck(TAOS* con, SShellArguments* args); void cleanup_handler(void* arg); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 36d2866fb56c3146543886b02a0307eb5874b83e..12206ce29067226e0ce15fd0c9c61e26e86e489d 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -29,7 +29,6 @@ #include "tutil.h" #include -#include /**************** Global variables ****************/ #ifdef _TD_POWER_ diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 0d397eb80b77276dbb988ce15e88736eb58de3cc..db73f2fc5c24140f8392d27b9d2afdad118335f8 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -18,9 +18,11 @@ #include "tglobal.h" #include "tlog.h" +#ifndef WINDOWS #include #include #include +#endif #define OPT_ABORT 1 /* abort */ @@ -35,7 +37,9 @@ static char args_doc[] = ""; TdThread pid; static tsem_t cancelSem; +extern void taos_init(); +#ifndef WINDOWS static struct argp_option options[] = { {"host", 'h', "HOST", 0, "TDengine server FQDN to connect. The default host is localhost."}, {"password", 'p', NULL, 0, "The password to use when connecting to the server."}, @@ -52,6 +56,8 @@ static struct argp_option options[] = { {"check", 'k', "CHECK", 0, "Check tables."}, {"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."}, {"timezone", 'z', "TIMEZONE", 0, "Time zone of the shell, default is local."}, + {"status", 't', NULL, 0, "Check the service status."}, + {"verbose", 'v', NULL, 0, "Check the details of the service status."}, {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speed|fqdn."}, {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, {"pktnum", 'N', "PKTNUM", 0, "Packet numbers used for net test, default is 100."}, @@ -138,6 +144,12 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'k': arguments->check = atoi(arg); break; + case 't': + arguments->status = true; + break; + case 'v': + arguments->verbose = true; + break; case 'd': arguments->database = arg; break; @@ -204,7 +216,10 @@ static void parse_args(int argc, char *argv[], SShellArguments *arguments) { } } +#endif void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { +#ifdef WINDOWS +#else static char verType[32] = {0}; sprintf(verType, "version: %s\n", version); @@ -224,9 +239,12 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { abort(); #endif } +#endif } int32_t shellReadCommand(TAOS *con, char *command) { +#ifdef WINDOWS +#else unsigned hist_counter = history.hend; char utf8_array[10] = "\0"; Command cmd; @@ -381,6 +399,7 @@ int32_t shellReadCommand(TAOS *con, char *command) { } } +#endif return 0; } @@ -426,6 +445,8 @@ void *shellLoopQuery(void *arg) { void get_history_path(char *_history) { snprintf(_history, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), HISTORY_FILE); } void clearScreen(int ecmd_pos, int cursor_pos) { +#ifdef WINDOWS +#else struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { // fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); @@ -444,9 +465,12 @@ void clearScreen(int ecmd_pos, int cursor_pos) { fprintf(stdout, "\033[2K"); } fflush(stdout); +#endif } void showOnScreen(Command *cmd) { +#ifdef WINDOWS +#else struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { // fprintf(stderr, "No stream device\n"); @@ -517,15 +541,16 @@ void showOnScreen(Command *cmd) { positionCursor(cursor_x, DOWN); positionCursor(cursor_y, RIGHT); fflush(stdout); +#endif } void cleanup_handler(void *arg) { resetTerminalMode(); } void exitShell() { - /*int32_t ret =*/resetTerminalMode(); taos_cleanup(); exit(EXIT_SUCCESS); } + void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { tsem_post(&cancelSem); } void *cancelHandler(void *arg) { @@ -537,21 +562,9 @@ void *cancelHandler(void *arg) { continue; } -#ifdef LINUX -#if 0 - int64_t rid = atomic_val_compare_exchange_64(&result, result, 0); - SSqlObj* pSql = taosAcquireRef(tscObjRef, rid); - taos_stop_query(pSql); - taosReleaseRef(tscObjRef, rid); -#endif -#else - resetTerminalMode(); - printf("\nReceive ctrl+c or other signal, quit shell.\n"); - exit(0); -#endif resetTerminalMode(); printf("\nReceive ctrl+c or other signal, quit shell.\n"); - exit(0); + exitShell(); } return NULL; @@ -578,93 +591,120 @@ int checkVersion() { } // Global configurations -SShellArguments args = {.host = NULL, +SShellArguments args = { + .host = NULL, + .user = NULL, + .database = NULL, + .timezone = NULL, + .is_raw_time = false, + .is_use_passwd = false, + .dump_config = false, + .file = "\0", + .dir = "\0", + .threadNum = 5, + .commands = NULL, + .pktLen = 1000, + .pktNum = 100, + .pktType = "TCP", + .netTestRole = NULL, #ifndef TD_WINDOWS - .password = NULL, + .password = NULL, #endif - .user = NULL, - .database = NULL, - .timezone = NULL, - .is_raw_time = false, - .is_use_passwd = false, - .dump_config = false, - .file = "\0", - .dir = "\0", - .threadNum = 5, - .commands = NULL, - .pktLen = 1000, - .pktNum = 100, - .pktType = "TCP", - .netTestRole = NULL}; +}; -/* - * Main function. - */ -int main(int argc, char *argv[]) { - /*setlocale(LC_ALL, "en_US.UTF-8"); */ +void shellDumpConfig() { + if (!args.dump_config) return; - if (!checkVersion()) { + SConfig *pCfg = taosGetCfg(); + if (NULL == pCfg) { + printf("TDengine read global config failed!\n"); exit(EXIT_FAILURE); } + cfgDumpCfg(pCfg, 0, 1); + exitShell(); +} - shellParseArgument(argc, argv, &args); +void shellTestNetWork() { + if (args.netTestRole && args.netTestRole[0] != 0) { + taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktNum, args.pktType); + exitShell(); + } +} - if (args.dump_config) { - taosInitCfg(configDir, NULL, NULL, NULL, 1); +void shellCheckServerStatus() { + if (!args.status && !args.verbose) return; - SConfig *pCfg = taosGetCfg(); - if (NULL == pCfg) { - printf("TDengine read global config failed!\n"); - exit(EXIT_FAILURE); + TSDB_SERVER_STATUS code; + do { + char details[1024] = {0}; + code = taos_check_server_status(args.host, args.port, details, args.verbose ? 1024 : 0); + switch (code) { + case TSDB_SRV_STATUS_UNAVAILABLE: + printf("0: unavailable\n"); + break; + case TSDB_SRV_STATUS_NETWORK_OK: + printf("1: network ok\n"); + break; + case TSDB_SRV_STATUS_SERVICE_OK: + printf("2: service ok\n"); + break; + case TSDB_SRV_STATUS_SERVICE_DEGRADED: + printf("3: service degraded\n"); + break; + case TSDB_SRV_STATUS_EXTING: + printf("4: exiting\n"); + break; } - cfgDumpCfg(pCfg, 0, 1); - exit(0); - } - - if (args.netTestRole && args.netTestRole[0] != 0) { - TAOS *con = NULL; - if (args.auth == NULL) { - con = taos_connect(args.host, args.user, args.password, args.database, args.port); + if (strlen(details) != 0) { + printf("%s\n\n", details); + } + if (code == TSDB_SRV_STATUS_NETWORK_OK && args.verbose) { + taosMsleep(1000); } else { - con = taos_connect_auth(args.host, args.user, args.auth, args.database, args.port); + break; } + } while (1); - // if (taos_init()) { - // printf("Failed to init taos"); - // exit(EXIT_FAILURE); - // } - - taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktNum, args.pktType); - taos_close(con); - exit(0); - } + exitShell(); +} - /* Initialize the shell */ +void shellExecute() { TAOS *con = shellInit(&args); if (con == NULL) { - exit(EXIT_FAILURE); + exitShell(); } if (tsem_init(&cancelSem, 0, 0) != 0) { printf("failed to create cancel semphore\n"); - exit(EXIT_FAILURE); + exitShell(); } TdThread spid; taosThreadCreate(&spid, NULL, cancelHandler, NULL); - /* Interrupt handler. */ taosSetSignal(SIGTERM, shellQueryInterruptHandler); taosSetSignal(SIGINT, shellQueryInterruptHandler); taosSetSignal(SIGHUP, shellQueryInterruptHandler); taosSetSignal(SIGABRT, shellQueryInterruptHandler); - /* Get grant information */ shellGetGrantInfo(con); - /* Loop to query the input. */ while (1) { taosThreadCreate(&pid, NULL, shellLoopQuery, con); taosThreadJoin(pid, NULL); } } + +int main(int argc, char *argv[]) { + if (!checkVersion()) exitShell(); + + shellParseArgument(argc, argv, &args); + + taos_init(); + shellDumpConfig(); + shellCheckServerStatus(); + shellTestNetWork(); + shellExecute(); + + return 0; +} diff --git a/tools/shell/src/tnettest.c b/tools/shell/src/tnettest.c index 9be341225664694d0eb05bb7d76fcfcd1cbe0c30..04b99ebc75e431bf9599965a1fb71b2bdc211d96 100644 --- a/tools/shell/src/tnettest.c +++ b/tools/shell/src/tnettest.c @@ -356,187 +356,6 @@ static void taosNetCheckPort(uint32_t hostIp, int32_t startPort, int32_t endPort } } -void *taosNetInitRpc(char *secretEncrypt, char spi) { - SRpcInit rpcInit; - void * pRpcConn = NULL; - - char user[] = "nettestinternal"; - char pass[] = "nettestinternal"; - taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt); - - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "NT"; - rpcInit.numOfThreads = 1; // every DB connection has only one thread - rpcInit.cfp = NULL; - rpcInit.sessions = 16; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.user = user; - rpcInit.idleTime = 2000; - rpcInit.ckey = "key"; - rpcInit.spi = spi; - rpcInit.secret = secretEncrypt; - - pRpcConn = rpcOpen(&rpcInit); - return pRpcConn; -} - -static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi, SStartupReq *pStep) { - SEpSet epSet; - SRpcMsg reqMsg; - SRpcMsg rspMsg; - void * pRpcConn; - - char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; - - pRpcConn = taosNetInitRpc(secretEncrypt, spi); - if (NULL == pRpcConn) { - uError("failed to init client rpc"); - return TSDB_CODE_RPC_NETWORK_UNAVAIL; - } - - memset(&epSet, 0, sizeof(SEpSet)); - strcpy(epSet.eps[0].fqdn, serverFqdn); - epSet.eps[0].port = port; - epSet.numOfEps = 1; - - reqMsg.msgType = TDMT_DND_NETWORK_TEST; - reqMsg.pCont = rpcMallocCont(pktLen); - reqMsg.contLen = pktLen; - reqMsg.code = 0; - reqMsg.handle = NULL; // rpc handle returned to app - reqMsg.ahandle = NULL; // app handle set by client - strcpy(reqMsg.pCont, "dnode-nettest"); - - rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg); - - if ((rspMsg.code != 0) || (rspMsg.msgType != TDMT_DND_NETWORK_TEST + 1)) { - uDebug("ret code 0x%x %s", rspMsg.code, tstrerror(rspMsg.code)); - return rspMsg.code; - } - - int32_t code = 0; - if (pStep != NULL && rspMsg.pCont != NULL && rspMsg.contLen > 0 && rspMsg.contLen <= sizeof(SStartupReq)) { - memcpy(pStep, rspMsg.pCont, rspMsg.contLen); - code = 1; - } - - rpcFreeCont(rspMsg.pCont); - rpcClose(pRpcConn); - return code; -} - -static int32_t taosNetParseStartup(SStartupReq *pCont) { - SStartupReq *pStep = pCont; - uInfo("step:%s desc:%s", pStep->name, pStep->desc); - - if (pStep->finished) { - uInfo("check startup finished"); - } - - return pStep->finished ? 0 : 1; -} - -static void taosNetTestStartup(char *host, int32_t port) { - uInfo("check startup, host:%s port:%d\n", host, port); - - SStartupReq *pStep = taosMemoryMalloc(sizeof(SStartupReq)); - while (1) { - int32_t code = taosNetCheckRpc(host, port, 20, 0, pStep); - if (code > 0) { - code = taosNetParseStartup(pStep); - } - - if (code > 0) { - uDebug("continue check startup step"); - } else { - if (code < 0) { - uError("failed to check startup step, code:0x%x %s", code, tstrerror(code)); - } - break; - } - } - - taosMemoryFree(pStep); -} - -static void taosNetCheckSync(char *host, int32_t port) { - uint32_t ip = taosGetIpv4FromFqdn(host); - if (ip == 0xffffffff) { - uError("failed to get IP address from %s since %s", host, strerror(errno)); - return; - } - - TdSocketPtr pSocket = taosOpenTcpClientSocket(ip, (uint16_t)port, 0); - if (pSocket == NULL) { - uError("failed to create socket while test port:%d since %s", port, strerror(errno)); - return; - } - - SSyncMsg msg; - memset(&msg, 0, sizeof(SSyncMsg)); - SSyncHead *pHead = &msg.head; - pHead->type = TAOS_SMSG_TEST; - pHead->protocol = SYNC_PROTOCOL_VERSION; - pHead->signature = SYNC_SIGNATURE; - pHead->code = 0; - pHead->cId = 0; - pHead->vgId = -1; - pHead->len = sizeof(SSyncMsg) - sizeof(SSyncHead); - taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SSyncHead)); - - if (taosWriteMsg(pSocket, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { - uError("failed to test port:%d while send msg since %s", port, strerror(errno)); - return; - } - - if (taosReadMsg(pSocket, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { - uError("failed to test port:%d while recv msg since %s", port, strerror(errno)); - } - - uInfo("successed to test TCP port:%d", port); - taosCloseSocket(&pSocket); -} - -static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { - char spi = 0; - - uInfo("check rpc, host:%s Port:%d pkgLen:%d\n", host, startPort, pkgLen); - - uint16_t port = startPort; - int32_t sendpkgLen; - if (pkgLen <= tsRpcMaxUdpSize) { - sendpkgLen = tsRpcMaxUdpSize + 1000; - } else { - sendpkgLen = pkgLen; - } - - tsRpcForceTcp = 1; - int32_t ret = taosNetCheckRpc(host, port, sendpkgLen, spi, NULL); - if (ret < 0) { - printf("failed to test TCP port:%d\n", port); - } else { - printf("successed to test TCP port:%d\n", port); - } - - if (pkgLen >= tsRpcMaxUdpSize) { - sendpkgLen = tsRpcMaxUdpSize - 1000; - } else { - sendpkgLen = pkgLen; - } -/* - tsRpcForceTcp = 0; - ret = taosNetCheckRpc(host, port, pkgLen, spi, NULL); - if (ret < 0) { - printf("failed to test UDP port:%d\n", port); - } else { - printf("successed to test UDP port:%d\n", port); - } - */ - - taosNetCheckSync(host, startPort); -} - static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) { uInfo("work as client, host:%s Port:%d pkgLen:%d\n", host, startPort, pkgLen); @@ -586,22 +405,10 @@ static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { } } -static void taosNetTestFqdn(char *host) { - int code = 0; - uint64_t startTime = taosGetTimestampUs(); - uint32_t ip = taosGetIpv4FromFqdn(host); - if (ip == 0xffffffff) { - uError("failed to get IP address from %s since %s", host, strerror(errno)); - code = -1; - } - uint64_t endTime = taosGetTimestampUs(); - uint64_t el = endTime - startTime; - printf("check convert fqdn spend, status: %d\tcost: %" PRIu64 " us\n", code, el); - return; -} - static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType) { +#if 0 + // record config int32_t compressTmp = tsCompressMsgSize; int32_t maxUdpSize = tsRpcMaxUdpSize; @@ -674,19 +481,19 @@ static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, tsRpcMaxUdpSize = maxUdpSize; tsRpcForceTcp = forceTcp; return; +#endif } -void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, - int32_t pkgNum, char *pkgType) { +void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType) { tsLogEmbedded = 1; if (host == NULL) host = tsLocalFqdn; if (port == 0) port = tsServerPort; - if (0 == strcmp("speed", role)){ + if (0 == strcmp("speed", role)) { if (pkgLen <= MIN_SPEED_PKG_LEN) pkgLen = MIN_SPEED_PKG_LEN; if (pkgLen > MAX_SPEED_PKG_LEN) pkgLen = MAX_SPEED_PKG_LEN; if (pkgNum <= MIN_SPEED_PKG_NUM) pkgNum = MIN_SPEED_PKG_NUM; if (pkgNum > MAX_SPEED_PKG_NUM) pkgNum = MAX_SPEED_PKG_NUM; - }else{ + } else { if (pkgLen <= 10) pkgLen = 1000; if (pkgLen > MAX_PKG_LEN) pkgLen = MAX_PKG_LEN; } @@ -695,21 +502,12 @@ void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, taosNetTestClient(host, port, pkgLen); } else if (0 == strcmp("server", role)) { taosNetTestServer(host, port, pkgLen); - } else if (0 == strcmp("rpc", role)) { - tsLogEmbedded = 0; - taosNetTestRpc(host, port, pkgLen); - } else if (0 == strcmp("sync", role)) { - taosNetCheckSync(host, port); - } else if (0 == strcmp("startup", role)) { - taosNetTestStartup(host, port); } else if (0 == strcmp("speed", role)) { tsLogEmbedded = 0; char type[10] = {0}; taosNetCheckSpeed(host, port, pkgLen, pkgNum, strtolower(type, pkgType)); - }else if (0 == strcmp("fqdn", role)) { - taosNetTestFqdn(host); } else { - taosNetTestStartup(host, port); + TASSERT(1); } tsLogEmbedded = 0;