diff --git a/cmake/cmake.define b/cmake/cmake.define index 0eb5206aab2aaf93222c2653a314a5a0bb42eb28..4e27ff5f479ab89cd683d4c3e581c238487ef926 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -14,25 +14,6 @@ MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR}) MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH}) MESSAGE(STATUS "Project library files output path: " ${LIBRARY_OUTPUT_PATH}) -find_package(Git QUIET) -if(GIT_FOUND AND EXISTS "${TD_SOURCE_DIR}/.git") -# Update submodules as needed - option(GIT_SUBMODULE "Check submodules during build" ON) - if(GIT_SUBMODULE) - message(STATUS "Submodule update") - execute_process(COMMAND cd ${TD_SOURCE_DIR} && ${GIT_EXECUTABLE} submodule update --init --recursive - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - RESULT_VARIABLE GIT_SUBMOD_RESULT) - if(NOT GIT_SUBMOD_RESULT EQUAL "0") - message(WARNING "git submodule update --init --recursive failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") - endif() - endif() -endif() - -if(NOT EXISTS "${TD_SOURCE_DIR}/tools/taos-tools/CMakeLists.txt") - message(WARNING "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules manually if you need build them.") -endif() - if (NOT DEFINED TD_GRANT) SET(TD_GRANT FALSE) endif() diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3e03a51852c7ccb517bc6102e08364b6ee0232e1..9b09230d2544a93b3e3b11f28afa99708673d7a8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -70,7 +70,7 @@ typedef uint16_t tmsg_t; #define TSDB_IE_TYPE_DNODE_EXT 6 #define TSDB_IE_TYPE_DNODE_STATE 7 -enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__MAX }; +enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__UDFD, CONN_TYPE__MAX }; enum { HEARTBEAT_KEY_USER_AUTHINFO = 1, diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 32bda084ed5ab1ddac400354264f18545d88e001..cc10348d24fea98aa852bab008886e36a92fb258 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -642,6 +642,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_INTERNAL_PK TAOS_DEF_ERROR_CODE(0, 0x2646) #define TSDB_CODE_PAR_INVALID_TIMELINE_FUNC TAOS_DEF_ERROR_CODE(0, 0x2647) #define TSDB_CODE_PAR_INVALID_PASSWD TAOS_DEF_ERROR_CODE(0, 0x2648) +#define TSDB_CODE_PAR_INVALID_ALTER_TABLE TAOS_DEF_ERROR_CODE(0, 0x2649) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8ed4308ea76fd9fbee7444eee4df0f33860ffa34..8fce5bfb00668cdf535d8dacf2e0e12b9e43849a 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1007,8 +1007,6 @@ static int32_t smlParseTelnetTags(const char* data, SArray *cols, SHashObj *dump while(*sql != '\0') { // parse value if (*sql == SPACE) { - valueLen = sql - value; - sql++; break; } if (*sql == EQUAL) { @@ -1017,9 +1015,7 @@ static int32_t smlParseTelnetTags(const char* data, SArray *cols, SHashObj *dump } sql++; } - if(valueLen == 0){ - valueLen = sql - value; - } + valueLen = sql - value; if(valueLen == 0){ smlBuildInvalidDataMsg(msg, "invalid value", value); @@ -1365,7 +1361,7 @@ static void smlDestroySTableMeta(SSmlSTableMeta *meta){ static void smlDestroyCols(SArray *cols) { if (!cols) return; for (int i = 0; i < taosArrayGetSize(cols); ++i) { - void *kv = taosArrayGet(cols, i); + void *kv = taosArrayGetP(cols, i); taosMemoryFree(kv); } } @@ -2077,12 +2073,16 @@ static int32_t smlParseTelnetLine(SSmlHandle* info, void *data) { if(ret != TSDB_CODE_SUCCESS){ uError("SML:0x%"PRIx64" smlParseTelnetLine failed", info->id); smlDestroyTableInfo(info, tinfo); + smlDestroyCols(cols); taosArrayDestroy(cols); return ret; } if(taosArrayGetSize(tinfo->tags) <= 0 || taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS){ smlBuildInvalidDataMsg(&info->msgBuf, "invalidate tags length:[1,128]", NULL); + smlDestroyTableInfo(info, tinfo); + smlDestroyCols(cols); + taosArrayDestroy(cols); return TSDB_CODE_SML_INVALID_DATA; } taosHashClear(info->dumplicateKey); diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 589c3d3a10df24ab064fabc5ed295e2ed854e614..e567a0c3e873e84485435e8c0d3d9050c3065250 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -516,8 +516,8 @@ TEST(testCase, smlProcess_influx_Test) { int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); ASSERT_EQ(ret, 0); - TAOS_RES *res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d"); - ASSERT_NE(res, nullptr); +// TAOS_RES *res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d"); +// ASSERT_NE(res, nullptr); // int fieldNum = taos_field_count(res); // ASSERT_EQ(fieldNum, 5); // int rowNum = taos_affected_rows(res); @@ -525,7 +525,7 @@ TEST(testCase, smlProcess_influx_Test) { // for (int i = 0; i < rowNum; ++i) { // TAOS_ROW rows = taos_fetch_row(res); // } - taos_free_result(res); +// taos_free_result(res); destroyRequest(request); smlDestroyInfo(info); } @@ -605,8 +605,8 @@ TEST(testCase, smlProcess_telnet_Test) { int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); ASSERT_EQ(ret, 0); - TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a"); - ASSERT_NE(res, nullptr); +// TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a"); +// ASSERT_NE(res, nullptr); // int fieldNum = taos_field_count(res); // ASSERT_EQ(fieldNum, 2); // int rowNum = taos_affected_rows(res); @@ -614,7 +614,7 @@ TEST(testCase, smlProcess_telnet_Test) { // for (int i = 0; i < rowNum; ++i) { // TAOS_ROW rows = taos_fetch_row(res); // } - taos_free_result(res); +// taos_free_result(res); // res = taos_query(taos, "select * from t_6931529054e5637ca92c78a1ad441961"); // ASSERT_NE(res, nullptr); @@ -670,16 +670,16 @@ TEST(testCase, smlProcess_json1_Test) { int ret = smlProcess(info, (char **)(&sql), -1); ASSERT_EQ(ret, 0); - TAOS_RES *res = taos_query(taos, "select * from t_cb27a7198d637b4f1c6464bd73f756a7"); - ASSERT_NE(res, nullptr); - int fieldNum = taos_field_count(res); - ASSERT_EQ(fieldNum, 2); +// TAOS_RES *res = taos_query(taos, "select * from t_cb27a7198d637b4f1c6464bd73f756a7"); +// ASSERT_NE(res, nullptr); +// int fieldNum = taos_field_count(res); +// ASSERT_EQ(fieldNum, 2); // int rowNum = taos_affected_rows(res); // ASSERT_EQ(rowNum, 1); // for (int i = 0; i < rowNum; ++i) { // TAOS_ROW rows = taos_fetch_row(res); // } - taos_free_result(res); +// taos_free_result(res); destroyRequest(request); smlDestroyInfo(info); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 52edd79f3bacefe10cd02e0b139d87a95fabc827..d180c2ff6e17a9cce4a2da730f256f629f2c0986 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4217,7 +4217,6 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) { if (tDecodeCStr(pDecoder, &pReq->tbName) < 0) return -1; if (tDecodeI8(pDecoder, &pReq->action) < 0) return -1; - if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1; switch (pReq->action) { case TSDB_ALTER_TABLE_ADD_COLUMN: if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index d8309850b3c3f71b59960b644c2db758d20d4f68..edc00909731d6474a415f0c50c7c4efde277bcdd 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -214,7 +214,23 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf // todo SRpcMsg *pRsp = NULL; - (void)vnodeProcessSyncReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); + int32_t ret = vnodeProcessSyncReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); + if (ret != 0) { + // if leader, send response + if (pMsg->rpcMsg.handle != NULL) { + SRpcMsg rsp; + rsp.pCont = NULL; + rsp.contLen = 0; + + rsp.code = terrno; + dTrace("vmProcessSyncQueue error, code:%d", terrno); + + rsp.ahandle = pMsg->rpcMsg.ahandle; + rsp.handle = pMsg->rpcMsg.handle; + rsp.refId = pMsg->rpcMsg.refId; + tmsgSendRsp(&rsp); + } + } rpcFreeCont(pMsg->rpcMsg.pCont); taosFreeQitem(pMsg); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index ac49a0b0786fe9e5ffc37d501df8b1de5e528bea..43d68fc9fdf90f58fa895909e50e385491d90ff7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -198,6 +198,8 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { } int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + int32_t ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; + if (syncEnvIsStart()) { SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); assert(pSyncNode != NULL); @@ -219,67 +221,70 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); + ret = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); syncTimeoutDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) { SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnPingCb(pSyncNode, pSyncMsg); + ret = syncNodeOnPingCb(pSyncNode, pSyncMsg); syncPingDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) { SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); + ret = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); syncPingReplyDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); syncClientRequestDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) { SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); + ret = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); syncRequestVoteDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) { SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); + ret = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); syncRequestVoteReplyDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) { SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); + ret = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); syncAppendEntriesDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) { SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); + ret = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); } else { vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType); + ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; } syncNodeRelease(pSyncNode); } else { vError("==vnodeProcessSyncReq== error syncEnv stop"); + ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; } - return 0; + + return ret; } static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 64090d02831205b155cd477cf91d5d2ce2442956..c96ad140a18aa7a7a8275410fed866be16664aa4 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2576,12 +2576,6 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - SHashObj *metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == metaCache) { - qError("taosHashInit failed, num:%d", gCtgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - code = taosHashPut(gCtgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); if (code) { if (HASH_NODE_EXIST(code)) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index a03264549306df55e63ab87dcd7fe5b294cfbffc..6100d416b11d22df45cca1119c1ebfad50891095 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -658,6 +658,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWin void cleanupAggSup(SAggSupporter* pAggSup); void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); +void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId); SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode); SColumn extractColumnFromColumnNode(SColumnNode* pColNode); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0ab172f03a3f0cfd07eedcd37e5336e43701057d..1cfabf2975c2805883ed9af802109cb9d5ed4442 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2055,6 +2055,11 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); + // it is a reserved column for scalar function, and no data in this column yet. + if (pSrc->pData == NULL) { + continue; + } + int32_t numOfRows = 0; for (int32_t j = 0; j < totalRows; ++j) { if (rowRes[j] == 0) { @@ -4372,9 +4377,9 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p goto _error; } - pInfo->limit = *pLimit; - pInfo->slimit = *pSlimit; - pInfo->curOffset = pLimit->offset; + pInfo->limit = *pLimit; + pInfo->slimit = *pSlimit; + pInfo->curOffset = pLimit->offset; pInfo->curSOffset = pSlimit->offset; pInfo->binfo.pRes = pResBlock; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 65bd8f4bda6384ccd92171c586f36a5b4ba1fda0..08539206a6271e54b4c3adaeda6a86dd6fd73ae5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -291,20 +291,7 @@ void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) // this is to handle the tbname if (fmIsScanPseudoColumnFunc(functionId)) { - struct SScalarFuncExecFuncs fpSet = {0}; - fmGetScalarFuncExecFuncs(functionId, &fpSet); - - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_BIGINT; - infoData.info.bytes = sizeof(uint64_t); - colInfoDataEnsureCapacity(&infoData, 0, 1); - - colDataAppendInt64(&infoData, 0, &pBlock->info.uid); - SScalarParam srcParam = { - .numOfRows = pBlock->info.rows, .param = pTableScanInfo->readHandle.meta, .columnData = &infoData}; - - SScalarParam param = {.columnData = pColInfoData}; - fpSet.process(&srcParam, 1, ¶m); + setTbNameColData(pTableScanInfo->readHandle.meta, pBlock, pColInfoData, functionId); } else { // these are tags const char* p = metaGetTableTagVal(&mr.me, pExpr->base.pParam[0].pCol->colId); for (int32_t i = 0; i < pBlock->info.rows; ++i) { @@ -316,6 +303,23 @@ void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) metaReaderClear(&mr); } +void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId) { + struct SScalarFuncExecFuncs fpSet = {0}; + fmGetScalarFuncExecFuncs(functionId, &fpSet); + + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_BIGINT; + infoData.info.bytes = sizeof(uint64_t); + colInfoDataEnsureCapacity(&infoData, 0, 1); + + colDataAppendInt64(&infoData, 0, (int64_t*) &pBlock->info.uid); + SScalarParam srcParam = { + .numOfRows = pBlock->info.rows, .param = pMeta, .columnData = &infoData}; + + SScalarParam param = {.columnData = pColInfoData}; + fpSet.process(&srcParam, 1, ¶m); +} + static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SSDataBlock* pBlock = pTableScanInfo->pResBlock; diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 807234a1b15a7a5ed6040112e12eec6e7a6503bc..c25d74911c916ed26c1a04241909975ab8ab793a 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -105,7 +105,12 @@ int32_t mavgFunction(SqlFunctionCtx* pCtx); bool getSampleFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool sampleFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t sampleFunction(SqlFunctionCtx* pCtx); -int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +//int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + +bool getTailFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool tailFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +int32_t tailFunction(SqlFunctionCtx* pCtx); +int32_t tailFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index fc930083128c8bfe79f69e21f3c6737547ea67f1..e41e3c7c39a2a18e262e892cf659fd3258361ace 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -386,6 +386,35 @@ static int32_t translateSample(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return TSDB_CODE_SUCCESS; } +static int32_t translateTail(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (2 != numOfParams && 3 != numOfParams) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + if (QUERY_NODE_COLUMN != nodeType(pPara)) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "The input parameter of TAIL function can only be column"); + } + + for (int32_t i = 1; i < numOfParams; ++i) { + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type; + if (!IS_INTEGER_TYPE(paraType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + } + + SExprNode* pCol = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); + uint8_t colType = pCol->resType.type; + if (IS_VAR_DATA_TYPE(colType)) { + pFunc->node.resType = (SDataType){.bytes = pCol->resType.bytes, .type = colType}; + } else { + pFunc->node.resType = (SDataType){.bytes = tDataTypes[colType].bytes, .type = colType}; + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateLastRow(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // todo return TSDB_CODE_SUCCESS; @@ -850,6 +879,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = sampleFunction, .finalizeFunc = NULL }, + { + .name = "tail", + .type = FUNCTION_TYPE_TAIL, + .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateTail, + .getEnvFunc = getTailFuncEnv, + .initFunc = tailFunctionSetup, + .processFunc = tailFunction, + .finalizeFunc = tailFinalize + }, { .name = "abs", .type = FUNCTION_TYPE_ABS, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index af86eb4e90d2fab886b2c9c7f49d5df64d6176ca..479a11ca4a2d38321c93c1bba7941f1874a76073 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -18,12 +18,15 @@ #include "function.h" #include "querynodes.h" #include "taggfunction.h" +#include "tcompare.h" #include "tdatablock.h" #include "tpercentile.h" #define HISTOGRAM_MAX_BINS_NUM 1000 #define MAVG_MAX_POINTS_NUM 1000 #define SAMPLE_MAX_POINTS_NUM 1000 +#define TAIL_MAX_POINTS_NUM 100 +#define TAIL_MAX_OFFSET 100 typedef struct SSumRes { union { @@ -161,6 +164,21 @@ typedef struct SSampleInfo { int64_t *timestamp; } SSampleInfo; +typedef struct STailItem { + int64_t timestamp; + bool isNull; + char data[]; +} STailItem; + +typedef struct STailInfo { + int32_t numOfPoints; + int32_t numAdded; + int32_t offset; + uint8_t colType; + int16_t colBytes; + STailItem **pItems; +} STailInfo; + #define SET_VAL(_info, numOfElem, res) \ do { \ if ((numOfElem) <= 0) { \ @@ -3107,7 +3125,7 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { int32_t startOffset = pCtx->offset; for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { - if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + if (colDataIsNull_s(pInputCol, i)) { //colDataAppendNULL(pOutput, i); continue; } @@ -3141,3 +3159,132 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { // // return pResInfo->numOfRes; //} + + +bool getTailFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0); + SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); + int32_t numOfPoints = pVal->datum.i; + pEnv->calcMemSize = sizeof(STailInfo) + numOfPoints * (POINTER_BYTES + sizeof(STailItem) + pCol->node.resType.bytes); + return true; +} + +bool tailFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo *pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + STailInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo); + pInfo->numAdded = 0; + pInfo->numOfPoints = pCtx->param[1].param.i; + if (pCtx->numOfParams == 4) { + pInfo->offset = pCtx->param[2].param.i; + } else { + pInfo->offset = 0; + } + pInfo->colType = pCtx->resDataInfo.type; + pInfo->colBytes = pCtx->resDataInfo.bytes; + if ((pInfo->numOfPoints < 1 || pInfo->numOfPoints > TAIL_MAX_POINTS_NUM) || + (pInfo->numOfPoints < 0 || pInfo->numOfPoints > TAIL_MAX_OFFSET)) { + return false; + } + + pInfo->pItems = (STailItem **)((char *)pInfo + sizeof(STailInfo)); + char *pItem = (char *)pInfo->pItems + pInfo->numOfPoints * POINTER_BYTES; + + size_t unitSize = sizeof(STailItem) + pInfo->colBytes; + for (int32_t i = 0; i < pInfo->numOfPoints; ++i) { + pInfo->pItems[i] = (STailItem *)(pItem + i * unitSize); + pInfo->pItems[i]->isNull = false; + } + + return true; +} + +static void tailAssignResult(STailItem* pItem, char *data, int32_t colBytes, TSKEY ts, bool isNull) { + pItem->timestamp = ts; + if (isNull) { + pItem->isNull = true; + } else { + memcpy(pItem->data, data, colBytes); + } +} + +static int32_t tailCompFn(const void *p1, const void *p2, const void *param) { + STailItem *d1 = *(STailItem **)p1; + STailItem *d2 = *(STailItem **)p2; + return compareInt64Val(&d1->timestamp, &d2->timestamp); +} + +static void doTailAdd(STailInfo* pInfo, char *data, TSKEY ts, bool isNull) { + STailItem **pList = pInfo->pItems; + if (pInfo->numAdded < pInfo->numOfPoints) { + tailAssignResult(pList[pInfo->numAdded], data, pInfo->colBytes, ts, isNull); + taosheapsort((void *)pList, sizeof(STailItem **), pInfo->numAdded + 1, NULL, tailCompFn, 0); + pInfo->numAdded++; + } else if (pList[0]->timestamp < ts) { + tailAssignResult(pList[0], data, pInfo->colBytes, ts, isNull); + taosheapadjust((void *)pList, sizeof(STailItem **), 0, pInfo->numOfPoints - 1, NULL, tailCompFn, NULL, 0); + } +} + +int32_t tailFunction(SqlFunctionCtx* pCtx) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + STailInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + TSKEY* tsList = (int64_t*)pInput->pPTS->pData; + + SColumnInfoData* pInputCol = pInput->pData[0]; + SColumnInfoData* pTsOutput = pCtx->pTsOutput; + SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; + + int32_t startOffset = pCtx->offset; + if (pInfo->offset >= pInput->numOfRows) { + return 0; + } else { + pInfo->numOfPoints = MIN(pInfo->numOfPoints, pInput->numOfRows - pInfo->offset); + } + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex - pInfo->offset; i += 1) { + + char* data = colDataGetData(pInputCol, i); + doTailAdd(pInfo, data, tsList[i], colDataIsNull_s(pInputCol, i)); + } + + taosqsort(pInfo->pItems, pInfo->numOfPoints, POINTER_BYTES, NULL, tailCompFn); + + for (int32_t i = 0; i < pInfo->numOfPoints; ++i) { + int32_t pos = startOffset + i; + STailItem *pItem = pInfo->pItems[i]; + if (pItem->isNull) { + colDataAppendNULL(pOutput, pos); + } else { + colDataAppend(pOutput, pos, pItem->data, false); + } + } + + return pInfo->numOfPoints; +} + +int32_t tailFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); + STailInfo* pInfo = GET_ROWCELL_INTERBUF(pEntryInfo); + pEntryInfo->complete = true; + + int32_t type = pCtx->input.pData[0]->info.type; + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + // todo assign the tag value and the corresponding row data + int32_t currentRow = pBlock->info.rows; + for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { + STailItem *pItem = pInfo->pItems[i]; + colDataAppend(pCol, currentRow, pItem->data, false); + + //setSelectivityValue(pCtx, pBlock, &pInfo->pItems[i].tuplePos, currentRow); + currentRow += 1; + } + + return pEntryInfo->numOfRes; +} diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 4841e052679ff697bcdf806632ac560aeb5134f8..388ec28b763c123518bb05cec29f6b8312ed9aa5 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -312,6 +312,7 @@ typedef struct SUdfcFuncStub { char udfName[TSDB_FUNC_NAME_LEN]; UdfcFuncHandle handle; int32_t refCount; + int64_t lastRefTime; } SUdfcFuncStub; typedef struct SUdfcProxy { @@ -1446,6 +1447,7 @@ int32_t accquireUdfFuncHandle(char* udfName, UdfcFuncHandle* pHandle) { uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); *pHandle = foundStub->handle; ++foundStub->refCount; + foundStub->lastRefTime = taosGetTimestampUs(); return 0; } *pHandle = NULL; @@ -1455,6 +1457,7 @@ int32_t accquireUdfFuncHandle(char* udfName, UdfcFuncHandle* pHandle) { strcpy(stub.udfName, udfName); stub.handle = *pHandle; ++stub.refCount; + stub.lastRefTime = taosGetTimestampUs(); taosArrayPush(gUdfdProxy.udfStubs, &stub); taosArraySort(gUdfdProxy.udfStubs, compareUdfcFuncSub); } else { @@ -1662,7 +1665,8 @@ int32_t cleanUpUdfs() { fnInfo("tear down udf. udf name: %s, handle: %p", stub->udfName, stub->handle); doTeardownUdf(stub->handle); } else { - fnInfo("udf still in use. udf name: %s, ref count: %d, handle: %p", stub->udfName, stub->refCount, stub->handle); + fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %"PRId64", handle: %p", + stub->udfName, stub->refCount, stub->lastRefTime, stub->handle); taosArrayPush(udfStubs, stub); } ++i; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 34681dc6cd18c6e85523a61084b5ec931bf326af..706bf28be0a7c8f514b68df35bc5ef36bdfd7afd 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -485,7 +485,146 @@ void udfdIntrSignalHandler(uv_signal_t *handle, int signum) { uv_stop(global.loop); } -void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; } +typedef enum EUdfdRpcReqRspType { + UDFD_RPC_MNODE_CONNECT = 0, + UDFD_RPC_RETRIVE_FUNC, +} EUdfdRpcReqRspType; + +typedef struct SUdfdRpcSendRecvInfo { + EUdfdRpcReqRspType rpcType; + int32_t code; + void* param; + uv_sem_t resultSem; +} SUdfdRpcSendRecvInfo; + + +void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SUdfdRpcSendRecvInfo *msgInfo = (SUdfdRpcSendRecvInfo *)pMsg->ahandle; + ASSERT(pMsg->ahandle != NULL); + + if (pEpSet) { + if (!isEpsetEqual(&global.mgmtEp.epSet, pEpSet)) { + updateEpSet_s(&global.mgmtEp, pEpSet); + } + } + + if (pMsg->code != TSDB_CODE_SUCCESS) { + fnError("udfd rpc error. code: %s", tstrerror(pMsg->code)); + msgInfo->code = pMsg->code; + goto _return; + } + + if (msgInfo->rpcType == UDFD_RPC_MNODE_CONNECT) { + SConnectRsp connectRsp = {0}; + tDeserializeSConnectRsp(pMsg->pCont, pMsg->contLen, &connectRsp); + if (connectRsp.epSet.numOfEps == 0) { + msgInfo->code = TSDB_CODE_MND_APP_ERROR; + goto _return; + } + + if (connectRsp.dnodeNum > 1 && !isEpsetEqual(&global.mgmtEp.epSet, &connectRsp.epSet)) { + updateEpSet_s(&global.mgmtEp, &connectRsp.epSet); + } + msgInfo->code = 0; + } else if (msgInfo->rpcType == UDFD_RPC_RETRIVE_FUNC) { + SRetrieveFuncRsp retrieveRsp = {0}; + tDeserializeSRetrieveFuncRsp(pMsg->pCont, pMsg->contLen, &retrieveRsp); + + SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0); + SUdf* udf = msgInfo->param; + udf->funcType = pFuncInfo->funcType; + udf->scriptType = pFuncInfo->scriptType; + udf->outputType = pFuncInfo->funcType; + udf->outputLen = pFuncInfo->outputLen; + udf->bufSize = pFuncInfo->bufSize; + + char path[PATH_MAX] = {0}; + snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", pFuncInfo->name); + TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); + // TODO check for failure of flush to disk + taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); + taosCloseFile(&file); + strncpy(udf->path, path, strlen(path)); + taosArrayDestroy(retrieveRsp.pFuncInfos); + msgInfo->code = 0; + } + +_return: + rpcFreeCont(pMsg->pCont); + uv_sem_post(&msgInfo->resultSem); + return; +} + +int32_t udfdConnectToMNode() { + SConnectReq connReq = {0}; + connReq.connType = CONN_TYPE__UDFD; + tstrncpy(connReq.app, "udfd",sizeof(connReq.app)); + tstrncpy(connReq.user, TSDB_DEFAULT_USER, sizeof(connReq.user)); + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(TSDB_DEFAULT_PASS), strlen(TSDB_DEFAULT_PASS), pass); + tstrncpy(connReq.passwd, pass, sizeof(connReq.passwd)); + connReq.pid = htonl(taosGetPId()); + connReq.startTime = htobe64(taosGetTimestampMs()); + + int32_t contLen = tSerializeSConnectReq(NULL, 0, &connReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSConnectReq(pReq, contLen, &connReq); + + SUdfdRpcSendRecvInfo *msgInfo = taosMemoryCalloc(1, sizeof(SUdfdRpcSendRecvInfo)); + msgInfo->rpcType = UDFD_RPC_MNODE_CONNECT; + uv_sem_init(&msgInfo->resultSem, 0); + + SRpcMsg rpcMsg = {0}; + rpcMsg.msgType = TDMT_MND_CONNECT; + rpcMsg.pCont = pReq; + rpcMsg.contLen = contLen; + rpcMsg.ahandle = msgInfo; + rpcSendRequest(global.clientRpc, &global.mgmtEp.epSet, &rpcMsg, NULL); + + uv_sem_wait(&msgInfo->resultSem); + int32_t code = msgInfo->code; + uv_sem_destroy(&msgInfo->resultSem); + taosMemoryFree(msgInfo); + return code; +} + +int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) { + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 1; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, udfName); + + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void *pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + taosArrayDestroy(retrieveReq.pFuncNames); + + SUdfdRpcSendRecvInfo* msgInfo = taosMemoryCalloc(1, sizeof(SUdfdRpcSendRecvInfo)); + msgInfo->rpcType = UDFD_RPC_RETRIVE_FUNC; + msgInfo->param = udf; + uv_sem_init(&msgInfo->resultSem, 0); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = contLen; + rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC; + rpcMsg.ahandle = msgInfo; + rpcSendRequest(clientRpc, &global.mgmtEp.epSet, &rpcMsg, NULL); + + uv_sem_wait(&msgInfo->resultSem); + uv_sem_destroy(&msgInfo->resultSem); + int32_t code = msgInfo->code; + taosMemoryFree(msgInfo); + return code; +} + +static bool udfdRpcRfp(int32_t code) { + if (code == TSDB_CODE_RPC_REDIRECT) { + return true; + } else { + return false; + } +} int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) { pEpSet->version = 0; @@ -528,69 +667,30 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe return 0; } -int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) { - SRetrieveFuncReq retrieveReq = {0}; - retrieveReq.numOfFuncs = 1; - retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); - taosArrayPush(retrieveReq.pFuncNames, udfName); - - int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); - void *pReq = rpcMallocCont(contLen); - tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); - taosArrayDestroy(retrieveReq.pFuncNames); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = contLen; - rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC; - - SRpcMsg rpcRsp = {0}; - rpcSendRecv(clientRpc, &global.mgmtEp.epSet, &rpcMsg, &rpcRsp); - SRetrieveFuncRsp retrieveRsp = {0}; - tDeserializeSRetrieveFuncRsp(rpcRsp.pCont, rpcRsp.contLen, &retrieveRsp); - - SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0); - - udf->funcType = pFuncInfo->funcType; - udf->scriptType = pFuncInfo->scriptType; - udf->outputType = pFuncInfo->funcType; - udf->outputLen = pFuncInfo->outputLen; - udf->bufSize = pFuncInfo->bufSize; - - char path[PATH_MAX] = {0}; - snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", udfName); - TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); - // TODO check for failure of flush to disk - taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); - taosCloseFile(&file); - strncpy(udf->path, path, strlen(path)); - taosArrayDestroy(retrieveRsp.pFuncInfos); - - rpcFreeCont(rpcRsp.pCont); - return 0; -} int32_t udfdOpenClientRpc() { - char *pass = "taosdata"; - char *user = "root"; - char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; - taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt); SRpcInit rpcInit = {0}; - rpcInit.label = (char *)"UDFD"; + rpcInit.label = "UDFD"; rpcInit.numOfThreads = 1; - rpcInit.cfp = udfdProcessRpcRsp; + rpcInit.cfp = (RpcCfp)udfdProcessRpcRsp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = 30 * 1000; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.user = TSDB_DEFAULT_USER; + rpcInit.ckey = "key"; + rpcInit.spi = 1; rpcInit.parent = &global; + rpcInit.rfp = udfdRpcRfp; - rpcInit.user = (char *)user; - rpcInit.ckey = (char *)"key"; - rpcInit.secret = (char *)secretEncrypt; - rpcInit.spi = 1; + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(TSDB_DEFAULT_PASS), strlen(TSDB_DEFAULT_PASS), pass); + rpcInit.secret = pass; global.clientRpc = rpcOpen(&rpcInit); - + if (global.clientRpc == NULL) { + fnError("failed to init dnode rpc client"); + return -1; + } return 0; } @@ -700,12 +800,6 @@ static int32_t udfdRun() { global.udfsHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); uv_mutex_init(&global.udfsMutex); - // TOOD: client rpc to fetch udf function info from mnode - if (udfdOpenClientRpc() != 0) { - fnError("open rpc connection to mnode failure"); - return -1; - } - if (udfdUvInit() != 0) { fnError("uv init failure"); return -2; @@ -717,7 +811,6 @@ static int32_t udfdRun() { int codeClose = uv_loop_close(global.loop); fnDebug("uv loop close. result: %s", uv_err_name(codeClose)); removeListeningPipe(); - udfdCloseClientRpc(); uv_mutex_destroy(&global.udfsMutex); taosHashCleanup(global.udfsHash); return 0; @@ -746,9 +839,22 @@ int main(int argc, char *argv[]) { if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) { fnError("failed to start since read config error"); - return -1; + return -2; } initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp); - return udfdRun(); + if (udfdOpenClientRpc() != 0) { + fnError("open rpc connection to mnode failure"); + return -3; + } + + if (udfdConnectToMNode() != 0) { + fnError("failed to start since can not connect to mnode"); + return -4; + } + + udfdRun(); + + udfdCloseClientRpc(); + } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 9c83b600d27a62c81505cb50e28fabd34324319e..836e97c4dbf3b562a4a72a3ee51ea1916af0d5fc 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -241,7 +241,7 @@ alter_table_clause(A) ::= alter_table_clause(A) ::= full_table_name(B) RENAME TAG column_name(C) column_name(D). { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &C, &D); } alter_table_clause(A) ::= - full_table_name(B) SET TAG column_name(C) NK_EQ literal(D). { A = createAlterTableSetTag(pCxt, B, &C, D); } + full_table_name(B) SET TAG column_name(C) NK_EQ literal(D). { A = createAlterTableSetTag(pCxt, B, &C, releaseRawExprNode(pCxt, D)); } %type multi_create_clause { SNodeList* } %destructor multi_create_clause { nodesDestroyList($$); } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 15e4b8ef3612a41c213f776ef301d5bd82700cad..01833e1776c794f8ba9ac6cecfb0ba3f1e91c1ff 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1041,18 +1041,6 @@ static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { destroyCreateSubTbReq(&pCxt->createTblReq); } -static void destroyDataBlock(STableDataBlocks* pDataBlock) { - if (pDataBlock == NULL) { - return; - } - - taosMemoryFreeClear(pDataBlock->pData); - if (!pDataBlock->cloned) { - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); - } - taosMemoryFreeClear(pDataBlock); -} - static void destroyInsertParseContext(SInsertParseContext* pCxt) { destroyInsertParseContextForTable(pCxt); taosHashCleanup(pCxt->pVgroupsHashObj); @@ -1301,6 +1289,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash CHECK_CODE(buildOutput(&insertCtx)); + destroyBlockArrayList(insertCtx.pVgDataBlocks); return TSDB_CODE_SUCCESS; } @@ -1580,16 +1569,25 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD** fields // schemaless logic start -typedef struct SmlExecHandle { - SHashObj* pBlockHash; - +typedef struct SmlExecTableHandle { SParsedDataColInfo tags; // each table SKVRowBuilder tagsBuilder; // each table SVCreateTbReq createTblReq; // each table +} SmlExecTableHandle; - SQuery* pQuery; +typedef struct SmlExecHandle { + SHashObj* pBlockHash; + SmlExecTableHandle tableExecHandle; + SQuery *pQuery; } SSmlExecHandle; +static void smlDestroyTableHandle(void* pHandle) { + SmlExecTableHandle* handle = (SmlExecTableHandle*)pHandle; + tdDestroyKVRowBuilder(&handle->tagsBuilder); + destroyBoundColumnInfo(&handle->tags); + destroyCreateSubTbReq(&handle->createTblReq); +} + static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema) { col_id_t nCols = pColList->numOfCols; @@ -1692,25 +1690,26 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; + smlDestroyTableHandle(&smlHandle->tableExecHandle); // free for each table SSchema* pTagsSchema = getTableTagSchema(pTableMeta); - setBoundColumnInfo(&smlHandle->tags, pTagsSchema, getNumOfTags(pTableMeta)); - int ret = smlBoundColumnData(tags, &smlHandle->tags, pTagsSchema); + setBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta)); + int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound tags error"); return ret; } SKVRow row = NULL; - ret = smlBuildTagRow(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &row, &pBuf); + ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tagsBuilder, &smlHandle->tableExecHandle.tags, pTagsSchema, &row, &pBuf); if (ret != TSDB_CODE_SUCCESS) { return ret; } - buildCreateTbReq(&smlHandle->createTblReq, tableName, row, pTableMeta->suid); + buildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, row, pTableMeta->suid); STableDataBlocks* pDataBlock = NULL; ret = getDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid), TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, - pTableMeta, &pDataBlock, NULL, &smlHandle->createTblReq); + pTableMeta, &pDataBlock, NULL, &smlHandle->tableExecHandle.createTblReq); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "create data block error"); return ret; @@ -1826,9 +1825,7 @@ void smlDestroyHandle(void* pHandle) { if (!pHandle) return; SSmlExecHandle* handle = (SSmlExecHandle*)pHandle; destroyBlockHashmap(handle->pBlockHash); - tdDestroyKVRowBuilder(&handle->tagsBuilder); - destroyBoundColumnInfo(&handle->tags); - destroyCreateSubTbReq(&handle->createTblReq); + smlDestroyTableHandle(&handle->tableExecHandle); taosMemoryFree(handle); } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 8b649350b70f8b8f7035a956748d0e7cb28a489b..deb899309ebf56f6470602dacdddaf2544edfe3e 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -237,9 +237,7 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { taosMemoryFreeClear(pDataBlock->pData); if (!pDataBlock->cloned) { // free the refcount for metermeta - if (pDataBlock->pTableMeta != NULL) { - taosMemoryFreeClear(pDataBlock->pTableMeta); - } + taosMemoryFreeClear(pDataBlock->pTableMeta); destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9773b69ee75992b2405cb17d09ce34f727d4b7ce..3983b53b6e001cefaf13cce940989268abeb90c5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2949,8 +2949,8 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p } static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) { - SEncoder encoder = {0}; - int32_t contLen = 0; + SEncoder encoder = {0}; + int32_t contLen = 0; SVDropTSmaReq dropSmaReq = {0}; strcpy(dropSmaReq.indexName, pStmt->indexName); @@ -2958,7 +2958,7 @@ static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } - + int32_t ret = 0; tEncodeSize(tEncodeSVDropTSmaReq, &dropSmaReq, contLen, ret); if (ret < 0) { @@ -3800,7 +3800,7 @@ static void destroyCreateTbReqBatch(SVgroupCreateTableBatch* pTbBatch) { taosArrayDestroy(pTbBatch->req.pArray); } -static int32_t rewriteToVnodeModifOpStmt(SQuery* pQuery, SArray* pBufArray) { +static int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray) { SVnodeModifOpStmt* pNewStmt = nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); if (pNewStmt == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -3855,7 +3855,7 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { code = buildCreateTableDataBlock(pCxt->pParseCxt->acctId, pStmt, &info, &pBufArray); } if (TSDB_CODE_SUCCESS == code) { - code = rewriteToVnodeModifOpStmt(pQuery, pBufArray); + code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray); if (TSDB_CODE_SUCCESS != code) { destroyCreateTbReqArray(pBufArray); } @@ -4111,7 +4111,7 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) return TSDB_CODE_OUT_OF_MEMORY; } - return rewriteToVnodeModifOpStmt(pQuery, pBufArray); + return rewriteToVnodeModifyOpStmt(pQuery, pBufArray); } typedef struct SVgroupDropTableBatch { @@ -4251,14 +4251,162 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } - return rewriteToVnodeModifOpStmt(pQuery, pBufArray); + return rewriteToVnodeModifyOpStmt(pQuery, pBufArray); } -static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { - // todo +static int32_t buildAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SVAlterTbReq* pReq) { + pReq->tbName = strdup(pStmt->tableName); + if (NULL == pReq->tbName) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pReq->action = pStmt->alterType; + + switch (pStmt->alterType) { + case TSDB_ALTER_TABLE_ADD_TAG: + case TSDB_ALTER_TABLE_DROP_TAG: + case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: + case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE); + case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: + pReq->tagName = strdup(pStmt->colName); + if (NULL == pReq->tagName) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (DEAL_RES_ERROR == translateValue(pCxt, pStmt->pVal)) { + return pCxt->errCode; + } + pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type); + pReq->nTagVal = pStmt->pVal->node.resType.bytes; + char* pVal = nodesGetValueFromNode(pStmt->pVal); + pReq->pTagVal = IS_VAR_DATA_TYPE(pStmt->pVal->node.resType.type) ? pVal + VARSTR_HEADER_SIZE : pVal; + break; + case TSDB_ALTER_TABLE_ADD_COLUMN: + case TSDB_ALTER_TABLE_DROP_COLUMN: + pReq->colName = strdup(pStmt->colName); + if (NULL == pReq->colName) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pReq->type = pStmt->dataType.type; + pReq->flags = COL_SMA_ON; + pReq->bytes = pStmt->dataType.bytes; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + pReq->colName = strdup(pStmt->colName); + if (NULL == pReq->colName) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pReq->colModBytes = calcTypeBytes(pStmt->dataType); + break; + case TSDB_ALTER_TABLE_UPDATE_OPTIONS: + if (-1 != pStmt->pOptions->ttl) { + pReq->updateTTL = true; + pReq->newTTL = pStmt->pOptions->ttl; + } + if ('\0' != pStmt->pOptions->comment[0]) { + pReq->updateComment = true; + pReq->newComment = strdup(pStmt->pOptions->comment); + if (NULL == pReq->newComment) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: + pReq->colName = strdup(pStmt->colName); + pReq->colNewName = strdup(pStmt->newColName); + if (NULL == pReq->colName || NULL == pReq->colNewName) { + return TSDB_CODE_OUT_OF_MEMORY; + } + break; + default: + break; + } + return TSDB_CODE_SUCCESS; } +static int32_t serializeAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SVAlterTbReq* pReq, + SArray* pArray) { + SVgroupInfo vg = {0}; + int32_t code = getTableHashVgroup(pCxt, pStmt->dbName, pStmt->tableName, &vg); + int tlen = 0; + if (TSDB_CODE_SUCCESS == code) { + tEncodeSize(tEncodeSVAlterTbReq, pReq, tlen, code); + } + if (TSDB_CODE_SUCCESS == code) { + tlen += sizeof(SMsgHead); + void* pMsg = taosMemoryMalloc(tlen); + if (NULL == pMsg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + ((SMsgHead*)pMsg)->vgId = htonl(vg.vgId); + ((SMsgHead*)pMsg)->contLen = htonl(tlen); + void* pBuf = POINTER_SHIFT(pMsg, sizeof(SMsgHead)); + SEncoder coder = {0}; + tEncoderInit(&coder, pBuf, tlen - sizeof(SMsgHead)); + tEncodeSVAlterTbReq(&coder, pReq); + tEncoderClear(&coder); + + SVgDataBlocks* pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == pVgData) { + taosMemoryFree(pMsg); + return TSDB_CODE_OUT_OF_MEMORY; + } + pVgData->vg = vg; + pVgData->pData = pMsg; + pVgData->size = tlen; + pVgData->numOfTables = 1; + taosArrayPush(pArray, &pVgData); + } + + return code; +} + +static int32_t buildModifyVnodeArray(STranslateContext* pCxt, SAlterTableStmt* pStmt, SVAlterTbReq* pReq, + SArray** pArray) { + SArray* pTmpArray = taosArrayInit(1, sizeof(void*)); + if (NULL == pTmpArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = serializeAlterTbReq(pCxt, pStmt, pReq, pTmpArray); + if (TSDB_CODE_SUCCESS == code) { + *pArray = pTmpArray; + } else { + taosArrayDestroy(pTmpArray); + } + + return code; +} + +static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { + SAlterTableStmt* pStmt = (SAlterTableStmt*)pQuery->pRoot; + + STableMeta* pTableMeta = NULL; + int32_t code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pTableMeta); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + if (TSDB_SUPER_TABLE == pTableMeta->tableType) { + return TSDB_CODE_SUCCESS; + } else if (TSDB_CHILD_TABLE != pTableMeta->tableType && TSDB_NORMAL_TABLE != pTableMeta->tableType) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE); + } + + SVAlterTbReq req = {0}; + code = buildAlterTbReq(pCxt, pStmt, &req); + + SArray* pArray = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = buildModifyVnodeArray(pCxt, pStmt, &req, &pArray); + } + if (TSDB_CODE_SUCCESS == code) { + code = rewriteToVnodeModifyOpStmt(pQuery, pArray); + } + + return code; +} + static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pQuery->pRoot)) { @@ -4296,9 +4444,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { code = rewriteDropTable(pCxt, pQuery); break; case QUERY_NODE_ALTER_TABLE_STMT: - if (TSDB_ALTER_TABLE_UPDATE_TAG_VAL == ((SAlterTableStmt*)pQuery->pRoot)->alterType) { - code = rewriteAlterTable(pCxt, pQuery); - } + code = rewriteAlterTable(pCxt, pQuery); break; default: break; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 0c703c9dfc40abd39a59673883c44806463a5e2b..e7716741ed5b7cd07fd01ca70c6fe0b59061d690 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -152,6 +152,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Invalid timeline function"; case TSDB_CODE_PAR_INVALID_PASSWD: return "Invalid password"; + case TSDB_CODE_PAR_INVALID_ALTER_TABLE: + return "Invalid alter table statement"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 42931313382a5d4b0d4fec438970ad52f6b7ff88..ed812405e05ad96ef4390507a0ab5490e3b9f709 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -3500,7 +3500,7 @@ static YYACTIONTYPE yy_reduce( yymsp[-4].minor.yy172 = yylhsminor.yy172; break; case 121: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ -{ yylhsminor.yy172 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy172, &yymsp[-2].minor.yy105, yymsp[0].minor.yy172); } +{ yylhsminor.yy172 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy172, &yymsp[-2].minor.yy105, releaseRawExprNode(pCxt, yymsp[0].minor.yy172)); } yymsp[-5].minor.yy172 = yylhsminor.yy172; break; case 123: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ diff --git a/source/libs/parser/test/parInitialATest.cpp b/source/libs/parser/test/parInitialATest.cpp index 4ddb7736b2cf545f0f05981d43ef0574a77154b6..e1fa8ffe8dd6a3cdd38b35edf3f0e231e3d929e9 100644 --- a/source/libs/parser/test/parInitialATest.cpp +++ b/source/libs/parser/test/parInitialATest.cpp @@ -69,7 +69,7 @@ TEST_F(ParserInitialATest, alterDatabase) { * | COMMENT 'string_value' * } */ -TEST_F(ParserInitialATest, alterTable) { +TEST_F(ParserInitialATest, alterSTable) { useDb("root", "test"); SMAlterStbReq expect = {0}; @@ -119,7 +119,7 @@ TEST_F(ParserInitialATest, alterTable) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_TABLE_STMT); SMAlterStbReq req = {0}; - ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSMAlterStbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + ASSERT_EQ(tDeserializeSMAlterStbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS); ASSERT_EQ(std::string(req.name), std::string(expect.name)); ASSERT_EQ(req.alterType, expect.alterType); ASSERT_EQ(req.numOfFields, expect.numOfFields); @@ -139,24 +139,24 @@ TEST_F(ParserInitialATest, alterTable) { } }); - setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, nullptr, 10); - run("ALTER TABLE t1 TTL 10"); + setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, nullptr, 10); + run("ALTER TABLE st1 TTL 10"); - setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, "test"); - run("ALTER TABLE t1 COMMENT 'test'"); + setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, "test"); + run("ALTER TABLE st1 COMMENT 'test'"); - setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_ADD_COLUMN, 1, "cc1", TSDB_DATA_TYPE_BIGINT); - run("ALTER TABLE t1 ADD COLUMN cc1 BIGINT"); + setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_ADD_COLUMN, 1, "cc1", TSDB_DATA_TYPE_BIGINT); + run("ALTER TABLE st1 ADD COLUMN cc1 BIGINT"); - setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_DROP_COLUMN, 1, "c1"); - run("ALTER TABLE t1 DROP COLUMN c1"); + setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_DROP_COLUMN, 1, "c1"); + run("ALTER TABLE st1 DROP COLUMN c1"); - setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, 1, "c1", TSDB_DATA_TYPE_VARCHAR, + setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, 1, "c1", TSDB_DATA_TYPE_VARCHAR, 20 + VARSTR_HEADER_SIZE); - run("ALTER TABLE t1 MODIFY COLUMN c1 VARCHAR(20)"); + run("ALTER TABLE st1 MODIFY COLUMN c1 VARCHAR(20)"); - setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, 2, "c1", 0, 0, "cc1"); - run("ALTER TABLE t1 RENAME COLUMN c1 cc1"); + setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, 2, "c1", 0, 0, "cc1"); + run("ALTER TABLE st1 RENAME COLUMN c1 cc1"); setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_ADD_TAG, 1, "tag11", TSDB_DATA_TYPE_BIGINT); run("ALTER TABLE st1 ADD TAG tag11 BIGINT"); @@ -171,7 +171,127 @@ TEST_F(ParserInitialATest, alterTable) { setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_TAG_NAME, 2, "tag1", 0, 0, "tag11"); run("ALTER TABLE st1 RENAME TAG tag1 tag11"); - // run("ALTER TABLE st1s1 SET TAG tag1=10"); + // todo + // ADD {FULLTEXT | SMA} INDEX index_name (col_name [, col_name] ...) [index_option] +} + +TEST_F(ParserInitialATest, alterTable) { + useDb("root", "test"); + + SVAlterTbReq expect = {0}; + + auto setAlterColFunc = [&](const char* pTbname, int8_t alterType, const char* pColName, int8_t dataType = 0, + int32_t dataBytes = 0, const char* pNewColName = nullptr) { + memset(&expect, 0, sizeof(SVAlterTbReq)); + expect.tbName = strdup(pTbname); + expect.action = alterType; + expect.colName = strdup(pColName); + + switch (alterType) { + case TSDB_ALTER_TABLE_ADD_COLUMN: + expect.type = dataType; + expect.flags = COL_SMA_ON; + expect.bytes = dataBytes > 0 ? dataBytes : (dataType > 0 ? tDataTypes[dataType].bytes : 0); + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + expect.colModBytes = dataBytes; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: + expect.colNewName = strdup(pNewColName); + break; + default: + break; + } + }; + + auto setAlterTagFunc = [&](const char* pTbname, const char* pTagName, const uint8_t* pNewVal, uint32_t bytes) { + memset(&expect, 0, sizeof(SVAlterTbReq)); + expect.tbName = strdup(pTbname); + expect.action = TSDB_ALTER_TABLE_UPDATE_TAG_VAL; + expect.tagName = strdup(pTagName); + + expect.isNull = (nullptr == pNewVal); + expect.nTagVal = bytes; + expect.pTagVal = pNewVal; + }; + + auto setAlterOptionsFunc = [&](const char* pTbname, int32_t ttl, const char* pComment = nullptr) { + memset(&expect, 0, sizeof(SVAlterTbReq)); + expect.tbName = strdup(pTbname); + expect.action = TSDB_ALTER_TABLE_UPDATE_OPTIONS; + if (-1 != ttl) { + expect.updateTTL = true; + expect.newTTL = ttl; + } + if (nullptr != pComment) { + expect.updateComment = true; + expect.newComment = pComment; + } + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_VNODE_MODIF_STMT); + SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; + + ASSERT_EQ(pStmt->sqlNodeType, QUERY_NODE_ALTER_TABLE_STMT); + ASSERT_NE(pStmt->pDataBlocks, nullptr); + ASSERT_EQ(taosArrayGetSize(pStmt->pDataBlocks), 1); + SVgDataBlocks* pVgData = (SVgDataBlocks*)taosArrayGetP(pStmt->pDataBlocks, 0); + void* pBuf = POINTER_SHIFT(pVgData->pData, sizeof(SMsgHead)); + SVAlterTbReq req = {0}; + SDecoder coder = {0}; + tDecoderInit(&coder, (const uint8_t*)pBuf, pVgData->size); + ASSERT_EQ(tDecodeSVAlterTbReq(&coder, &req), TSDB_CODE_SUCCESS); + + ASSERT_EQ(std::string(req.tbName), std::string(expect.tbName)); + ASSERT_EQ(req.action, expect.action); + if (nullptr != expect.colName) { + ASSERT_EQ(std::string(req.colName), std::string(expect.colName)); + } + ASSERT_EQ(req.type, expect.type); + ASSERT_EQ(req.flags, expect.flags); + ASSERT_EQ(req.bytes, expect.bytes); + ASSERT_EQ(req.colModBytes, expect.colModBytes); + if (nullptr != expect.colNewName) { + ASSERT_EQ(std::string(req.colNewName), std::string(expect.colNewName)); + } + if (nullptr != expect.tagName) { + ASSERT_EQ(std::string(req.tagName), std::string(expect.tagName)); + } + ASSERT_EQ(req.isNull, expect.isNull); + ASSERT_EQ(req.nTagVal, expect.nTagVal); + ASSERT_EQ(memcmp(req.pTagVal, expect.pTagVal, expect.nTagVal), 0); + ASSERT_EQ(req.updateTTL, expect.updateTTL); + ASSERT_EQ(req.newTTL, expect.newTTL); + ASSERT_EQ(req.updateComment, expect.updateComment); + if (nullptr != expect.newComment) { + ASSERT_EQ(std::string(req.newComment), std::string(expect.newComment)); + } + + tDecoderClear(&coder); + }); + + setAlterOptionsFunc("t1", 10, nullptr); + run("ALTER TABLE t1 TTL 10"); + + setAlterOptionsFunc("t1", -1, "test"); + run("ALTER TABLE t1 COMMENT 'test'"); + + setAlterColFunc("t1", TSDB_ALTER_TABLE_ADD_COLUMN, "cc1", TSDB_DATA_TYPE_BIGINT); + run("ALTER TABLE t1 ADD COLUMN cc1 BIGINT"); + + setAlterColFunc("t1", TSDB_ALTER_TABLE_DROP_COLUMN, "c1"); + run("ALTER TABLE t1 DROP COLUMN c1"); + + setAlterColFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, "c1", TSDB_DATA_TYPE_VARCHAR, 20 + VARSTR_HEADER_SIZE); + run("ALTER TABLE t1 MODIFY COLUMN c1 VARCHAR(20)"); + + setAlterColFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, "c1", 0, 0, "cc1"); + run("ALTER TABLE t1 RENAME COLUMN c1 cc1"); + + int64_t val = 10; + setAlterTagFunc("st1s1", "tag1", (const uint8_t*)&val, sizeof(val)); + run("ALTER TABLE st1s1 SET TAG tag1=10"); // todo // ADD {FULLTEXT | SMA} INDEX index_name (col_name [, col_name] ...) [index_option] diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index d4b9f5b2922c01d833244ed99ba9903f23e2615a..c2434e60f3dff0857bc5b92efbc4d27e583caf45 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -985,6 +985,8 @@ static int32_t getMsgType(ENodeType sqlType) { return TDMT_VND_CREATE_TABLE; case QUERY_NODE_DROP_TABLE_STMT: return TDMT_VND_DROP_TABLE; + case QUERY_NODE_ALTER_TABLE_STMT: + return TDMT_VND_ALTER_TABLE; default: break; } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 2710e54f9533ce9ed34552bfa0745d5854c7ad34..5637539feab58969af070ae812b589797800ce76 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -256,6 +256,7 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m return TSDB_CODE_SUCCESS; case TDMT_VND_CREATE_TABLE_RSP: case TDMT_VND_DROP_TABLE_RSP: + case TDMT_VND_ALTER_TABLE_RSP: case TDMT_VND_SUBMIT_RSP: break; default: @@ -1131,6 +1132,24 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); break; } + case TDMT_VND_ALTER_TABLE_RSP: { + SVAlterTbRsp rsp = {0}; + if (msg) { + SDecoder coder = {0}; + tDecoderInit(&coder, msg, msgSize); + code = tDecodeSVAlterTbRsp(&coder, &rsp); + tDecoderClear(&coder); + SCH_ERR_JRET(code); + SCH_ERR_JRET(rsp.code); + } + + SCH_ERR_JRET(rspCode); + + if (NULL == msg) { + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + break; + } case TDMT_VND_SUBMIT_RSP: { SCH_ERR_JRET(rspCode); @@ -1391,6 +1410,10 @@ int32_t schHandleDropTableCallback(void *param, const SDataBuf *pMsg, int32_t co return schHandleCallback(param, pMsg, TDMT_VND_DROP_TABLE_RSP, code); } +int32_t schHandleAlterTableCallback(void *param, const SDataBuf *pMsg, int32_t code) { + return schHandleCallback(param, pMsg, TDMT_VND_ALTER_TABLE_RSP, code); +} + int32_t schHandleQueryCallback(void *param, const SDataBuf *pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_QUERY_RSP, code); } @@ -1490,6 +1513,9 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { case TDMT_VND_DROP_TABLE: *fp = schHandleDropTableCallback; break; + case TDMT_VND_ALTER_TABLE: + *fp = schHandleAlterTableCallback; + break; case TDMT_VND_SUBMIT: *fp = schHandleSubmitCallback; break; @@ -2010,6 +2036,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, switch (msgType) { case TDMT_VND_CREATE_TABLE: case TDMT_VND_DROP_TABLE: + case TDMT_VND_ALTER_TABLE: case TDMT_VND_SUBMIT: { msgSize = pTask->msgLen; msg = taosMemoryCalloc(1, msgSize); diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index b02ca79ed46451a6dcaf6c45ef81598715f749fa..24ddcc1b75c70865f334f553a6b0f1ee176d62ca 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c udf -v 1 print ========= start dnode1 as LEADER system sh/exec.sh -n dnode1 -s start -sleep 2000 +sleep 1000 sql connect print ======== step1 udf diff --git a/tests/script/tsim/sync/threeReplica1VgElect.sim b/tests/script/tsim/sync/threeReplica1VgElect.sim index 7f8c8339cbb44de39cb0da9dca12c19159209d3f..1496d7c778b475895479eb3661fae7ad86a121d3 100644 --- a/tests/script/tsim/sync/threeReplica1VgElect.sim +++ b/tests/script/tsim/sync/threeReplica1VgElect.sim @@ -220,7 +220,6 @@ if $data[0][4] == LEADER then print ---- vgroup $data[0][0] leader switch to dnode $data[0][3] elif $data[0][6] == LEADER then print ---- vgroup $data[0][0] leader switch to dnode $data[0][5] -endi elif $data[0][8] == LEADER then print ---- vgroup $data[0][0] leader switch to dnode $data[0][7] else @@ -342,7 +341,6 @@ elif $data[0][6] == LEADER then goto check_vg_ready_3 endi print ---- vgroup $data[0][0] leader locating dnode $data[0][7] -endi elif $data[0][8] == LEADER then if $data[0][4] == LEADER then goto check_vg_ready_3 diff --git a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim index 1e12e8565feedc200151b64d37bc6663b4e09b53..2dd9b4ed80f3ea4e3c2d4b6f7264d84f68c2d731 100644 --- a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim +++ b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim @@ -420,7 +420,6 @@ elif $data[0][6] == LEADER then goto check_vg_ready_3 endi print ---- vgroup $data[0][0] leader locating dnode $data[0][7] -endi elif $data[0][8] == LEADER then if $data[0][4] == LEADER then goto check_vg_ready_3 diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index 0c96635a788ba18ab391cfad869e2a68e675954c..ee9e87cf047ae35cf771d1c92dc89b737bb584de 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -1,278 +1,288 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics -#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-1vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 1 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 3 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$cdb_index = 0 -#=============================== start consume =============================# - -print ================ test consume from stb -$loop_cnt = 0 -loop_consume_diff_topic_from_stb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_stb_column - $topicList = ' . topic_stb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_stb_all - $topicList = ' . topic_stb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_stb_function - $topicList = ' . topic_stb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_stb_end -endi - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $expectmsgcnt then - return -1 -endi -if $data[0][3] != $expectmsgcnt then - return -1 -endi -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_stb -loop_consume_diff_topic_from_stb_end: - -print ================ test consume from ctb -$loop_cnt = 0 -loop_consume_diff_topic_from_ctb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ctb_column - $topicList = ' . topic_ctb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ctb_all - $topicList = ' . topic_ctb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ctb_function - $topicList = ' . topic_ctb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ctb_end -endi - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfCtb then - return -1 -endi -if $data[0][3] != $totalMsgOfCtb then - return -1 -endi -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ctb -loop_consume_diff_topic_from_ctb_end: - -print ================ test consume from ntb -$loop_cnt = 0 -loop_consume_diff_topic_from_ntb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ntb_column - $topicList = ' . topic_ntb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ntb_all - $topicList = ' . topic_ntb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ntb_function - $topicList = ' . topic_ntb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ntb_end -endi - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfNtb then - return -1 -endi -if $data[0][3] != $totalMsgOfNtb then - return -1 -endi -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ntb -loop_consume_diff_topic_from_ntb_end: - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-1vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 3 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + + +$cdb_index = 0 +#=============================== start consume =============================# + +print ================ test consume from stb +$loop_cnt = 0 +loop_consume_diff_topic_from_stb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_stb_column + $topicList = ' . topic_stb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_stb_all + $topicList = ' . topic_stb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_stb_function + $topicList = ' . topic_stb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_stb_end +endi + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_stb +loop_consume_diff_topic_from_stb_end: + +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ctb_column + $topicList = ' . topic_ctb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ctb_all + $topicList = ' . topic_ctb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ctb_function + $topicList = ' . topic_ctb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ctb_end +endi + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ctb +loop_consume_diff_topic_from_ctb_end: + +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ntb_column + $topicList = ' . topic_ntb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ntb_all + $topicList = ' . topic_ntb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ntb_function + $topicList = ' . topic_ntb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ntb_end +endi + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ntb +loop_consume_diff_topic_from_ntb_end: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic1Of2Cons.sim b/tests/script/tsim/tmq/basic1Of2Cons.sim index 957f1774f9e8dbefca393900478f9649373de8ef..9d4b0e75da7be0490167318ffb5bc21b81acadfd 100644 --- a/tests/script/tsim/tmq/basic1Of2Cons.sim +++ b/tests/script/tsim/tmq/basic1Of2Cons.sim @@ -1,372 +1,382 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics -#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-1vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 1 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 5 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$cdb_index = 0 -#=============================== start consume =============================# - -print ================ test consume from stb -$loop_cnt = 0 -loop_consume_diff_topic_from_stb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table for stb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_stb_column - $topicList = ' . topic_stb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_stb_all - $topicList = ' . topic_stb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_stb_function - $topicList = ' . topic_stb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_stb_end -endi - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb -if $data[0][2] == $totalMsgOfStb then - if $data[1][2] == 0 then - goto check_ok_0 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfStb then - goto check_ok_0 - endi -endi -return -1 -check_ok_0: - -if $data[0][3] == $totalMsgOfStb then - if $data[1][3] == 0 then - goto check_ok_1 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfStb then - goto check_ok_1 - endi -endi -return -1 -check_ok_1: - -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_stb -loop_consume_diff_topic_from_stb_end: - -print ================ test consume from ctb -$loop_cnt = 0 -loop_consume_diff_topic_from_ctb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table for ctb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ctb_column - $topicList = ' . topic_ctb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ctb_all - $topicList = ' . topic_ctb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ctb_function - $topicList = ' . topic_ctb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ctb_end -endi - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then - if $data[1][2] == 0 then - goto check_ok_2 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then - goto check_ok_2 - endi -endi -return -1 -check_ok_2: - -if $data[0][3] == $totalMsgOfCtb then - if $data[1][3] == 0 then - goto check_ok_3 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfCtb then - goto check_ok_3 - endi -endi -return -1 -check_ok_3: - -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ctb -loop_consume_diff_topic_from_ctb_end: - -print ================ test consume from ntb -$loop_cnt = 0 -loop_consume_diff_topic_from_ntb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table for ntb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ntb_column - $topicList = ' . topic_ntb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ntb_all - $topicList = ' . topic_ntb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ntb_function - $topicList = ' . topic_ntb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ntb_end -endi - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[1][1] == 0 then - if $data[0][1] != 1 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then - if $data[1][2] == 0 then - goto check_ok_4 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then - goto check_ok_4 - endi -endi -return -1 -check_ok_4: - -if $data[0][3] == $totalMsgOfNtb then - if $data[1][3] == 0 then - goto check_ok_5 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfNtb then - goto check_ok_5 - endi -endi -return -1 -check_ok_5: - -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ntb -loop_consume_diff_topic_from_ntb_end: - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics +#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-1vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + +$cdb_index = 0 + +#=============================== start consume =============================# + +print ================ test consume from stb +$loop_cnt = 0 +loop_consume_diff_topic_from_stb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for stb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_stb_column + $topicList = ' . topic_stb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_stb_all + $topicList = ' . topic_stb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_stb_function + $topicList = ' . topic_stb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_stb_end +endi + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb +if $data[0][2] == $totalMsgOfStb then + if $data[1][2] == 0 then + goto check_ok_0 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfStb then + goto check_ok_0 + endi +endi +return -1 +check_ok_0: + +if $data[0][3] == $totalMsgOfStb then + if $data[1][3] == 0 then + goto check_ok_1 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfStb then + goto check_ok_1 + endi +endi +return -1 +check_ok_1: + +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_stb +loop_consume_diff_topic_from_stb_end: + +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for ctb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ctb_column + $topicList = ' . topic_ctb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ctb_all + $topicList = ' . topic_ctb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ctb_function + $topicList = ' . topic_ctb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ctb_end +endi + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb +if $data[0][2] == $totalMsgOfCtb then + if $data[1][2] == 0 then + goto check_ok_2 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfCtb then + goto check_ok_2 + endi +endi +return -1 +check_ok_2: + +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == 0 then + goto check_ok_3 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfCtb then + goto check_ok_3 + endi +endi +return -1 +check_ok_3: + +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ctb +loop_consume_diff_topic_from_ctb_end: + +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for ntb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ntb_column + $topicList = ' . topic_ntb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ntb_all + $topicList = ' . topic_ntb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ntb_function + $topicList = ' . topic_ntb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ntb_end +endi + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[1][1] == 0 then + if $data[0][1] != 1 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb +if $data[0][2] == $totalMsgOfNtb then + if $data[1][2] == 0 then + goto check_ok_4 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfNtb then + goto check_ok_4 + endi +endi +return -1 +check_ok_4: + +if $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == 0 then + goto check_ok_5 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfNtb then + goto check_ok_5 + endi +endi +return -1 +check_ok_5: + +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ntb +loop_consume_diff_topic_from_ntb_end: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic2.sim b/tests/script/tsim/tmq/basic2.sim index 53f10e22475cc067a25b86b7aa0c46c342d0111c..dce73be592e459264916f67b4390141115c05bc8 100644 --- a/tests/script/tsim/tmq/basic2.sim +++ b/tests/script/tsim/tmq/basic2.sim @@ -1,219 +1,229 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics -#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-1vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 1 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 3 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$topicNum = 3 - -#=============================== start consume =============================# - - -print ================ test consume from stb -print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function -$topicList = ' . topic_stb_column -$topicList = $topicList . , -$topicList = $topicList . topic_stb_all -$topicList = $topicList . , -$topicList = $topicList . topic_stb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $expectmsgcnt then - return -1 -endi -if $data[0][3] != $expectmsgcnt then - return -1 -endi - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb1 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ctb -print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function -$topicList = ' . topic_ctb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfCtb then - return -1 -endi -if $data[0][3] != $totalMsgOfCtb then - return -1 -endi - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb2 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ntb -print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function -$topicList = ' . topic_ntb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfNtb then - return -1 -endi -if $data[0][3] != $totalMsgOfNtb then - return -1 -endi - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-1vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 3 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + + +$topicNum = 3 + +#=============================== start consume =============================# + + +print ================ test consume from stb +print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfStb * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic2Of2Cons.sim b/tests/script/tsim/tmq/basic2Of2Cons.sim index 01ccb2b5154ee8f865265432ff977607c009de20..0494ddb5b826fa346d9460fa28e0b8f9133dd986 100644 --- a/tests/script/tsim/tmq/basic2Of2Cons.sim +++ b/tests/script/tsim/tmq/basic2Of2Cons.sim @@ -1,309 +1,318 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics -#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-1vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 1 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 5 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$topicNum = 3 - -#=============================== start consume =============================# - - -print ================ test consume from stb -print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function -$topicList = ' . topic_stb_column -$topicList = $topicList . , -$topicList = $topicList . topic_stb_all -$topicList = $topicList . , -$topicList = $topicList . topic_stb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb -if $data[0][2] == $totalMsgOfStb then - if $data[1][2] == 0 then - goto check_ok_0 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfStb then - goto check_ok_0 - endi -endi -return -1 -check_ok_0: - -if $data[0][3] == $totalMsgOfStb then - if $data[1][3] == 0 then - goto check_ok_1 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfStb then - goto check_ok_1 - endi -endi -return -1 -check_ok_1: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb1 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table for ctb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ctb -print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function -$topicList = ' . topic_ctb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then - if $data[1][2] == 0 then - goto check_ok_2 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then - goto check_ok_2 - endi -endi -return -1 -check_ok_2: - -if $data[0][3] == $totalMsgOfCtb then - if $data[1][3] == 0 then - goto check_ok_3 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfCtb then - goto check_ok_3 - endi -endi -return -1 -check_ok_3: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb2 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table for ntb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ntb -print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function -$topicList = ' . topic_ntb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[1][1] == 0 then - if $data[0][1] != 1 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then - if $data[1][2] == 0 then - goto check_ok_4 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then - goto check_ok_4 - endi -endi -return -1 -check_ok_4: - -if $data[0][3] == $totalMsgOfNtb then - if $data[1][3] == 0 then - goto check_ok_5 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfNtb then - goto check_ok_5 - endi -endi -return -1 -check_ok_5: - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics +#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-1vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + +$topicNum = 3 + +#=============================== start consume =============================# + + +print ================ test consume from stb +print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfStb * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb +if $data[0][2] == $totalMsgOfStb then + if $data[1][2] == 0 then + goto check_ok_0 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfStb then + goto check_ok_0 + endi +endi +return -1 +check_ok_0: + +if $data[0][3] == $totalMsgOfStb then + if $data[1][3] == 0 then + goto check_ok_1 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfStb then + goto check_ok_1 + endi +endi +return -1 +check_ok_1: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for ctb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb +if $data[0][2] == $totalMsgOfCtb then + if $data[1][2] == 0 then + goto check_ok_2 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfCtb then + goto check_ok_2 + endi +endi +return -1 +check_ok_2: + +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == 0 then + goto check_ok_3 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfCtb then + goto check_ok_3 + endi +endi +return -1 +check_ok_3: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for ntb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[1][1] == 0 then + if $data[0][1] != 1 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb +if $data[0][2] == $totalMsgOfNtb then + if $data[1][2] == 0 then + goto check_ok_4 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfNtb then + goto check_ok_4 + endi +endi +return -1 +check_ok_4: + +if $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == 0 then + goto check_ok_5 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfNtb then + goto check_ok_5 + endi +endi +return -1 +check_ok_5: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim index d5c800b0e9fbaa97b08ac3198fdd6a7892336cb4..480cf520d9c942a11c70395bd347540da773ad39 100644 --- a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -27,6 +27,7 @@ $tstart = 1640966400000 # 2022-01-01 00:00:00.000 $pullDelay = 5 $ifcheckdata = 1 +$ifmanualcommit = 1 $showMsg = 1 $showRow = 0 @@ -53,8 +54,16 @@ sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 # return -1 #endi +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' $keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest $keyList = $keyList . ' +print ========== key list: $keyList $topicNum = 2 @@ -72,7 +81,7 @@ $consumerId = 0 $totalMsgOfOneTopic = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfOneTopic * $topicNum $expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_stb_all @@ -80,7 +89,7 @@ $topicList = $topicList . , $topicList = $topicList . topic_stb_function $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -158,7 +167,7 @@ sleep 500 sql use $cdbName print == create consume info table and consume result table for ctb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) sql show tables @@ -179,14 +188,14 @@ $consumerId = 0 $totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum $expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_ctb_function $topicList = $topicList . , $topicList = $topicList . topic_ctb_all $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -249,7 +258,7 @@ sleep 500 sql use $cdbName print == create consume info table and consume result table for ntb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) sql show tables @@ -270,7 +279,7 @@ $consumerId = 0 $totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum $expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_ntb_function @@ -278,7 +287,7 @@ $topicList = $topicList . , $topicList = $topicList . topic_ntb_all $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start diff --git a/tests/script/tsim/tmq/basic3.sim b/tests/script/tsim/tmq/basic3.sim index de771ba892ec01825fbed08464a7d3794a5226bf..8d677766d74371209e53bab94b62a524318146ee 100644 --- a/tests/script/tsim/tmq/basic3.sim +++ b/tests/script/tsim/tmq/basic3.sim @@ -1,278 +1,288 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics -#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-4vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 4 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 3 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$cdb_index = 0 -#=============================== start consume =============================# - -print ================ test consume from stb -$loop_cnt = 0 -loop_consume_diff_topic_from_stb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_stb_column - $topicList = ' . topic_stb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_stb_all - $topicList = ' . topic_stb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_stb_function - $topicList = ' . topic_stb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_stb_end -endi - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $expectmsgcnt then - return -1 -endi -if $data[0][3] != $expectmsgcnt then - return -1 -endi -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_stb -loop_consume_diff_topic_from_stb_end: - -print ================ test consume from ctb -$loop_cnt = 0 -loop_consume_diff_topic_from_ctb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ctb_column - $topicList = ' . topic_ctb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ctb_all - $topicList = ' . topic_ctb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ctb_function - $topicList = ' . topic_ctb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ctb_end -endi - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfCtb then - return -1 -endi -if $data[0][3] != $totalMsgOfCtb then - return -1 -endi -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ctb -loop_consume_diff_topic_from_ctb_end: - -print ================ test consume from ntb -$loop_cnt = 0 -loop_consume_diff_topic_from_ntb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ntb_column - $topicList = ' . topic_ntb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ntb_all - $topicList = ' . topic_ntb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ntb_function - $topicList = ' . topic_ntb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ntb_end -endi - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfNtb then - return -1 -endi -if $data[0][3] != $totalMsgOfNtb then - return -1 -endi -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ntb -loop_consume_diff_topic_from_ntb_end: - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-4vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 3 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + + +$cdb_index = 0 +#=============================== start consume =============================# + +print ================ test consume from stb +$loop_cnt = 0 +loop_consume_diff_topic_from_stb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_stb_column + $topicList = ' . topic_stb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_stb_all + $topicList = ' . topic_stb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_stb_function + $topicList = ' . topic_stb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_stb_end +endi + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_stb +loop_consume_diff_topic_from_stb_end: + +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ctb_column + $topicList = ' . topic_ctb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ctb_all + $topicList = ' . topic_ctb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ctb_function + $topicList = ' . topic_ctb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ctb_end +endi + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ctb +loop_consume_diff_topic_from_ctb_end: + +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ntb_column + $topicList = ' . topic_ntb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ntb_all + $topicList = ' . topic_ntb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ntb_function + $topicList = ' . topic_ntb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ntb_end +endi + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ntb +loop_consume_diff_topic_from_ntb_end: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic3Of2Cons.sim b/tests/script/tsim/tmq/basic3Of2Cons.sim index bf640ae1a1be5d710b6fbe588ff0b99662a3f5f2..afaf824acbfd914531af27ac19257e6f6844efa7 100644 --- a/tests/script/tsim/tmq/basic3Of2Cons.sim +++ b/tests/script/tsim/tmq/basic3Of2Cons.sim @@ -1,384 +1,393 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics -#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-4vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 4 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 5 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$cdb_index = 0 -#=============================== start consume =============================# - -print ================ test consume from stb -$loop_cnt = 0 -loop_consume_diff_topic_from_stb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_stb_column - $topicList = ' . topic_stb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_stb_all - $topicList = ' . topic_stb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_stb_function - $topicList = ' . topic_stb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_stb_end -endi - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -if $data[0][2] <= 0 then - return -1 -endi -if $data[0][2] >= $expectmsgcnt then - return -1 -endi - -if $data[1][2] <= 0 then - return -1 -endi -if $data[1][2] >= $expectmsgcnt then - return -1 -endi - -$sumOfMsgCnt = $data[0][2] + $data[1][2] -if $sumOfMsgCnt != $expectmsgcnt then - return -1 -endi - - -if $data[0][3] <= 0 then - return -1 -endi -if $data[0][3] >= $expectmsgcnt then - return -1 -endi - -if $data[1][3] <= 0 then - return -1 -endi -if $data[1][3] >= $expectmsgcnt then - return -1 -endi - -$sumOfMsgRows = $data[0][3] + $data[1][3] -if $sumOfMsgRows != $expectmsgcnt then - return -1 -endi - -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_stb -loop_consume_diff_topic_from_stb_end: - -print ================ test consume from ctb -$loop_cnt = 0 -loop_consume_diff_topic_from_ctb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ctb_column - $topicList = ' . topic_ctb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ctb_all - $topicList = ' . topic_ctb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ctb_function - $topicList = ' . topic_ctb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ctb_end -endi - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then - if $data[1][2] == 0 then - goto check_ok_0 - endi -elif $data[1][2] == $totalMsgOfCtb then - if $data[0][2] == 0 then - goto check_ok_0 - endi -endi -return -1 -check_ok_0: - -if $data[0][3] == $totalMsgOfCtb then - if $data[1][3] == 0 then - goto check_ok_1 - endi -elif $data[1][3] == $totalMsgOfCtb then - if $data[0][3] == 0 then - goto check_ok_1 - endi -endi -return -1 -check_ok_1: - -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ctb -loop_consume_diff_topic_from_ctb_end: - -print ================ test consume from ntb -$loop_cnt = 0 -loop_consume_diff_topic_from_ntb: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdb_index = $cdb_index + 1 -$cdbName = cdb . $cdb_index -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - -if $loop_cnt == 0 then - print == scenario 1: topic_ntb_column - $topicList = ' . topic_ntb_column - $topicList = $topicList . ' -elif $loop_cnt == 1 then - print == scenario 2: topic_ntb_all - $topicList = ' . topic_ntb_all - $topicList = $topicList . ' -elif $loop_cnt == 2 then - print == scenario 3: topic_ntb_function - $topicList = ' . topic_ntb_function - $topicList = $topicList . ' -else - goto loop_consume_diff_topic_from_ntb_end -endi - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[1][1] == 0 then - if $data[0][1] != 1 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then - if $data[1][2] == 0 then - goto check_ok_2 - endi -elif $data[1][2] == $totalMsgOfNtb then - if $data[0][2] == 0 then - goto check_ok_2 - endi -endi -return -1 -check_ok_2: - -if $data[0][3] == $totalMsgOfNtb then - if $data[1][3] == 0 then - goto check_ok_3 - endi -elif $data[1][3] == $totalMsgOfNtb then - if $data[0][3] == 0 then - goto check_ok_3 - endi -endi -return -1 -check_ok_3: - -$loop_cnt = $loop_cnt + 1 -goto loop_consume_diff_topic_from_ntb -loop_consume_diff_topic_from_ntb_end: - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics +#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-4vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + +$cdb_index = 0 +#=============================== start consume =============================# + +print ================ test consume from stb +$loop_cnt = 0 +loop_consume_diff_topic_from_stb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_stb_column + $topicList = ' . topic_stb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_stb_all + $topicList = ' . topic_stb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_stb_function + $topicList = ' . topic_stb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_stb_end +endi + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +if $data[0][2] <= 0 then + return -1 +endi +if $data[0][2] >= $expectmsgcnt then + return -1 +endi + +if $data[1][2] <= 0 then + return -1 +endi +if $data[1][2] >= $expectmsgcnt then + return -1 +endi + +$sumOfMsgCnt = $data[0][2] + $data[1][2] +if $sumOfMsgCnt != $expectmsgcnt then + return -1 +endi + + +if $data[0][3] <= 0 then + return -1 +endi +if $data[0][3] >= $expectmsgcnt then + return -1 +endi + +if $data[1][3] <= 0 then + return -1 +endi +if $data[1][3] >= $expectmsgcnt then + return -1 +endi + +$sumOfMsgRows = $data[0][3] + $data[1][3] +if $sumOfMsgRows != $expectmsgcnt then + return -1 +endi + +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_stb +loop_consume_diff_topic_from_stb_end: + +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ctb_column + $topicList = ' . topic_ctb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ctb_all + $topicList = ' . topic_ctb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ctb_function + $topicList = ' . topic_ctb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ctb_end +endi + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb +if $data[0][2] == $totalMsgOfCtb then + if $data[1][2] == 0 then + goto check_ok_0 + endi +elif $data[1][2] == $totalMsgOfCtb then + if $data[0][2] == 0 then + goto check_ok_0 + endi +endi +return -1 +check_ok_0: + +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == 0 then + goto check_ok_1 + endi +elif $data[1][3] == $totalMsgOfCtb then + if $data[0][3] == 0 then + goto check_ok_1 + endi +endi +return -1 +check_ok_1: + +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ctb +loop_consume_diff_topic_from_ctb_end: + +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ntb_column + $topicList = ' . topic_ntb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ntb_all + $topicList = ' . topic_ntb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ntb_function + $topicList = ' . topic_ntb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ntb_end +endi + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[1][1] == 0 then + if $data[0][1] != 1 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb +if $data[0][2] == $totalMsgOfNtb then + if $data[1][2] == 0 then + goto check_ok_2 + endi +elif $data[1][2] == $totalMsgOfNtb then + if $data[0][2] == 0 then + goto check_ok_2 + endi +endi +return -1 +check_ok_2: + +if $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == 0 then + goto check_ok_3 + endi +elif $data[1][3] == $totalMsgOfNtb then + if $data[0][3] == 0 then + goto check_ok_3 + endi +endi +return -1 +check_ok_3: + +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ntb +loop_consume_diff_topic_from_ntb_end: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic4.sim b/tests/script/tsim/tmq/basic4.sim index 42023bda7e1f5d5defd0266ae5a4146f50851314..9b418f12f2b95410994cf52fd4e140a8dcc73889 100644 --- a/tests/script/tsim/tmq/basic4.sim +++ b/tests/script/tsim/tmq/basic4.sim @@ -1,216 +1,226 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics -#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics -#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-4vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 4 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 3 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$topicNum = 3 - -print ================ test consume from stb -print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function -$topicList = ' . topic_stb_column -$topicList = $topicList . , -$topicList = $topicList . topic_stb_all -$topicList = $topicList . , -$topicList = $topicList . topic_stb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $expectmsgcnt then - return -1 -endi -if $data[0][3] != $expectmsgcnt then - return -1 -endi - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb1 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ctb -print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function -$topicList = ' . topic_ctb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfCtb then - return -1 -endi -if $data[0][3] != $totalMsgOfCtb then - return -1 -endi - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb2 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ntb -print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function -$topicList = ' . topic_ntb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -if $rows != 1 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] != $consumerId then - return -1 -endi -if $data[0][2] != $totalMsgOfNtb then - return -1 -endi -if $data[0][3] != $totalMsgOfNtb then - return -1 -endi - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-4vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 3 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + + +$topicNum = 3 + +print ================ test consume from stb +print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfStb * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectmsgcnt then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic4Of2Cons.sim b/tests/script/tsim/tmq/basic4Of2Cons.sim index a17d8a2f455bef554aba4117d6c2f3f4e2429e4b..510aaf0e1a43c35b47e205333462a1215e7d4e63 100644 --- a/tests/script/tsim/tmq/basic4Of2Cons.sim +++ b/tests/script/tsim/tmq/basic4Of2Cons.sim @@ -1,319 +1,328 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics -#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-4vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 4 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 5 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$topicNum = 3 - -print ================ test consume from stb -print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function -$topicList = ' . topic_stb_column -$topicList = $topicList . , -$topicList = $topicList . topic_stb_all -$topicList = $topicList . , -$topicList = $topicList . topic_stb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfStb = $ctbNum * $rowsPerCtb -$totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -if $data[0][2] <= 0 then - return -1 -endi -if $data[0][2] >= $expectmsgcnt then - return -1 -endi - -if $data[1][2] <= 0 then - return -1 -endi -if $data[1][2] >= $expectmsgcnt then - return -1 -endi - -$sumOfConsMsg = $data[0][2] + $data[1][2] -if $sumOfConsMsg != $expectmsgcnt then - return -1 -endi - -if $data[0][3] <= 0 then - return -1 -endi -if $data[0][3] >= $expectmsgcnt then - return -1 -endi - -if $data[1][3] <= 0 then - return -1 -endi -if $data[1][3] >= $expectmsgcnt then - return -1 -endi - -$sumOfConsRow = $data[0][3] + $data[1][3] -if $sumOfConsRow != $expectmsgcnt then - return -1 -endi - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb1 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ctb -print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function -$topicList = ' . topic_ctb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then - if $data[1][2] == 0 then - goto check_ok_0 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then - goto check_ok_0 - endi -endi -return -1 -check_ok_0: - -if $data[0][3] == $totalMsgOfCtb then - if $data[1][3] == 0 then - goto check_ok_1 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfCtb then - goto check_ok_1 - endi -endi -return -1 -check_ok_1: - - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb2 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ntb -print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function -$topicList = ' . topic_ntb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_all -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_function -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[1][1] == 0 then - if $data[0][1] != 1 then - return -1 - endi -endi - -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then - if $data[1][2] == 0 then - goto check_ok_2 - endi -elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then - goto check_ok_2 - endi -endi -return -1 -check_ok_2: - -if $data[0][3] == $totalMsgOfNtb then - if $data[1][3] == 0 then - goto check_ok_3 - endi -elif $data[0][3] == 0 then - if $data[1][3] == $totalMsgOfNtb then - goto check_ok_3 - endi -endi -return -1 -check_ok_3: - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic2Of2Cons.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics +#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-4vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + +$topicNum = 3 + +print ================ test consume from stb +print == multi toipcs: topic_stb_column + topic_stb_all + topic_stb_function +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfStb * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +if $data[0][2] <= 0 then + return -1 +endi +if $data[0][2] >= $expectmsgcnt then + return -1 +endi + +if $data[1][2] <= 0 then + return -1 +endi +if $data[1][2] >= $expectmsgcnt then + return -1 +endi + +$sumOfConsMsg = $data[0][2] + $data[1][2] +if $sumOfConsMsg != $expectmsgcnt then + return -1 +endi + +if $data[0][3] <= 0 then + return -1 +endi +if $data[0][3] >= $expectmsgcnt then + return -1 +endi + +if $data[1][3] <= 0 then + return -1 +endi +if $data[1][3] >= $expectmsgcnt then + return -1 +endi + +$sumOfConsRow = $data[0][3] + $data[1][3] +if $sumOfConsRow != $expectmsgcnt then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == multi toipcs: topic_ctb_column + topic_ctb_all + topic_ctb_function +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb +if $data[0][2] == $totalMsgOfCtb then + if $data[1][2] == 0 then + goto check_ok_0 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfCtb then + goto check_ok_0 + endi +endi +return -1 +check_ok_0: + +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == 0 then + goto check_ok_1 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfCtb then + goto check_ok_1 + endi +endi +return -1 +check_ok_1: + + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == multi toipcs: topic_ntb_column + topic_ntb_all + topic_ntb_function +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_function +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[1][1] == 0 then + if $data[0][1] != 1 then + return -1 + endi +endi + +# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb +if $data[0][2] == $totalMsgOfNtb then + if $data[1][2] == 0 then + goto check_ok_2 + endi +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfNtb then + goto check_ok_2 + endi +endi +return -1 +check_ok_2: + +if $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == 0 then + goto check_ok_3 + endi +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfNtb then + goto check_ok_3 + endi +endi +return -1 +check_ok_3: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim b/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim index e32a1df8025f1105ffcaee8e0f081149cb756f91..fbdea96f9387e4b204e208725f3bcb474ea6eece 100644 --- a/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim +++ b/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim @@ -1,88 +1,88 @@ -# stop all dnodes before start this case -system sh/stop_dnodes.sh - -# deploy dnode 1 -system sh/deploy.sh -n dnode1 -i 1 - -# add some config items for this case -#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 - -# start dnode 1 -system sh/exec.sh -n dnode1 -s start - -sql connect - -#---- global parameters start ----# -$dbName = db -$vgroups = 1 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -print == create database $dbName vgroups $vgroups -sql create database $dbName vgroups $vgroups - -#wait database ready -$loop_cnt = 0 -check_db_ready: -if $loop_cnt == 10 then - print ====> database not ready! - return -1 -endi -sql show databases -print ==> rows: $rows -print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] -print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] -if $data(db)[19] != nostrict then - sleep 100 - $loop_cnt = $loop_cnt + 1 - goto check_db_ready -endi - -sql use $dbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi - -print == create super table -sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) -sql show stables -if $rows != 1 then - return -1 -endi - -print == create child table, normal table and insert data -$i = 0 -while $i < $ctbNum - $ctb = $ctbPrefix . $i - $ntb = $ntbPrefix . $i - sql create table $ctb using $stbPrefix tags( $i ) - sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) - - $x = 0 - while $x < $rowsPerCtb - $binary = ' . binary- - $binary = $binary . $i - $binary = $binary . ' - - sql insert into $ctb values ($tstart , $i , $x , $binary ) - sql insert into $ntb values ($tstart , $i , $x , $binary ) - $tstart = $tstart + 1 - $x = $x + 1 - endw - - $i = $i + 1 - $tstart = 1640966400000 -endw +# stop all dnodes before start this case +system sh/stop_dnodes.sh + +# deploy dnode 1 +system sh/deploy.sh -n dnode1 -i 1 + +# add some config items for this case +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +# start dnode 1 +system sh/exec.sh -n dnode1 -s start + +sql connect + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +print == create database $dbName vgroups $vgroups +sql create database $dbName vgroups $vgroups + +#wait database ready +$loop_cnt = 0 +check_db_ready: +if $loop_cnt == 10 then + print ====> database not ready! + return -1 +endi +sql show databases +print ==> rows: $rows +print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $data(db)[19] != nostrict then + sleep 100 + $loop_cnt = $loop_cnt + 1 + goto check_db_ready +endi + +sql use $dbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi + +print == create super table +sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) +sql show stables +if $rows != 1 then + return -1 +endi + +print == create child table, normal table and insert data +$i = 0 +while $i < $ctbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + sql create table $ctb using $stbPrefix tags( $i ) + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) + + $x = 0 + while $x < $rowsPerCtb + $binary = ' . binary- + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , $i , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw diff --git a/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim b/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim index 4750aab214e074408f3076cdd0533315fa436ea5..8b5573486d46419ba8b500fb392406e6e31705ed 100644 --- a/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim +++ b/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim @@ -1,88 +1,88 @@ -# stop all dnodes before start this case -system sh/stop_dnodes.sh - -# deploy dnode 1 -system sh/deploy.sh -n dnode1 -i 1 - -# add some config items for this case -#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 - -# start dnode 1 -system sh/exec.sh -n dnode1 -s start - -sql connect - -#---- global parameters start ----# -$dbName = db -$vgroups = 4 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -print == create database $dbName vgroups $vgroups -sql create database $dbName vgroups $vgroups - -#wait database ready -$loop_cnt = 0 -check_db_ready: -if $loop_cnt == 10 then - print ====> database not ready! - return -1 -endi -sql show databases -print ==> rows: $rows -print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] -print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] -if $data(db)[19] != nostrict then - sleep 100 - $loop_cnt = $loop_cnt + 1 - goto check_db_ready -endi - -sql use $dbName - -print == create consume info table and consume result table -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi - -print == create super table -sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) -sql show stables -if $rows != 1 then - return -1 -endi - -print == create child table, normal table and insert data -$i = 0 -while $i < $ctbNum - $ctb = $ctbPrefix . $i - $ntb = $ntbPrefix . $i - sql create table $ctb using $stbPrefix tags( $i ) - sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) - - $x = 0 - while $x < $rowsPerCtb - $binary = ' . binary- - $binary = $binary . $i - $binary = $binary . ' - - sql insert into $ctb values ($tstart , $i , $x , $binary ) - sql insert into $ntb values ($tstart , $i , $x , $binary ) - $tstart = $tstart + 1 - $x = $x + 1 - endw - - $i = $i + 1 - $tstart = 1640966400000 -endw +# stop all dnodes before start this case +system sh/stop_dnodes.sh + +# deploy dnode 1 +system sh/deploy.sh -n dnode1 -i 1 + +# add some config items for this case +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +# start dnode 1 +system sh/exec.sh -n dnode1 -s start + +sql connect + +#---- global parameters start ----# +$dbName = db +$vgroups = 4 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +print == create database $dbName vgroups $vgroups +sql create database $dbName vgroups $vgroups + +#wait database ready +$loop_cnt = 0 +check_db_ready: +if $loop_cnt == 10 then + print ====> database not ready! + return -1 +endi +sql show databases +print ==> rows: $rows +print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $data(db)[19] != nostrict then + sleep 100 + $loop_cnt = $loop_cnt + 1 + goto check_db_ready +endi + +sql use $dbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi + +print == create super table +sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) +sql show stables +if $rows != 1 then + return -1 +endi + +print == create child table, normal table and insert data +$i = 0 +while $i < $ctbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + sql create table $ctb using $stbPrefix tags( $i ) + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) + + $x = 0 + while $x < $rowsPerCtb + $binary = ' . binary- + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , $i , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw diff --git a/tests/system-test/2-query/nestedQuery.py b/tests/system-test/2-query/nestedQuery.py new file mode 100755 index 0000000000000000000000000000000000000000..1f1766f8e5531f542ebd6a04dbf3bb9fb3040529 --- /dev/null +++ b/tests/system-test/2-query/nestedQuery.py @@ -0,0 +1,2284 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import os +import time +import taos +from faker import Faker +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +from util.dnodes import * + +class TDTestCase: + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + testcasePath = os.path.split(__file__)[0] + testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf %s/%s.sql" % (testcasePath,testcaseFilename)) + + now = time.time() + self.ts = int(round(now * 1000)) + self.num = 10 + self.fornum = 5 + + # def case_common(self): + # db = "nested" + # self.dropandcreateDB("%s" % db, 1) + + # conn1 = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos/") + # cur1 = conn1.cursor() + # cur1.execute('use "%s";' %self.db) + # sql = 'select * from stable_1 limit 5;' + # cur1.execute(sql) + + # return(conn1,cur1) + + def restartDnodes(self): + pass + # tdDnodes.stop(1) + # tdDnodes.start(1) + + def dropandcreateDB_random(self,database,n): + ts = 1630000000000 + num_random = 100 + fake = Faker('zh_CN') + tdSql.execute('''drop database if exists %s ;''' %database) + tdSql.execute('''create database %s keep 36500;'''%database) + tdSql.execute('''use %s;'''%database) + + tdSql.execute('''create stable stable_1 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \ + tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);''') + tdSql.execute('''create stable stable_2 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \ + tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);''') + + tdSql.execute('''create stable stable_null_data (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \ + tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);''') + + tdSql.execute('''create stable stable_null_childtable (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \ + tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);''') + + #tdSql.execute('''create table stable_1_1 using stable_1 tags('stable_1_1', '0' , '0' , '0' , '0' , 0 , 'binary1' , 'nchar1' , '0' , '0' ,'0') ;''') + tdSql.execute('''create table stable_1_1 using stable_1 tags('stable_1_1', '%d' , '%d', '%d' , '%d' , 0 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + tdSql.execute('''create table stable_1_2 using stable_1 tags('stable_1_2', '2147483647' , '9223372036854775807' , '32767' , '127' , 1 , 'binary2' , 'nchar2' , '2' , '22' , \'1999-09-09 09:09:09.090\') ;''') + tdSql.execute('''create table stable_1_3 using stable_1 tags('stable_1_3', '-2147483647' , '-9223372036854775807' , '-32767' , '-127' , false , 'binary3' , 'nchar3nchar3' , '-3.3' , '-33.33' , \'2099-09-09 09:09:09.090\') ;''') + #tdSql.execute('''create table stable_1_4 using stable_1 tags('stable_1_4', '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0') ;''') + tdSql.execute('''create table stable_1_4 using stable_1 tags('stable_1_4', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + + # tdSql.execute('''create table stable_2_1 using stable_2 tags('stable_2_1' , '0' , '0' , '0' , '0' , 0 , 'binary21' , 'nchar21' , '0' , '0' ,'0') ;''') + # tdSql.execute('''create table stable_2_2 using stable_2 tags('stable_2_2' , '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0') ;''') + + # tdSql.execute('''create table stable_null_data_1 using stable_null_data tags('stable_null_data_1', '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0') ;''') + + tdSql.execute('''create table stable_2_1 using stable_2 tags('stable_2_1' , '0' , '0' , '0' , '0' , 0 , 'binary21' , 'nchar21' , '0' , '0' ,\'2099-09-09 09:09:09.090\') ;''') + tdSql.execute('''create table stable_2_2 using stable_2 tags('stable_2_2' , '%d' , '%d', '%d' , '%d' , 0 , 'binary2.%s' , 'nchar2.%s' , '%f', '%f' ,'%d') ;''' + %(fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + + tdSql.execute('''create table stable_null_data_1 using stable_null_data tags('stable_null_data_1', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + + #regular table + tdSql.execute('''create table regular_table_1 \ + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) ;''') + tdSql.execute('''create table regular_table_2 \ + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) ;''') + tdSql.execute('''create table regular_table_3 \ + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) ;''') + + tdSql.execute('''create table regular_table_null \ + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) ;''') + + + for i in range(num_random*n): + tdSql.execute('''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d) ;''' + % (ts + i*1000, fake.random_int(min=-2147483647, max=2147483647, step=1), + fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.address() , ts + i)) + tdSql.execute('''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d) ;''' + % (ts + i*1000, fake.random_int(min=-2147483647, max=2147483647, step=1) , + fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1) , + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.address() , ts + i)) + + tdSql.execute('''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(%d, %d, %d, %d, %d, %f, %f, 1, 'binary.%s', 'nchar.%s', %d) ;''' + % (ts + i*1000, fake.random_int(min=0, max=2147483647, step=1), + fake.random_int(min=0, max=9223372036854775807, step=1), + fake.random_int(min=0, max=32767, step=1) , fake.random_int(min=0, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.address() , ts + i)) + tdSql.execute('''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(%d, %d, %d, %d, %d, %f, %f, 1, 'binary.%s', 'nchar.%s', %d) ;''' + % (ts + i*1000, fake.random_int(min=0, max=2147483647, step=1), + fake.random_int(min=0, max=9223372036854775807, step=1), + fake.random_int(min=0, max=32767, step=1) , fake.random_int(min=0, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.address() , ts + i)) + + tdSql.execute('''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(%d, %d, %d, %d, %d, %f, %f, 1, 'binary.%s', 'nchar.%s', %d) ;''' + % (ts + i*1000 +1, fake.random_int(min=-2147483647, max=0, step=1), + fake.random_int(min=-9223372036854775807, max=0, step=1), + fake.random_int(min=-32767, max=0, step=1) , fake.random_int(min=-127, max=0, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.address() , ts + i +1)) + tdSql.execute('''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(%d, %d, %d, %d, %d, %f, %f, 1, 'binary.%s', 'nchar.%s', %d) ;''' + % (ts + i*1000 +1, fake.random_int(min=-2147483647, max=0, step=1), + fake.random_int(min=-9223372036854775807, max=0, step=1), + fake.random_int(min=-32767, max=0, step=1) , fake.random_int(min=-127, max=0, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.address() , ts + i +1)) + + tdSql.execute('''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d) ;''' + % (ts + i*1000, fake.random_int(min=-2147483647, max=2147483647, step=1), + fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.address() , ts + i)) + + # tdSql.execute('''insert into regular_table_3 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(%d, %d, %d, %d, %d, %f, %f, 1, 'binary.%s', 'nchar.%s', %d)''' + # % (ts + i*1000, fake.random_int(min=-2147483647, max=0, step=1), + # fake.random_int(min=-9223372036854775807, max=0, step=1), + # fake.random_int(min=-32767, max=0, step=1) , fake.random_int(min=-127, max=0, step=1) , + # fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.address() , ts + i)) + + tdSql.query("select count(*) from stable_1;") + tdSql.checkData(0,0,3*num_random*n) + tdSql.query("select count(*) from regular_table_1;") + tdSql.checkData(0,0,num_random*n) + + + def run(self): + tdSql.prepare() + os.system("rm -rf nestedQuery3.py.sql") + + startTime = time.time() + + db = "nest" + self.dropandcreateDB_random("%s" %db, 1) + + # regular column select + q_select= ['ts' , '*' , 'q_int', 'q_bigint' , 'q_bigint' , 'q_smallint' , 'q_tinyint' , 'q_bool' , 'q_binary' , 'q_nchar' ,'q_float' , 'q_double' ,'q_ts '] + q_select= ['ts' , 'q_int', 'q_bigint' , 'q_bigint' , 'q_smallint' , 'q_tinyint' , 'q_bool' , 'q_binary' , 'q_nchar' ,'q_float' , 'q_double' ,'q_ts ', 'q_int_null ', 'q_bigint_null ' , 'q_bigint_null ' , 'q_smallint_null ' , 'q_tinyint_null ' , 'q_bool_null ' , 'q_binary_null ' , 'q_nchar_null ' ,'q_float_null ' , 'q_double_null ' ,'q_ts_null '] + + # tag column select + t_select= ['*' , 'loc' ,'t_int', 't_bigint' , 't_bigint' , 't_smallint' , 't_tinyint' , 't_bool' , 't_binary' , 't_nchar' ,'t_float' , 't_double' ,'t_ts '] + t_select= ['loc' ,'tbname','t_int', 't_bigint' , 't_bigint' , 't_smallint' , 't_tinyint' , 't_bool' , 't_binary' , 't_nchar' ,'t_float' , 't_double' ,'t_ts '] + + # regular and tag column select + qt_select= q_select + t_select + + # distinct regular column select + dq_select= ['distinct q_int', 'distinct q_bigint' , 'distinct q_smallint' , 'distinct q_tinyint' , + 'distinct q_bool' , 'distinct q_binary' , 'distinct q_nchar' ,'distinct q_float' , 'distinct q_double' ,'distinct q_ts '] + + # distinct tag column select + dt_select= ['distinct loc', 'distinct t_int', 'distinct t_bigint' , 'distinct t_smallint' , 'distinct t_tinyint' , + 'distinct t_bool' , 'distinct t_binary' , 'distinct t_nchar' ,'distinct t_float' , 'distinct t_double' ,'distinct t_ts '] + + # distinct regular and tag column select + dqt_select= dq_select + dt_select + + # special column select + s_r_select= ['_c0', '_rowts' , '_C0' ] + s_s_select= ['tbname' , '_rowts' , '_c0', '_C0' ] + unionall_or_union= [ ' union ' , ' union all ' ] + + # regular column where + q_where = ['ts < now +1s','q_bigint >= -9223372036854775807 and q_bigint <= 9223372036854775807', 'q_int <= 2147483647 and q_int >= -2147483647', + 'q_smallint >= -32767 and q_smallint <= 32767','q_tinyint >= -127 and q_tinyint <= 127','q_float >= -1.7E308 and q_float <= 1.7E308', + 'q_double >= -1.7E308 and q_double <= 1.7E308', 'q_binary like \'binary%\' or q_binary = \'0\' ' , 'q_nchar like \'nchar%\' or q_nchar = \'0\' ' , + 'q_bool = true or q_bool = false' , 'q_bool in (0 , 1)' , 'q_bool in ( true , false)' , 'q_bool = 0 or q_bool = 1', + 'q_bigint between -9223372036854775807 and 9223372036854775807',' q_int between -2147483647 and 2147483647','q_smallint between -32767 and 32767', + 'q_tinyint between -127 and 127 ','q_float >= -3.4E38 ','q_float <= 3.4E38 ','q_double >= -1.7E308 ', + 'q_double <= 1.7E308 ','q_float between -3.4E38 and 3.4E38 ','q_double between -1.7E308 and 1.7E308 ' , + 'q_float is not null ' ,'q_double is not null ' ,] + #TD-6201 ,'q_bool between 0 and 1' + + # regular column where for test union,join + q_u_where = ['t1.ts < now +1s' , 't2.ts < now +1s','t1.q_bigint >= -9223372036854775807 and t1.q_bigint <= 9223372036854775807 and t2.q_bigint >= -9223372036854775807 and t2.q_bigint <= 9223372036854775807', + 't1.q_int <= 2147483647 and t1.q_int >= -2147483647 and t2.q_int <= 2147483647 and t2.q_int >= -2147483647', + 't1.q_smallint >= -32767 and t1.q_smallint <= 32767 and t2.q_smallint >= -32767 and t2.q_smallint <= 32767', + 't1.q_tinyint >= -127 and t1.q_tinyint <= 127 and t2.q_tinyint >= -127 and t2.q_tinyint <= 127', + 't1.q_float >= - 1.7E308 and t1.q_float <= 1.7E308 and t2.q_float >= - 1.7E308 and t2.q_float <= 1.7E308', + 't1.q_double >= - 1.7E308 and t1.q_double <= 1.7E308 and t2.q_double >= - 1.7E308 and t2.q_double <= 1.7E308', + 't1.q_binary like \'binary%\' and t2.q_binary like \'binary%\' ' , + 't1.q_nchar like \'nchar%\' and t2.q_nchar like \'nchar%\' ' , + 't1.q_bool in (0 , 1) and t2.q_bool in (0 , 1)' , 't1.q_bool in ( true , false) and t2.q_bool in ( true , false)' , + 't1.q_bigint between -9223372036854775807 and 9223372036854775807 and t2.q_bigint between -9223372036854775807 and 9223372036854775807', + 't1.q_int between -2147483647 and 2147483647 and t2.q_int between -2147483647 and 2147483647', + 't1.q_smallint between -32767 and 32767 and t2.q_smallint between -32767 and 32767', + 't1.q_tinyint between -127 and 127 and t2.q_tinyint between -127 and 127 ','t1.q_float between -1.7E308 and 1.7E308 and t2.q_float between -1.7E308 and 1.7E308', + 't1.q_double between -1.7E308 and 1.7E308 and t2.q_double between -1.7E308 and 1.7E308'] + #TD-6201 ,'t1.q_bool between 0 and 1 or t2.q_bool between 0 and 1'] + #'t1.q_bool = true and t1.q_bool = false and t2.q_bool = true and t2.q_bool = false' , 't1.q_bool = 0 and t1.q_bool = 1 and t2.q_bool = 0 and t2.q_bool = 1' , + + q_u_or_where = ['(t1.q_binary like \'binary%\' or t1.q_binary = \'0\' or t2.q_binary like \'binary%\' or t2.q_binary = \'0\' )' , + '(t1.q_nchar like \'nchar%\' or t1.q_nchar = \'0\' or t2.q_nchar like \'nchar%\' or t2.q_nchar = \'0\' )' , '(t1.q_bool = true or t1.q_bool = false or t2.q_bool = true or t2.q_bool = false)' , + '(t1.q_bool in (0 , 1) or t2.q_bool in (0 , 1)' , 't1.q_bool in ( true , false) or t2.q_bool in ( true , false))' , '(t1.q_bool = 0 or t1.q_bool = 1 or t2.q_bool = 0 or t2.q_bool = 1)' , + '(t1.q_bigint between -9223372036854775807 and 9223372036854775807 or t2.q_bigint between -9223372036854775807 and 9223372036854775807)', + '(t1.q_int between -2147483647 and 2147483647 or t2.q_int between -2147483647 and 2147483647)', + '(t1.q_smallint between -32767 and 32767 or t2.q_smallint between -32767 and 32767)', + '(t1.q_tinyint between -127 and 127 or t2.q_tinyint between -127 and 127 )','(t1.q_float between -1.7E308 and 1.7E308 or t2.q_float between -1.7E308 and 1.7E308)', + '(t1.q_double between -1.7E308 and 1.7E308 or t2.q_double between -1.7E308 and 1.7E308)'] + + # tag column where + t_where = ['ts < now +1s','t_bigint >= -9223372036854775807 and t_bigint <= 9223372036854775807','t_int <= 2147483647 and t_int >= -2147483647', + 't_smallint >= -32767 and t_smallint <= 32767','q_tinyint >= -127 and t_tinyint <= 127','t_float >= -1.7E308 and t_float <= 1.7E308', + 't_double >= -1.7E308 and t_double <= 1.7E308', 't_binary like \'binary%\' or t_binary = \'0\' ' , 't_nchar like \'nchar%\' or t_nchar = \'0\'' , + 't_bool = true or t_bool = false' , 't_bool in (0 , 1)' , 't_bool in ( true , false)' , 't_bool = 0 or t_bool = 1', + 't_bigint between -9223372036854775807 and 9223372036854775807',' t_int between -2147483647 and 2147483647','t_smallint between -32767 and 32767', + 't_tinyint between -127 and 127 ','t_float between -1.7E308 and 1.7E308','t_double between -1.7E308 and 1.7E308'] + #TD-6201,'t_bool between 0 and 1' + + # tag column where for test union,join | this is not support + t_u_where = ['t1.ts < now +1s' , 't2.ts < now +1s','t1.t_bigint >= -9223372036854775807 and t1.t_bigint <= 9223372036854775807 and t2.t_bigint >= -9223372036854775807 and t2.t_bigint <= 9223372036854775807', + 't1.t_int <= 2147483647 and t1.t_int >= -2147483647 and t2.t_int <= 2147483647 and t2.t_int >= -2147483647', + 't1.t_smallint >= -32767 and t1.t_smallint <= 32767 and t2.t_smallint >= -32767 and t2.t_smallint <= 32767', + 't1.t_tinyint >= -127 and t1.t_tinyint <= 127 and t2.t_tinyint >= -127 and t2.t_tinyint <= 127', + 't1.t_float >= -1.7E308 and t1.t_float <= 1.7E308 and t2.t_float >= -1.7E308 and t2.t_float <= 1.7E308', + 't1.t_double >= -1.7E308 and t1.t_double <= 1.7E308 and t2.t_double >= -1.7E308 and t2.t_double <= 1.7E308', + '(t1.t_binary like \'binary%\' or t1.t_binary = \'0\' or t2.t_binary like \'binary%\' or t2.t_binary = \'0\') ' , + '(t1.t_nchar like \'nchar%\' or t1.t_nchar = \'0\' or t2.t_nchar like \'nchar%\' or t2.t_nchar = \'0\' )' , '(t1.t_bool = true or t1.t_bool = false or t2.t_bool = true or t2.t_bool = false)' , + 't1.t_bool in (0 , 1) and t2.t_bool in (0 , 1)' , 't1.t_bool in ( true , false) and t2.t_bool in ( true , false)' , '(t1.t_bool = 0 or t1.t_bool = 1 or t2.t_bool = 0 or t2.t_bool = 1)', + 't1.t_bigint between -9223372036854775807 and 9223372036854775807 and t2.t_bigint between -9223372036854775807 and 9223372036854775807', + 't1.t_int between -2147483647 and 2147483647 and t2.t_int between -2147483647 and 2147483647', + 't1.t_smallint between -32767 and 32767 and t2.t_smallint between -32767 and 32767', + '(t1.t_tinyint between -127 and 127 and t2.t_tinyint between -127 and 127) ','t1.t_float between -1.7E308 and 1.7E308 and t2.t_float between -1.7E308 and 1.7E308', + '(t1.t_double between -1.7E308 and 1.7E308 and t2.t_double between -1.7E308 and 1.7E308)'] + #TD-6201,'t1.t_bool between 0 and 1 or t2.q_bool between 0 and 1'] + + t_u_or_where = ['(t1.t_binary like \'binary%\' or t1.t_binary = \'0\' or t2.t_binary like \'binary%\' or t2.t_binary = \'0\' )' , + '(t1.t_nchar like \'nchar%\' or t1.t_nchar = \'0\' or t2.t_nchar like \'nchar%\' or t2.t_nchar = \'0\' )' , '(t1.t_bool = true or t1.t_bool = false or t2.t_bool = true or t2.t_bool = false)' , + '(t1.t_bool in (0 , 1) or t2.t_bool in (0 , 1))' , '(t1.t_bool in ( true , false) or t2.t_bool in ( true , false))' , '(t1.t_bool = 0 or t1.t_bool = 1 or t2.t_bool = 0 or t2.t_bool = 1)', + '(t1.t_bigint between -9223372036854775807 and 9223372036854775807 or t2.t_bigint between -9223372036854775807 and 9223372036854775807)', + '(t1.t_int between -2147483647 and 2147483647 or t2.t_int between -2147483647 and 2147483647)', + '(t1.t_smallint between -32767 and 32767 or t2.t_smallint between -32767 and 32767)', + '(t1.t_tinyint between -127 and 127 or t2.t_tinyint between -127 and 127 )','(t1.t_float between -1.7E308 and 1.7E308 or t2.t_float between -1.7E308 and 1.7E308)', + '(t1.t_double between -1.7E308 and 1.7E308 or t2.t_double between -1.7E308 and 1.7E308)'] + + # regular and tag column where + qt_where = q_where + t_where + qt_u_where = q_u_where + t_u_where + # now,qt_u_or_where is not support + qt_u_or_where = q_u_or_where + t_u_or_where + + # tag column where for test super join | this is support , 't1.t_bool = t2.t_bool ' ??? + t_join_where = ['t1.t_bigint = t2.t_bigint ', 't1.t_int = t2.t_int ', 't1.t_smallint = t2.t_smallint ', 't1.t_tinyint = t2.t_tinyint ', + 't1.t_float = t2.t_float ', 't1.t_double = t2.t_double ', 't1.t_binary = t2.t_binary ' , 't1.t_nchar = t2.t_nchar ' ] + + # session && fill + session_where = ['session(ts,10a)' , 'session(ts,10s)', 'session(ts,10m)' , 'session(ts,10h)','session(ts,10d)' , 'session(ts,10w)'] + session_u_where = ['session(t1.ts,10a)' , 'session(t1.ts,10s)', 'session(t1.ts,10m)' , 'session(t1.ts,10h)','session(t1.ts,10d)' , 'session(t1.ts,10w)', + 'session(t2.ts,10a)' , 'session(t2.ts,10s)', 'session(t2.ts,10m)' , 'session(t2.ts,10h)','session(t2.ts,10d)' , 'session(t2.ts,10w)'] + + fill_where = ['FILL(NONE)','FILL(PREV)','FILL(NULL)','FILL(LINEAR)','FILL(NEXT)','FILL(VALUE, 1.23)'] + + state_window = ['STATE_WINDOW(q_tinyint)','STATE_WINDOW(q_bigint)','STATE_WINDOW(q_int)','STATE_WINDOW(q_bool)','STATE_WINDOW(q_smallint)'] + state_u_window = ['STATE_WINDOW(t1.q_tinyint)','STATE_WINDOW(t1.q_bigint)','STATE_WINDOW(t1.q_int)','STATE_WINDOW(t1.q_bool)','STATE_WINDOW(t1.q_smallint)', + 'STATE_WINDOW(t2.q_tinyint)','STATE_WINDOW(t2.q_bigint)','STATE_WINDOW(t2.q_int)','STATE_WINDOW(t2.q_bool)','STATE_WINDOW(t2.q_smallint)'] + + # order by where + order_where = ['order by ts' , 'order by ts asc'] + order_u_where = ['order by t1.ts' , 'order by t1.ts asc' , 'order by t2.ts' , 'order by t2.ts asc'] + order_desc_where = ['order by ts' , 'order by ts asc' , 'order by ts desc' ] + orders_desc_where = ['order by ts' , 'order by ts asc' , 'order by ts desc' , 'order by loc' , 'order by loc asc' , 'order by loc desc'] + + group_where = ['group by tbname , loc' , 'group by tbname', 'group by tbname, t_bigint', 'group by tbname,t_int', 'group by tbname, t_smallint', 'group by tbname,t_tinyint', + 'group by tbname,t_float', 'group by tbname,t_double' , 'group by tbname,t_binary', 'group by tbname,t_nchar', 'group by tbname,t_bool' ,'group by tbname ,loc ,t_bigint', + 'group by tbname,t_binary ,t_nchar ,t_bool' , 'group by tbname,t_int ,t_smallint ,t_tinyint' , 'group by tbname,t_float ,t_double ' ] + having_support = ['having count(q_int) > 0','having count(q_bigint) > 0','having count(q_smallint) > 0','having count(q_tinyint) > 0','having count(q_float) > 0','having count(q_double) > 0','having count(q_bool) > 0', + 'having avg(q_int) > 0','having avg(q_bigint) > 0','having avg(q_smallint) > 0','having avg(q_tinyint) > 0','having avg(q_float) > 0','having avg(q_double) > 0', + 'having sum(q_int) > 0','having sum(q_bigint) > 0','having sum(q_smallint) > 0','having sum(q_tinyint) > 0','having sum(q_float) > 0','having sum(q_double) > 0', + 'having STDDEV(q_int) > 0','having STDDEV(q_bigint) > 0','having STDDEV(q_smallint) > 0','having STDDEV(q_tinyint) > 0','having STDDEV(q_float) > 0','having STDDEV(q_double) > 0', + 'having TWA(q_int) > 0','having TWA(q_bigint) > 0','having TWA(q_smallint) > 0','having TWA(q_tinyint) > 0','having TWA(q_float) > 0','having TWA(q_double) > 0', + 'having IRATE(q_int) > 0','having IRATE(q_bigint) > 0','having IRATE(q_smallint) > 0','having IRATE(q_tinyint) > 0','having IRATE(q_float) > 0','having IRATE(q_double) > 0', + 'having MIN(q_int) > 0','having MIN(q_bigint) > 0','having MIN(q_smallint) > 0','having MIN(q_tinyint) > 0','having MIN(q_float) > 0','having MIN(q_double) > 0', + 'having MAX(q_int) > 0','having MAX(q_bigint) > 0','having MAX(q_smallint) > 0','having MAX(q_tinyint) > 0','having MAX(q_float) > 0','having MAX(q_double) > 0', + 'having FIRST(q_int) > 0','having FIRST(q_bigint) > 0','having FIRST(q_smallint) > 0','having FIRST(q_tinyint) > 0','having FIRST(q_float) > 0','having FIRST(q_double) > 0', + 'having LAST(q_int) > 0','having LAST(q_bigint) > 0','having LAST(q_smallint) > 0','having LAST(q_tinyint) > 0','having LAST(q_float) > 0','having LAST(q_double) > 0', + 'having APERCENTILE(q_int,10) > 0','having APERCENTILE(q_bigint,10) > 0','having APERCENTILE(q_smallint,10) > 0','having APERCENTILE(q_tinyint,10) > 0','having APERCENTILE(q_float,10) > 0','having APERCENTILE(q_double,10) > 0'] + having_not_support = ['having TOP(q_int,10) > 0','having TOP(q_bigint,10) > 0','having TOP(q_smallint,10) > 0','having TOP(q_tinyint,10) > 0','having TOP(q_float,10) > 0','having TOP(q_double,10) > 0','having TOP(q_bool,10) > 0', + 'having BOTTOM(q_int,10) > 0','having BOTTOM(q_bigint,10) > 0','having BOTTOM(q_smallint,10) > 0','having BOTTOM(q_tinyint,10) > 0','having BOTTOM(q_float,10) > 0','having BOTTOM(q_double,10) > 0','having BOTTOM(q_bool,10) > 0', + 'having LEASTSQUARES(q_int) > 0','having LEASTSQUARES(q_bigint) > 0','having LEASTSQUARES(q_smallint) > 0','having LEASTSQUARES(q_tinyint) > 0','having LEASTSQUARES(q_float) > 0','having LEASTSQUARES(q_double) > 0','having LEASTSQUARES(q_bool) > 0', + 'having FIRST(q_bool) > 0','having IRATE(q_bool) > 0','having PERCENTILE(q_bool,10) > 0','having avg(q_bool) > 0','having LAST_ROW(q_bool) > 0','having sum(q_bool) > 0','having STDDEV(q_bool) > 0','having APERCENTILE(q_bool,10) > 0','having TWA(q_bool) > 0','having LAST(q_bool) > 0', + 'having PERCENTILE(q_int,10) > 0','having PERCENTILE(q_bigint,10) > 0','having PERCENTILE(q_smallint,10) > 0','having PERCENTILE(q_tinyint,10) > 0','having PERCENTILE(q_float,10) > 0','having PERCENTILE(q_double,10) > 0'] + having_tagnot_support = ['having LAST_ROW(q_int) > 0','having LAST_ROW(q_bigint) > 0','having LAST_ROW(q_smallint) > 0','having LAST_ROW(q_tinyint) > 0','having LAST_ROW(q_float) > 0','having LAST_ROW(q_double) > 0'] + + # limit offset where + limit_where = ['limit 1 offset 1' , 'limit 1' , 'limit 2 offset 1' , 'limit 2', 'limit 12 offset 1' , 'limit 20', 'limit 20 offset 10' , 'limit 200'] + limit1_where = ['limit 1 offset 1' , 'limit 1' ] + limit_u_where = ['limit 100 offset 10' , 'limit 50' , 'limit 100' , 'limit 10' ] + + # slimit soffset where + slimit_where = ['slimit 1 soffset 1' , 'slimit 1' , 'slimit 2 soffset 1' , 'slimit 2'] + slimit1_where = ['slimit 2 soffset 1' , 'slimit 1' ] + + # aggregate function include [all:count(*)\avg\sum\stddev ||regualr:twa\irate\leastsquares ||group by tbname:twa\irate\] + # select function include [all: min\max\first(*)\last(*)\top\bottom\apercentile\last_row(*)(not with interval)\interp(*)(FILL) ||regualr: percentile] + # calculation function include [all:spread\+-*/ ||regualr:diff\derivative ||group by tbname:diff\derivative\] + # **_ns_** express is not support stable, therefore, separated from regular tables + # calc_select_all calc_select_regular calc_select_in_ts calc_select_fill calc_select_not_interval + # calc_aggregate_all calc_aggregate_regular calc_aggregate_groupbytbname + # calc_calculate_all calc_calculate_regular calc_calculate_groupbytbname + + # calc_select_all calc_select_regular calc_select_in_ts calc_select_fill calc_select_not_interval + # select function include [all: min\max\first(*)\last(*)\top\bottom\apercentile\last_row(*)(not with interval)\interp(*)(FILL) ||regualr: percentile] + + calc_select_all = ['bottom(q_int,20)' , 'bottom(q_bigint,20)' , 'bottom(q_smallint,20)' , 'bottom(q_tinyint,20)' ,'bottom(q_float,20)' , 'bottom(q_double,20)' , + 'top(q_int,20)' , 'top(q_bigint,20)' , 'top(q_smallint,20)' ,'top(q_tinyint,20)' ,'top(q_float,20)' ,'top(q_double,20)' , + 'first(q_int)' , 'first(q_bigint)' , 'first(q_smallint)' , 'first(q_tinyint)' , 'first(q_float)' ,'first(q_double)' ,'first(q_binary)' ,'first(q_nchar)' ,'first(q_bool)' ,'first(q_ts)' , + 'last(q_int)' , 'last(q_bigint)' , 'last(q_smallint)' , 'last(q_tinyint)' , 'last(q_float)' ,'last(q_double)' , 'last(q_binary)' ,'last(q_nchar)' ,'last(q_bool)' ,'last(q_ts)' , + 'min(q_int)' , 'min(q_bigint)' , 'min(q_smallint)' , 'min(q_tinyint)' , 'min(q_float)' ,'min(q_double)' , + 'max(q_int)' , 'max(q_bigint)' , 'max(q_smallint)' , 'max(q_tinyint)' ,'max(q_float)' ,'max(q_double)' , + 'apercentile(q_int,20)' , 'apercentile(q_bigint,20)' ,'apercentile(q_smallint,20)' ,'apercentile(q_tinyint,20)' ,'apercentile(q_float,20)' ,'apercentile(q_double,20)' , + 'last_row(q_int)' , 'last_row(q_bigint)' , 'last_row(q_smallint)' , 'last_row(q_tinyint)' , 'last_row(q_float)' , + 'last_row(q_double)' , 'last_row(q_bool)' ,'last_row(q_binary)' ,'last_row(q_nchar)' ,'last_row(q_ts)'] + + calc_select_in_ts = ['bottom(q_int,20)' , 'bottom(q_bigint,20)' , 'bottom(q_smallint,20)' , 'bottom(q_tinyint,20)' ,'bottom(q_float,20)' , 'bottom(q_double,20)' , + 'top(q_int,20)' , 'top(q_bigint,20)' , 'top(q_smallint,20)' ,'top(q_tinyint,20)' ,'top(q_float,20)' ,'top(q_double,20)' , + 'first(q_int)' , 'first(q_bigint)' , 'first(q_smallint)' , 'first(q_tinyint)' , 'first(q_float)' ,'first(q_double)' ,'first(q_binary)' ,'first(q_nchar)' ,'first(q_bool)' ,'first(q_ts)' , + 'last(q_int)' , 'last(q_bigint)' , 'last(q_smallint)' , 'last(q_tinyint)' , 'last(q_float)' ,'last(q_double)' , 'last(q_binary)' ,'last(q_nchar)' ,'last(q_bool)' ,'last(q_ts)' ] + + calc_select_in = ['min(q_int)' , 'min(q_bigint)' , 'min(q_smallint)' , 'min(q_tinyint)' , 'min(q_float)' ,'min(q_double)' , + 'max(q_int)' , 'max(q_bigint)' , 'max(q_smallint)' , 'max(q_tinyint)' ,'max(q_float)' ,'max(q_double)' , + 'apercentile(q_int,20)' , 'apercentile(q_bigint,20)' ,'apercentile(q_smallint,20)' ,'apercentile(q_tinyint,20)' ,'apercentile(q_float,20)' ,'apercentile(q_double,20)' , + 'last_row(q_int)' , 'last_row(q_bigint)' , 'last_row(q_smallint)' , 'last_row(q_tinyint)' , 'last_row(q_float)' , + 'last_row(q_double)' , 'last_row(q_bool)' ,'last_row(q_binary)' ,'last_row(q_nchar)' ,'last_row(q_ts)'] + + + calc_select_regular = [ 'PERCENTILE(q_int,10)' ,'PERCENTILE(q_bigint,20)' , 'PERCENTILE(q_smallint,30)' ,'PERCENTILE(q_tinyint,40)' ,'PERCENTILE(q_float,50)' ,'PERCENTILE(q_double,60)'] + + + calc_select_fill = ['INTERP(q_int)' ,'INTERP(q_bigint)' ,'INTERP(q_smallint)' ,'INTERP(q_tinyint)', 'INTERP(q_float)' ,'INTERP(q_double)'] + interp_where = ['ts = now' , 'ts = \'2020-09-13 20:26:40.000\'' , 'ts = \'2020-09-13 20:26:40.009\'' ,'tbname in (\'table_1\') and ts = now' ,'tbname in (\'table_0\' ,\'table_1\',\'table_2\',\'table_3\',\'table_4\',\'table_5\') and ts = \'2020-09-13 20:26:40.000\'','tbname like \'table%\' and ts = \'2020-09-13 20:26:40.002\''] + + #two table join + calc_select_in_ts_j = ['bottom(t1.q_int,20)' , 'bottom(t1.q_bigint,20)' , 'bottom(t1.q_smallint,20)' , 'bottom(t1.q_tinyint,20)' ,'bottom(t1.q_float,20)' , 'bottom(t1.q_double,20)' , + 'top(t1.q_int,20)' , 'top(t1.q_bigint,20)' , 'top(t1.q_smallint,20)' ,'top(t1.q_tinyint,20)' ,'top(t1.q_float,20)' ,'top(t1.q_double,20)' , + 'first(t1.q_int)' , 'first(t1.q_bigint)' , 'first(t1.q_smallint)' , 'first(t1.q_tinyint)' , 'first(t1.q_float)' ,'first(t1.q_double)' ,'first(t1.q_binary)' ,'first(t1.q_nchar)' ,'first(t1.q_bool)' ,'first(t1.q_ts)' , + 'last(t1.q_int)' , 'last(t1.q_bigint)' , 'last(t1.q_smallint)' , 'last(t1.q_tinyint)' , 'last(t1.q_float)' ,'last(t1.q_double)' , 'last(t1.q_binary)' ,'last(t1.q_nchar)' ,'last(t1.q_bool)' ,'last(t1.q_ts)' , + 'bottom(t2.q_int,20)' , 'bottom(t2.q_bigint,20)' , 'bottom(t2.q_smallint,20)' , 'bottom(t2.q_tinyint,20)' ,'bottom(t2.q_float,20)' , 'bottom(t2.q_double,20)' , + 'top(t2.q_int,20)' , 'top(t2.q_bigint,20)' , 'top(t2.q_smallint,20)' ,'top(t2.q_tinyint,20)' ,'top(t2.q_float,20)' ,'top(t2.q_double,20)' , + 'first(t2.q_int)' , 'first(t2.q_bigint)' , 'first(t2.q_smallint)' , 'first(t2.q_tinyint)' , 'first(t2.q_float)' ,'first(t2.q_double)' ,'first(t2.q_binary)' ,'first(t2.q_nchar)' ,'first(t2.q_bool)' ,'first(t2.q_ts)' , + 'last(t2.q_int)' , 'last(t2.q_bigint)' , 'last(t2.q_smallint)' , 'last(t2.q_tinyint)' , 'last(t2.q_float)' ,'last(t2.q_double)' , 'last(t2.q_binary)' ,'last(t2.q_nchar)' ,'last(t2.q_bool)' ,'last(t2.q_ts)'] + + calc_select_in_j = ['min(t1.q_int)' , 'min(t1.q_bigint)' , 'min(t1.q_smallint)' , 'min(t1.q_tinyint)' , 'min(t1.q_float)' ,'min(t1.q_double)' , + 'max(t1.q_int)' , 'max(t1.q_bigint)' , 'max(t1.q_smallint)' , 'max(t1.q_tinyint)' ,'max(t1.q_float)' ,'max(t1.q_double)' , + 'apercentile(t1.q_int,20)' , 'apercentile(t1.q_bigint,20)' ,'apercentile(t1.q_smallint,20)' ,'apercentile(t1.q_tinyint,20)' ,'apercentile(t1.q_float,20)' ,'apercentile(t1.q_double,20)' , + 'last_row(t1.q_int)' , 'last_row(t1.q_bigint)' , 'last_row(t1.q_smallint)' , 'last_row(t1.q_tinyint)' , 'last_row(t1.q_float)' , + 'last_row(t1.q_double)' , 'last_row(t1.q_bool)' ,'last_row(t1.q_binary)' ,'last_row(t1.q_nchar)' ,'last_row(t1.q_ts)' , + 'min(t2.q_int)' , 'min(t2.q_bigint)' , 'min(t2.q_smallint)' , 'min(t2.q_tinyint)' , 'min(t2.q_float)' ,'min(t2.q_double)' , + 'max(t2.q_int)' , 'max(t2.q_bigint)' , 'max(t2.q_smallint)' , 'max(t2.q_tinyint)' ,'max(t2.q_float)' ,'max(t2.q_double)' , + 'apercentile(t2.q_int,20)' , 'apercentile(t2.q_bigint,20)' ,'apercentile(t2.q_smallint,20)' ,'apercentile(t2.q_tinyint,20)' ,'apercentile(t2.q_float,20)' ,'apercentile(t2.q_double,20)' , + 'last_row(t2.q_int)' , 'last_row(t2.q_bigint)' , 'last_row(t2.q_smallint)' , 'last_row(t2.q_tinyint)' , 'last_row(t2.q_float)' , + 'last_row(t2.q_double)' , 'last_row(t2.q_bool)' ,'last_row(t2.q_binary)' ,'last_row(t2.q_nchar)' ,'last_row(t2.q_ts)'] + + calc_select_all_j = calc_select_in_ts_j + calc_select_in_j + + calc_select_regular_j = [ 'PERCENTILE(t1.q_int,10)' ,'PERCENTILE(t1.q_bigint,20)' , 'PERCENTILE(t1.q_smallint,30)' ,'PERCENTILE(t1.q_tinyint,40)' ,'PERCENTILE(t1.q_float,50)' ,'PERCENTILE(t1.q_double,60)' , + 'PERCENTILE(t2.q_int,10)' ,'PERCENTILE(t2.q_bigint,20)' , 'PERCENTILE(t2.q_smallint,30)' ,'PERCENTILE(t2.q_tinyint,40)' ,'PERCENTILE(t2.q_float,50)' ,'PERCENTILE(t2.q_double,60)'] + + + calc_select_fill_j = ['INTERP(t1.q_int)' ,'INTERP(t1.q_bigint)' ,'INTERP(t1.q_smallint)' ,'INTERP(t1.q_tinyint)', 'INTERP(t1.q_float)' ,'INTERP(t1.q_double)' , + 'INTERP(t2.q_int)' ,'INTERP(t2.q_bigint)' ,'INTERP(t2.q_smallint)' ,'INTERP(t2.q_tinyint)', 'INTERP(t2.q_float)' ,'INTERP(t2.q_double)'] + interp_where_j = ['t1.ts = now' , 't1.ts = \'2020-09-13 20:26:40.000\'' , 't1.ts = \'2020-09-13 20:26:40.009\'' ,'t2.ts = now' , 't2.ts = \'2020-09-13 20:26:40.000\'' , 't2.ts = \'2020-09-13 20:26:40.009\'' , + 't1.tbname in (\'table_1\') and t1.ts = now' ,'t1.tbname in (\'table_0\' ,\'table_1\',\'table_2\',\'table_3\',\'table_4\',\'table_5\') and t1.ts = \'2020-09-13 20:26:40.000\'','t1.tbname like \'table%\' and t1.ts = \'2020-09-13 20:26:40.002\'', + 't2.tbname in (\'table_1\') and t2.ts = now' ,'t2.tbname in (\'table_0\' ,\'table_1\',\'table_2\',\'table_3\',\'table_4\',\'table_5\') and t2.ts = \'2020-09-13 20:26:40.000\'','t2.tbname like \'table%\' and t2.ts = \'2020-09-13 20:26:40.002\''] + + # calc_aggregate_all calc_aggregate_regular calc_aggregate_groupbytbname APERCENTILE\PERCENTILE + # aggregate function include [all:count(*)\avg\sum\stddev ||regualr:twa\irate\leastsquares ||group by tbname:twa\irate\] + calc_aggregate_all = ['count(*)' , 'count(q_int)' ,'count(q_bigint)' , 'count(q_smallint)' ,'count(q_tinyint)' ,'count(q_float)' , + 'count(q_double)' ,'count(q_binary)' ,'count(q_nchar)' ,'count(q_bool)' ,'count(q_ts)' , + 'avg(q_int)' ,'avg(q_bigint)' , 'avg(q_smallint)' ,'avg(q_tinyint)' ,'avg(q_float)' ,'avg(q_double)' , + 'sum(q_int)' ,'sum(q_bigint)' , 'sum(q_smallint)' ,'sum(q_tinyint)' ,'sum(q_float)' ,'sum(q_double)' , + 'STDDEV(q_int)' ,'STDDEV(q_bigint)' , 'STDDEV(q_smallint)' ,'STDDEV(q_tinyint)' ,'STDDEV(q_float)' ,'STDDEV(q_double)', + 'APERCENTILE(q_int,10)' ,'APERCENTILE(q_bigint,20)' , 'APERCENTILE(q_smallint,30)' ,'APERCENTILE(q_tinyint,40)' ,'APERCENTILE(q_float,50)' ,'APERCENTILE(q_double,60)'] + + calc_aggregate_regular = ['twa(q_int)' ,'twa(q_bigint)' , 'twa(q_smallint)' ,'twa(q_tinyint)' ,'twa (q_float)' ,'twa(q_double)' , + 'IRATE(q_int)' ,'IRATE(q_bigint)' , 'IRATE(q_smallint)' ,'IRATE(q_tinyint)' ,'IRATE (q_float)' ,'IRATE(q_double)' , + 'LEASTSQUARES(q_int,15,3)' , 'LEASTSQUARES(q_bigint,10,1)' , 'LEASTSQUARES(q_smallint,20,3)' ,'LEASTSQUARES(q_tinyint,10,4)' ,'LEASTSQUARES(q_float,6,4)' ,'LEASTSQUARES(q_double,3,1)' , + 'PERCENTILE(q_int,10)' ,'PERCENTILE(q_bigint,20)' , 'PERCENTILE(q_smallint,30)' ,'PERCENTILE(q_tinyint,40)' ,'PERCENTILE(q_float,50)' ,'PERCENTILE(q_double,60)'] + + calc_aggregate_groupbytbname = ['twa(q_int)' ,'twa(q_bigint)' , 'twa(q_smallint)' ,'twa(q_tinyint)' ,'twa (q_float)' ,'twa(q_double)' , + 'IRATE(q_int)' ,'IRATE(q_bigint)' , 'IRATE(q_smallint)' ,'IRATE(q_tinyint)' ,'IRATE (q_float)' ,'IRATE(q_double)' ] + + #two table join + calc_aggregate_all_j = ['count(t1.*)' , 'count(t1.q_int)' ,'count(t1.q_bigint)' , 'count(t1.q_smallint)' ,'count(t1.q_tinyint)' ,'count(t1.q_float)' , + 'count(t1.q_double)' ,'count(t1.q_binary)' ,'count(t1.q_nchar)' ,'count(t1.q_bool)' ,'count(t1.q_ts)' , + 'avg(t1.q_int)' ,'avg(t1.q_bigint)' , 'avg(t1.q_smallint)' ,'avg(t1.q_tinyint)' ,'avg(t1.q_float)' ,'avg(t1.q_double)' , + 'sum(t1.q_int)' ,'sum(t1.q_bigint)' , 'sum(t1.q_smallint)' ,'sum(t1.q_tinyint)' ,'sum(t1.q_float)' ,'sum(t1.q_double)' , + 'STDDEV(t1.q_int)' ,'STDDEV(t1.q_bigint)' , 'STDDEV(t1.q_smallint)' ,'STDDEV(t1.q_tinyint)' ,'STDDEV(t1.q_float)' ,'STDDEV(t1.q_double)', + 'APERCENTILE(t1.q_int,10)' ,'APERCENTILE(t1.q_bigint,20)' , 'APERCENTILE(t1.q_smallint,30)' ,'APERCENTILE(t1.q_tinyint,40)' ,'APERCENTILE(t1.q_float,50)' ,'APERCENTILE(t1.q_double,60)' , + 'count(t2.*)' , 'count(t2.q_int)' ,'count(t2.q_bigint)' , 'count(t2.q_smallint)' ,'count(t2.q_tinyint)' ,'count(t2.q_float)' , + 'count(t2.q_double)' ,'count(t2.q_binary)' ,'count(t2.q_nchar)' ,'count(t2.q_bool)' ,'count(t2.q_ts)' , + 'avg(t2.q_int)' ,'avg(t2.q_bigint)' , 'avg(t2.q_smallint)' ,'avg(t2.q_tinyint)' ,'avg(t2.q_float)' ,'avg(t2.q_double)' , + 'sum(t2.q_int)' ,'sum(t2.q_bigint)' , 'sum(t2.q_smallint)' ,'sum(t2.q_tinyint)' ,'sum(t2.q_float)' ,'sum(t2.q_double)' , + 'STDDEV(t2.q_int)' ,'STDDEV(t2.q_bigint)' , 'STDDEV(t2.q_smallint)' ,'STDDEV(t2.q_tinyint)' ,'STDDEV(t2.q_float)' ,'STDDEV(t2.q_double)', + 'APERCENTILE(t2.q_int,10)' ,'APERCENTILE(t2.q_bigint,20)' , 'APERCENTILE(t2.q_smallint,30)' ,'APERCENTILE(t2.q_tinyint,40)' ,'APERCENTILE(t2.q_float,50)' ,'APERCENTILE(t2.q_double,60)'] + + calc_aggregate_regular_j = ['twa(t1.q_int)' ,'twa(t1.q_bigint)' , 'twa(t1.q_smallint)' ,'twa(t1.q_tinyint)' ,'twa (t1.q_float)' ,'twa(t1.q_double)' , + 'IRATE(t1.q_int)' ,'IRATE(t1.q_bigint)' , 'IRATE(t1.q_smallint)' ,'IRATE(t1.q_tinyint)' ,'IRATE (t1.q_float)' ,'IRATE(t1.q_double)' , + 'LEASTSQUARES(t1.q_int,15,3)' , 'LEASTSQUARES(t1.q_bigint,10,1)' , 'LEASTSQUARES(t1.q_smallint,20,3)' ,'LEASTSQUARES(t1.q_tinyint,10,4)' ,'LEASTSQUARES(t1.q_float,6,4)' ,'LEASTSQUARES(t1.q_double,3,1)' , + 'PERCENTILE(t1.q_int,10)' ,'PERCENTILE(t1.q_bigint,20)' , 'PERCENTILE(t1.q_smallint,30)' ,'PERCENTILE(t1.q_tinyint,40)' ,'PERCENTILE(t1.q_float,50)' ,'PERCENTILE(t1.q_double,60)' , + 'twa(t2.q_int)' ,'twa(t2.q_bigint)' , 'twa(t2.q_smallint)' ,'twa(t2.q_tinyint)' ,'twa (t2.q_float)' ,'twa(t2.q_double)' , + 'IRATE(t2.q_int)' ,'IRATE(t2.q_bigint)' , 'IRATE(t2.q_smallint)' ,'IRATE(t2.q_tinyint)' ,'IRATE (t2.q_float)' ,'IRATE(t2.q_double)', + 'LEASTSQUARES(t2.q_int,15,3)' , 'LEASTSQUARES(t2.q_bigint,10,1)' , 'LEASTSQUARES(t2.q_smallint,20,3)' ,'LEASTSQUARES(t2.q_tinyint,10,4)' ,'LEASTSQUARES(t2.q_float,6,4)' ,'LEASTSQUARES(t2.q_double,3,1)' , + 'PERCENTILE(t2.q_int,10)' ,'PERCENTILE(t2.q_bigint,20)' , 'PERCENTILE(t2.q_smallint,30)' ,'PERCENTILE(t2.q_tinyint,40)' ,'PERCENTILE(t2.q_float,50)' ,'PERCENTILE(t2.q_double,60)'] + + calc_aggregate_groupbytbname_j = ['twa(t1.q_int)' ,'twa(t1.q_bigint)' , 'twa(t1.q_smallint)' ,'twa(t1.q_tinyint)' ,'twa (t1.q_float)' ,'twa(t1.q_double)' , + 'IRATE(t1.q_int)' ,'IRATE(t1.q_bigint)' , 'IRATE(t1.q_smallint)' ,'IRATE(t1.q_tinyint)' ,'IRATE (t1.q_float)' ,'IRATE(t1.q_double)' , + 'twa(t2.q_int)' ,'twa(t2.q_bigint)' , 'twa(t2.q_smallint)' ,'twa(t2.q_tinyint)' ,'twa (t2.q_float)' ,'twa(t2.q_double)' , + 'IRATE(t2.q_int)' ,'IRATE(t2.q_bigint)' , 'IRATE(t2.q_smallint)' ,'IRATE(t2.q_tinyint)' ,'IRATE (t2.q_float)' ,'IRATE(t2.q_double)' ] + + # calc_calculate_all calc_calculate_regular calc_calculate_groupbytbname + # calculation function include [all:spread\+-*/ ||regualr:diff\derivative ||group by tbname:diff\derivative\] + calc_calculate_all = ['SPREAD(ts)' , 'SPREAD(q_ts)' , 'SPREAD(q_int)' ,'SPREAD(q_bigint)' , 'SPREAD(q_smallint)' ,'SPREAD(q_tinyint)' ,'SPREAD(q_float)' ,'SPREAD(q_double)' , + '(SPREAD(q_int) + SPREAD(q_bigint))' , '(SPREAD(q_smallint) - SPREAD(q_float))', '(SPREAD(q_double) * SPREAD(q_tinyint))' , '(SPREAD(q_double) / SPREAD(q_float))'] + calc_calculate_regular = ['DIFF(q_int)' ,'DIFF(q_bigint)' , 'DIFF(q_smallint)' ,'DIFF(q_tinyint)' ,'DIFF(q_float)' ,'DIFF(q_double)' , + 'DERIVATIVE(q_int,15s,0)' , 'DERIVATIVE(q_bigint,10s,1)' , 'DERIVATIVE(q_smallint,20s,0)' ,'DERIVATIVE(q_tinyint,10s,1)' ,'DERIVATIVE(q_float,6s,0)' ,'DERIVATIVE(q_double,3s,1)' ] + calc_calculate_groupbytbname = calc_calculate_regular + + #two table join + calc_calculate_all_j = ['SPREAD(t1.ts)' , 'SPREAD(t1.q_ts)' , 'SPREAD(t1.q_int)' ,'SPREAD(t1.q_bigint)' , 'SPREAD(t1.q_smallint)' ,'SPREAD(t1.q_tinyint)' ,'SPREAD(t1.q_float)' ,'SPREAD(t1.q_double)' , + 'SPREAD(t2.ts)' , 'SPREAD(t2.q_ts)' , 'SPREAD(t2.q_int)' ,'SPREAD(t2.q_bigint)' , 'SPREAD(t2.q_smallint)' ,'SPREAD(t2.q_tinyint)' ,'SPREAD(t2.q_float)' ,'SPREAD(t2.q_double)' , + '(SPREAD(t1.q_int) + SPREAD(t1.q_bigint))' , '(SPREAD(t1.q_tinyint) - SPREAD(t1.q_float))', '(SPREAD(t1.q_double) * SPREAD(t1.q_tinyint))' , '(SPREAD(t1.q_double) / SPREAD(t1.q_tinyint))', + '(SPREAD(t2.q_int) + SPREAD(t2.q_bigint))' , '(SPREAD(t2.q_smallint) - SPREAD(t2.q_float))', '(SPREAD(t2.q_double) * SPREAD(t2.q_tinyint))' , '(SPREAD(t2.q_double) / SPREAD(t2.q_tinyint))', + '(SPREAD(t1.q_int) + SPREAD(t1.q_smallint))' , '(SPREAD(t2.q_smallint) - SPREAD(t2.q_float))', '(SPREAD(t1.q_double) * SPREAD(t1.q_tinyint))' , '(SPREAD(t1.q_double) / SPREAD(t1.q_float))'] + calc_calculate_regular_j = ['DIFF(t1.q_int)' ,'DIFF(t1.q_bigint)' , 'DIFF(t1.q_smallint)' ,'DIFF(t1.q_tinyint)' ,'DIFF(t1.q_float)' ,'DIFF(t1.q_double)' , + 'DERIVATIVE(t1.q_int,15s,0)' , 'DERIVATIVE(t1.q_bigint,10s,1)' , 'DERIVATIVE(t1.q_smallint,20s,0)' ,'DERIVATIVE(t1.q_tinyint,10s,1)' ,'DERIVATIVE(t1.q_float,6s,0)' ,'DERIVATIVE(t1.q_double,3s,1)' , + 'DIFF(t2.q_int)' ,'DIFF(t2.q_bigint)' , 'DIFF(t2.q_smallint)' ,'DIFF(t2.q_tinyint)' ,'DIFF(t2.q_float)' ,'DIFF(t2.q_double)' , + 'DERIVATIVE(t2.q_int,15s,0)' , 'DERIVATIVE(t2.q_bigint,10s,1)' , 'DERIVATIVE(t2.q_smallint,20s,0)' ,'DERIVATIVE(t2.q_tinyint,10s,1)' ,'DERIVATIVE(t2.q_float,6s,0)' ,'DERIVATIVE(t2.q_double,3s,1)' ] + calc_calculate_groupbytbname_j = calc_calculate_regular_j + + #inter && calc_aggregate_all\calc_aggregate_regular\calc_select_all + interval_sliding = ['interval(4w) sliding(1w) ','interval(1w) sliding(1d) ','interval(1d) sliding(1h) ' , + 'interval(1h) sliding(1m) ','interval(1m) sliding(1s) ','interval(1s) sliding(10a) ', + 'interval(1y) ','interval(1n) ','interval(1w) ','interval(1d) ','interval(1h) ','interval(1m) ','interval(1s) ' ,'interval(10a)', + 'interval(1y,1n) ','interval(1n,1w) ','interval(1w,1d) ','interval(1d,1h) ','interval(1h,1m) ','interval(1m,1s) ','interval(1s,10a) ' ,'interval(100a,30a)'] + + #1 select * from (select column form regular_table where <\>\in\and\or order by) + tdSql.query("select 1-1 from stable_1;") + for i in range(self.fornum): + #sql = "select ts , * from ( select " ===暂时不支持select * ,用下面这一行 + sql = "select ts from ( select " + sql += "%s, " % random.choice(s_s_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += ");" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + tdSql.checkRows(100) + + #1 outer union not support + self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 1-2 from stable_1;") + for i in range(self.fornum): + #sql = "select ts , * from ( select " + sql = "select ts from ( select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + #sql += "%s, " % q_select[len(q_select) -i-1] + sql += "ts from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += ") union " + #sql += "select ts , * from ( select " + sql += "select ts from ( select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += ")" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + tdSql.checkRows(100) + + self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 1-2 from stable_1;") + for i in range(self.fornum): + #sql = "select ts , * from ( select " + sql = "select ts from ( select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + #sql += "%s, " % q_select[len(q_select) -i-1] + sql += "ts from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += ") union all " + #sql += "select ts , * from ( select " + sql += "select ts from ( select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += ")" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + tdSql.checkRows(200) + + #1 inter union not support + tdSql.query("select 1-3 from stable_1;") + for i in range(self.fornum): + #sql = "select ts , * from ( select " + sql = "select ts from ( select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "" + sql += " union select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from regular_table_2 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += ")" + tdLog.info(sql) + tdLog.info(len(sql)) + #TD-15606 tdSql.query(sql) + # tdSql.checkRows(200) + tdSql.query("select 1-3 from stable_1;") + for i in range(self.fornum): + #sql = "select ts , * from ( select " + sql = "select ts from ( select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += " union all select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from regular_table_2 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += ")" + tdLog.info(sql) + tdLog.info(len(sql)) + #TD-15607 tdSql.query(sql) + # tdSql.checkRows(300) + + #join:TD-6020\TD-6149 select * from (select column form regular_table1,regular_table2 where t1.ts=t2.ts and <\>\in\and\or order by) + #self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 1-4 from stable_1;") + for i in range(self.fornum): + #sql = "select ts , * from ( select t1.ts ," + sql = "select * from ( select t1.ts ," + sql += "t1.%s, " % random.choice(q_select) + sql += "t1.%s, " % random.choice(q_select) + sql += "t2.%s, " % random.choice(q_select) + sql += "t2.%s, " % random.choice(q_select) + sql += "t2.ts from regular_table_1 t1 , regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_where) + sql += "%s " % random.choice(order_u_where) + sql += ");" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + tdSql.checkRows(100) + + tdSql.query("select 1-5 from stable_1;") + for i in range(self.fornum): + sql = "select ts , * from ( select t1.ts ," + sql += "t1.%s, " % random.choice(q_select) + sql += "t1.%s, " % random.choice(q_select) + sql += "t2.%s, " % random.choice(q_select) + sql += "t2.%s, " % random.choice(q_select) + sql += "t2.ts from regular_table_1 t1 , regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += ");" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + # TD-15587 tdSql.query(sql) + # tdSql.checkRows(100) + + #2 select column from (select * form regular_table ) where <\>\in\and\or order by + #self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 2-1 from stable_1;") + for i in range(self.fornum): + sql = "select ts ," + sql += "%s, " % random.choice(s_r_select) + sql += "%s " % random.choice(q_select) + sql += " from ( select * from regular_table_1 ) where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += " ;" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + tdSql.checkRows(100) + + #join: select column from (select column form regular_table1,regular_table2 )where t1.ts=t2.ts and <\>\in\and\or order by + #cross join not supported yet + tdSql.query("select 2-2 from stable_1;") + for i in range(self.fornum): + sql = "select ts , * from ( select t1.ts ," + sql += "t1.%s, " % random.choice(q_select) + sql += "t1.%s, " % random.choice(q_select) + sql += "t2.%s, " % random.choice(q_select) + sql += "t2.%s, " % random.choice(q_select) + sql += "t2.ts from regular_table_1 t1 , regular_table_2 t2 ) where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_where) + sql += "%s " % random.choice(order_u_where) + #sql += ");" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + + #3 select * from (select column\tag form stable where <\>\in\and\or order by ) + #self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 3-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s, " % random.choice(s_s_select) + sql += "%s, " % random.choice(q_select) + sql += "%s, " % random.choice(t_select) + sql += "ts from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(order_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + tdSql.checkRows(300) + tdSql.query("select 3-1 from stable_1;") + for i in range(self.fornum): + sql = "select ts, " + sql += "%s " % random.choice(s_r_select) + sql += "from ( select " + sql += "%s, " % random.choice(s_s_select) + sql += "%s, " % random.choice(q_select) + sql += "%s, " % random.choice(t_select) + sql += "ts from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(order_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + tdSql.checkRows(300) + + # select ts,* from (select column\tag form stable1,stable2 where t1.ts = t2.ts and <\>\in\and\or order by ) + #self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 3-2 from stable_1;") + for i in range(self.fornum): + sql = "select ts , * from ( select t1.ts , " + sql += "t1.%s, " % random.choice(s_s_select) + sql += "t1.%s, " % random.choice(q_select) + sql += "t2.%s, " % random.choice(s_s_select) + sql += "t2.%s, " % random.choice(q_select) + sql += "t2.ts from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_join_where) + sql += "%s " % random.choice(order_u_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + # TD-15609 tdSql.query(sql) + # tdSql.checkRows(100) + + #3 outer union not support + rsDn = self.restartDnodes() + tdSql.query("select 3-3 from stable_1;") + for i in range(self.fornum): + #sql = "select ts , * from ( select " + sql = "select ts from ( select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += ")" + sql += " %s " % random.choice(unionall_or_union) + sql += "select ts from ( select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from stable_2 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += ")" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + #TD-15610 tdSql.query(sql) + # tdSql.checkRows(100) + + #3 inter union not support + tdSql.query("select 3-4 from stable_1;") + for i in range(self.fornum): + sql = "select ts , * from ( select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += " %s " % random.choice(unionall_or_union) + sql += " select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from stable_2 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += ")" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + + #join:TD-6020\TD-6155 select * from (select column form stable1,stable2 where t1.ts=t2.ts and <\>\in\and\or order by) + tdSql.query("select 3-5 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select t1.ts ," + sql += "t1.%s, " % random.choice(q_select) + sql += "t1.%s, " % random.choice(q_select) + sql += "t2.%s, " % random.choice(q_select) + sql += "t2.%s, " % random.choice(q_select) + sql += "t2.ts from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_u_where) + sql += "%s " % random.choice(order_u_where) + sql += ");" + tdLog.info(sql) + tdLog.info(len(sql)) + # TD-15609 tdSql.query(sql) + # tdSql.checkRows(100) + + tdSql.query("select 3-6 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select t1.ts ," + sql += "t1.%s, " % random.choice(q_select) + sql += "t1.%s, " % random.choice(q_select) + sql += "t2.%s, " % random.choice(q_select) + sql += "t2.%s, " % random.choice(q_select) + sql += "t2.ts from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += ");" + tdLog.info(sql) + tdLog.info(len(sql)) + # TD-15609 同上 tdSql.query(sql) + # tdSql.checkRows(100) + + #4 select column from (select * form stable where <\>\in\and\or order by ) + #self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 4-1 from stable_1;") + for i in range(self.fornum): + sql = "select ts , " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "%s " % random.choice(t_select) + sql += " from ( select * from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(order_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #TD-15616 tdSql.query(sql) + # tdSql.checkRows(300) + + #5 select distinct column\tag from (select * form stable where <\>\in\and\or order by limit offset ) + tdSql.query("select 5-1 from stable_1;") + for i in range(self.fornum): + sql = "select " + sql += "%s " % random.choice(dqt_select) + sql += " from ( select * from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(order_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #TD-15500 tdSql.query(sql) + + #5-1 select distinct column\tag from (select calc form stable where <\>\in\and\or order by limit offset ) + tdSql.query("select 5-2 from stable_1;") + for i in range(self.fornum): + sql = "select distinct c5_1 " + sql += " from ( select " + sql += "%s " % random.choice(calc_select_in_ts) + sql += " as c5_1 from stable_1 where " + sql += "%s " % random.choice(qt_where) + #sql += "%s " % random.choice(order_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + #tdSql.checkRows(1)有的函数还没有提交,会不返回结果,先忽略 + + #6-error select * from (select distinct(tag) form stable where <\>\in\and\or order by limit ) + tdSql.query("select 6-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(dt_select) + sql += " from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(order_desc_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + tdSql.query("select 6-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(dt_select) + sql += " from stable_1 where " + sql += "%s ) ;" % random.choice(qt_where) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + #tdSql.checkRows(1)#数量不一致,不在校验 + + #7-error select * from (select distinct(tag) form stable where <\>\in\and\or order by limit ) + tdSql.query("select 7-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(dq_select) + sql += " from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(order_desc_where) + sql += "%s " % random.choice([limit_where[0] , limit_where[1]] ) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + + #calc_select,TWA/Diff/Derivative/Irate are not allowed to apply to super table directly + #8 select * from (select ts,calc form ragular_table where <\>\in\and\or order by ) + + # dcDB = self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 8-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select ts ," + sql += "%s " % random.choice(calc_select_in_ts) + sql += "from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 8-2 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(calc_select_in_ts_j) + sql += "from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 8-3 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(calc_select_in_ts_j) + sql += "from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.error(sql) + #tdSql.query(sql) + + #9 select * from (select ts,calc form stable where <\>\in\and\or order by ) + # self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 9-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select ts ," + sql += "%s " % random.choice(calc_select_in_ts) + sql += "from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 9-2 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(calc_select_in_ts_j) + sql += "from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_join_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 9-3 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(calc_select_in_ts_j) + sql += "from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.error(sql) + + #functions or others can not be mixed up ,calc out select not use with ts + + #10 select calc from (select * form regualr_table where <\>\in\and\or order by ) + tdSql.query("select 10-1 from stable_1;") + for i in range(self.fornum): + sql = "select " + sql += "%s " % random.choice(calc_select_in_ts) + sql += "as calc10_1 from ( select * from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_desc_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + # TD-15503 tdSql.query(sql) + # tdSql.checkRows(1) + + #10-1 select calc from (select * form regualr_table where <\>\in\and\or order by ) + # rsDn = self.restartDnodes() + # self.dropandcreateDB_random("%s" %db, 1) + # rsDn = self.restartDnodes() + tdSql.query("select 10-2 from stable_1;") + for i in range(self.fornum): + sql = "select " + sql += "%s " % random.choice(calc_select_all) + sql += "as calc10_1 from ( select * from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_desc_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.query(sql) + # tdSql.checkRows(1) + + #10-2 select calc from (select * form regualr_tables where <\>\in\and\or order by ) + tdSql.query("select 10-3 from stable_1;") + for i in range(self.fornum): + sql = "select " + sql += "%s as calc10_1 " % random.choice(calc_select_all) + sql += " from ( select * from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s ;" % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 10-4 from stable_1;") + for i in range(self.fornum): + sql = "select " + sql += "%s as calc10_1 " % random.choice(calc_select_all) + sql += " from ( select * from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s ;" % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.error(sql) + + #11 select calc from (select * form stable where <\>\in\and\or order by limit ) + tdSql.query("select 11-1 from stable_1;") + for i in range(self.fornum): + sql = "select " + sql += "%s " % random.choice(calc_select_in_ts) + sql += "as calc11_1 from ( select * from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(order_desc_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.query(sql) + # tdSql.checkRows(1) + + #11-1 select calc from (select * form stable where <\>\in\and\or order by limit ) + tdSql.query("select 11-2 from stable_1;") + for i in range(self.fornum): + sql = "select " + sql += "%s " % random.choice(calc_select_all) + sql += "as calc11_1 from ( select * from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(order_desc_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.query(sql) + # tdSql.checkRows(1) + + #11-2 select calc from (select * form stables where <\>\in\and\or order by limit ) + tdSql.query("select 11-3 from stable_1;") + for i in range(self.fornum): + sql = "select " + sql += "%s " % random.choice(calc_select_all) + sql += "as calc11_1 from ( select * from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_join_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s ;" % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + #TD-15493 tdSql.query(sql) + + tdSql.query("select 11-4 from stable_1;") + for i in range(self.fornum): + sql = "select " + sql += "%s " % random.choice(calc_select_all) + sql += "as calc11_1 from ( select * from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s ;" % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.error(sql) + + #12 select calc-diff from (select * form regualr_table where <\>\in\and\or order by limit ) + ##self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 12-1 from stable_1;") + for i in range(self.fornum): + sql = "select " + sql += "%s " % random.choice(calc_calculate_regular) + sql += " from ( select * from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_desc_where) + sql += "%s " % random.choice([limit_where[2] , limit_where[3]] ) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.query(sql) + # tdSql.checkRows(1) + + tdSql.query("select 12-2 from stable_1;") + for i in range(self.fornum): + sql = "select " + sql += "%s " % random.choice(calc_calculate_regular) + sql += " from ( select * from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice([limit_where[2] , limit_where[3]] ) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + #tdSql.checkRows(1) + + tdSql.query("select 12-2.2 from stable_1;") + for i in range(self.fornum): + sql = "select " + sql += "%s " % random.choice(calc_calculate_regular) + sql += " from ( select * from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice([limit_where[2] , limit_where[3]] ) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.error(sql) + + #12-1 select calc-diff from (select * form stable where <\>\in\and\or order by limit ) + tdSql.query("select 12-3 from stable_1;") + rsDn = self.restartDnodes() + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(calc_calculate_regular) + sql += " from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(group_where) + sql += ") " + sql += "%s " % random.choice(order_desc_where) + sql += "%s " % random.choice([limit_where[2] , limit_where[3]] ) + sql += " ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 12-4 from stable_1;") + #join query does not support group by + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(calc_calculate_regular_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_join_where) + sql += "%s " % random.choice(group_where) + sql += ") " + sql += "%s " % random.choice(order_desc_where) + sql += "%s " % random.choice([limit_where[2] , limit_where[3]] ) + sql += " ;" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + + tdSql.query("select 12-5 from stable_1;") + #join query does not support group by + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(calc_calculate_regular_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(group_where) + sql += ") " + sql += "%s " % random.choice(order_desc_where) + sql += "%s " % random.choice([limit_where[2] , limit_where[3]] ) + sql += " ;" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + + + #13 select calc-diff as diffns from (select * form stable where <\>\in\and\or order by limit ) + tdSql.query("select 13-1 from stable_1;") + for i in range(self.fornum): + sql = "select " + sql += "%s " % random.choice(calc_calculate_regular) + sql += " as calc13_1 from ( select * from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(orders_desc_where) + sql += "%s " % random.choice([limit_where[2] , limit_where[3]] ) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.error(sql) + + #14 select * from (select calc_aggregate_alls as agg from stable where <\>\in\and\or group by order by slimit soffset ) + # TD-5955 select * from ( select count (q_double) from stable_1 where t_bool = true or t_bool = false group by loc order by ts asc slimit 1 ) ; + tdSql.query("select 14-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc14_1, " % random.choice(calc_aggregate_all) + sql += "%s as calc14_2, " % random.choice(calc_aggregate_all) + sql += "%s " % random.choice(calc_aggregate_all) + sql += " as calc14_3 from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(group_where) + sql += "%s " % random.choice(order_desc_where) + sql += "%s " % random.choice(slimit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + #tdSql.checkRows(1) + + # error group by in out query + tdSql.query("select 14-2 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc14_1, " % random.choice(calc_aggregate_all) + sql += "%s as calc14_2, " % random.choice(calc_aggregate_all) + sql += "%s " % random.choice(calc_aggregate_all) + sql += " as calc14_3 from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(group_where) + sql += "%s " % random.choice(having_support) + sql += "%s " % random.choice(orders_desc_where) + sql += "%s " % random.choice(slimit1_where) + sql += ") " + sql += "%s " % random.choice(group_where) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + #14-2 TD-6426 select * from (select calc_aggregate_all_js as agg from stables where <\>\in\and\or group by order by slimit soffset ) + tdSql.query("select 14-3 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc14_1, " % random.choice(calc_aggregate_all_j) + sql += "%s as calc14_2, " % random.choice(calc_aggregate_all_j) + sql += "%s " % random.choice(calc_aggregate_all_j) + sql += " as calc14_3 from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_join_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(slimit1_where) + sql += ") " + sql += "%s ;" % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 14-4 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc14_1, " % random.choice(calc_aggregate_all_j) + sql += "%s as calc14_2, " % random.choice(calc_aggregate_all_j) + sql += "%s " % random.choice(calc_aggregate_all_j) + sql += " as calc14_3 from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(slimit1_where) + sql += ") " + sql += "%s ;" % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + #15 TD-6320 select * from (select calc_aggregate_regulars as agg from regular_table where <\>\in\and\or order by slimit soffset ) + tdSql.query("select 15-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc15_1, " % random.choice(calc_aggregate_regular) + sql += "%s as calc15_2, " % random.choice(calc_aggregate_regular) + sql += "%s " % random.choice(calc_aggregate_regular) + sql += " as calc15_3 from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_desc_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.query(sql) + # tdSql.checkRows(1) + + tdSql.query("select 15-2 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc15_1, " % random.choice(calc_aggregate_regular_j) + sql += "%s as calc15_2, " % random.choice(calc_aggregate_regular_j) + sql += "%s " % random.choice(calc_aggregate_regular_j) + sql += " as calc15_3 from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s ;" % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 15-2.2 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc15_1, " % random.choice(calc_aggregate_regular_j) + sql += "%s as calc15_2, " % random.choice(calc_aggregate_regular_j) + sql += "%s " % random.choice(calc_aggregate_regular_j) + sql += " as calc15_3 from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s ;" % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.error(sql) + + rsDn = self.restartDnodes() + tdSql.query("select 15-3 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc15_1, " % random.choice(calc_aggregate_groupbytbname) + sql += "%s as calc15_2, " % random.choice(calc_aggregate_groupbytbname) + sql += "%s " % random.choice(calc_aggregate_groupbytbname) + sql += " as calc15_3 from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(group_where) + sql += "%s " % random.choice(having_support) + sql += "%s " % random.choice(order_desc_where) + sql += ") " + sql += "order by calc15_1 " + sql += "%s " % random.choice(limit_where) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 15-4 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc15_1, " % random.choice(calc_aggregate_groupbytbname_j) + sql += "%s as calc15_2, " % random.choice(calc_aggregate_groupbytbname_j) + sql += "%s " % random.choice(calc_aggregate_groupbytbname_j) + sql += " as calc15_3 from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_join_where) + sql += "%s " % random.choice(group_where) + sql += "%s " % random.choice(having_support) + sql += "%s " % random.choice(orders_desc_where) + sql += ") " + sql += "order by calc15_1 " + sql += "%s " % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 15-4.2 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc15_1, " % random.choice(calc_aggregate_groupbytbname_j) + sql += "%s as calc15_2, " % random.choice(calc_aggregate_groupbytbname_j) + sql += "%s " % random.choice(calc_aggregate_groupbytbname_j) + sql += " as calc15_3 from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(group_where) + sql += "%s " % random.choice(having_support) + sql += "%s " % random.choice(orders_desc_where) + sql += ") " + sql += "order by calc15_1 " + sql += "%s " % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 15-5 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc15_1, " % random.choice(calc_aggregate_groupbytbname) + sql += "%s as calc15_2, " % random.choice(calc_aggregate_groupbytbname) + sql += "%s " % random.choice(calc_aggregate_groupbytbname) + sql += " as calc15_3 from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(orders_desc_where) + sql += ") " + sql += "order by calc15_1 " + sql += "%s " % random.choice(limit_where) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + #16 select * from (select calc_aggregate_regulars as agg from regular_table where <\>\in\and\or order by limit offset ) + #self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 16-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc16_0 , " % random.choice(calc_calculate_all) + sql += "%s as calc16_1 , " % random.choice(calc_aggregate_all) + sql += "%s as calc16_2 " % random.choice(calc_select_in) + sql += " from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(group_where) + sql += "%s " % random.choice(having_support) + sql += ") " + sql += "order by calc16_0 " + sql += "%s " % random.choice(limit1_where) + tdLog.info(sql) + tdLog.info(len(sql)) + # TD-15497 tdSql.query(sql) + # tdSql.checkRows(1) + + tdSql.query("select 16-2 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc16_0 " % random.choice(calc_calculate_all_j) + sql += ", %s as calc16_1 " % random.choice(calc_aggregate_all_j) + #sql += ", %s as calc16_2 " % random.choice(calc_select_in_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_join_where) + sql += ") " + sql += "order by calc16_0 " + sql += "%s " % random.choice(limit1_where) + tdLog.info(sql) + tdLog.info(len(sql)) + #TD-15493 tdSql.query(sql) + + tdSql.query("select 16-2.2 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc16_0 " % random.choice(calc_calculate_all_j) + sql += ", %s as calc16_1 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += ") " + sql += "order by calc16_0 " + sql += "%s " % random.choice(limit1_where) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 16-3 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc16_1 " % random.choice(calc_calculate_regular) + sql += " from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "limit 2 ) " + sql += "%s " % random.choice(limit1_where) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.query(sql) + # tdSql.checkRows(1) + + tdSql.query("select 16-4 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc16_1 " % random.choice(calc_calculate_regular_j) + sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_where) + sql += "limit 2 ) " + sql += "%s " % random.choice(limit1_where) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.query(sql) + # tdSql.checkRows(1) + + tdSql.query("select 16-4.2 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc16_1 " % random.choice(calc_calculate_regular_j) + sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "limit 2 ) " + sql += "%s " % random.choice(limit1_where) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 16-5 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc16_1 , " % random.choice(calc_calculate_all) + sql += "%s as calc16_1 , " % random.choice(calc_calculate_regular) + sql += "%s as calc16_2 " % random.choice(calc_select_all) + sql += " from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(group_where) + sql += "%s " % random.choice(having_support) + sql += ") " + sql += "order by calc16_1 " + sql += "%s " % random.choice(limit1_where) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 16-6 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc16_1 " % random.choice(calc_calculate_groupbytbname) + sql += " from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(group_where) + sql += "limit 2 ) " + sql += "%s " % random.choice(limit1_where) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.query(sql) + # tdSql.checkRows(1) + + tdSql.query("select 16-7 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc16_1 " % random.choice(calc_calculate_groupbytbname_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_join_where) + sql += "limit 2 ) " + sql += "%s " % random.choice(limit1_where) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 16-8 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc16_1 " % random.choice(calc_calculate_groupbytbname_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "limit 2 ) " + sql += "%s " % random.choice(limit1_where) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + #17 select apercentile from (select calc_aggregate_alls form regualr_table or stable where <\>\in\and\or interval_sliding group by having order by limit offset )interval_sliding + #self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 17-1 from stable_1;") + for i in range(self.fornum): + #this is having_support , but tag-select cannot mix with last_row,other select can + sql = "select apercentile(cal17_0, %d)/10 ,apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_0 , " % random.choice(calc_calculate_all) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all) + sql += " from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(group_where) + sql += "%s " % random.choice(having_support) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit1_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 17-2 from stable_1;") + for i in range(self.fornum): + #this is having_support , but tag-select cannot mix with last_row,other select can + sql = "select apercentile(cal17_0, %d)/10 ,apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_0 , " % random.choice(calc_calculate_all_j) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_join_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + #Column ambiguously defined: ts tdSql.query(sql) + + tdSql.query("select 17-2.2 from stable_1;") + for i in range(self.fornum): + #this is having_support , but tag-select cannot mix with last_row,other select can + sql = "select apercentile(cal17_0, %d)/10 ,apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_0 , " % random.choice(calc_calculate_all_j) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + rsDn = self.restartDnodes() + tdSql.query("select 17-3 from stable_1;") + for i in range(self.fornum): + #this is having_tagnot_support , because tag-select cannot mix with last_row... + sql = "select apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all) + sql += " from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(group_where) + sql += "%s " % random.choice(having_tagnot_support) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit1_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 17-4 from stable_1;") + for i in range(self.fornum): + #this is having_tagnot_support , because tag-select cannot mix with last_row... + sql = "select apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_join_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 17-4.2 from stable_1;") + for i in range(self.fornum): + #this is having_tagnot_support , because tag-select cannot mix with last_row... + sql = "select apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 17-5 from stable_1;") + for i in range(self.fornum): + #having_not_support + sql = "select apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all) + sql += " from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(group_where) + sql += "%s " % random.choice(having_not_support) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit1_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 17-6 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all) + sql += " from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit1_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 17-7 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1_1 t1, stable_1_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit1_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 17-7.2 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1_1 t1, stable_1_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit1_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + self.restartDnodes() + tdSql.query("select 17-8 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all) + sql += " from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit1_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 17-9 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all_j) + sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 17-10 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all_j) + sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + #18 select apercentile from (select calc_aggregate_alls form regualr_table or stable where <\>\in\and\or session order by limit )interval_sliding + tdSql.query("select 18-1 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal18_1, %d)/1000 ,apercentile(cal18_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal18_1 ," % random.choice(calc_aggregate_all) + sql += "%s as cal18_2 " % random.choice(calc_aggregate_all) + sql += " from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(session_where) + sql += "%s " % random.choice(fill_where) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit1_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 18-2 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal18_1, %d)/1000 ,apercentile(cal18_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal18_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal18_2 " % random.choice(calc_aggregate_all_j) + sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_where) + sql += "%s " % random.choice(session_u_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 18-2.2 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal18_1, %d)/1000 ,apercentile(cal18_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal18_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal18_2 " % random.choice(calc_aggregate_all_j) + sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(session_u_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + self.restartDnodes() + tdSql.query("select 18-3 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal18_1, %d)/1000 ,apercentile(cal18_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal18_1 ," % random.choice(calc_aggregate_all) + sql += "%s as cal18_2 " % random.choice(calc_aggregate_all) + sql += " from stable_1_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(session_where) + sql += "%s " % random.choice(fill_where) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit1_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 18-4 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal18_1, %d)/1000 ,apercentile(cal18_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal18_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal18_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_where) + sql += "%s " % random.choice(session_u_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.query(sql) + + tdSql.query("select 18-4.2 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal18_1, %d)/1000 ,apercentile(cal18_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal18_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal18_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(session_u_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + + tdSql.query("select 18-5 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal18_1, %d)/1000 ,apercentile(cal18_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal18_1 ," % random.choice(calc_aggregate_all) + sql += "%s as cal18_2 " % random.choice(calc_aggregate_all) + sql += " from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(session_where) + sql += "%s " % random.choice(fill_where) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit1_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 18-6 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal18_1, %d)/1000 ,apercentile(cal18_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal18_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal18_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_join_where) + sql += "%s " % random.choice(session_u_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 18-7 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal18_1, %d)/1000 ,apercentile(cal18_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal18_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal18_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(session_u_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + #19 select apercentile from (select calc_aggregate_alls form regualr_table or stable where <\>\in\and\or session order by limit )interval_sliding + #self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 19-1 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal19_1 ," % random.choice(calc_aggregate_all) + sql += "%s as cal19_2 " % random.choice(calc_aggregate_all) + sql += " from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(state_window) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit1_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + #TD-15545 tdSql.query(sql) + + tdSql.query("select 19-2 from stable_1;") + #TD-6435 state_window not support join + for i in range(self.fornum): + sql = "select apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal19_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal19_2 " % random.choice(calc_aggregate_all_j) + sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_where) + sql += "%s " % random.choice(state_u_window) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 19-2.2 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal19_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal19_2 " % random.choice(calc_aggregate_all_j) + sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(state_u_window) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 19-3 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal19_1 ," % random.choice(calc_aggregate_all) + sql += "%s as cal19_2 " % random.choice(calc_aggregate_all) + sql += " from stable_1_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(state_window) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit1_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + #不支持order by tdSql.query(sql) + + tdSql.query("select 19-4 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal19_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal19_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1_1 t1, stable_1_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_where) + #sql += "%s " % random.choice(state_window) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 19-4.2 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal19_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal19_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1_1 t1, stable_1_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 19-5 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal19_1 ," % random.choice(calc_aggregate_all) + sql += "%s as cal19_2 " % random.choice(calc_aggregate_all) + sql += " from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(state_window) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit1_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 19-6 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal19_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal19_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_where) + #sql += "%s " % random.choice(state_window) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 19-7 from stable_1;") + for i in range(self.fornum): + sql = "select apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal19_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal19_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + #20 select * from (select calc_select_fills form regualr_table or stable where <\>\in\and\or fill_where group by order by limit offset ) + #self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 20-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s , " % random.choice(calc_select_fill) + sql += "%s ," % random.choice(calc_select_fill) + sql += "%s " % random.choice(calc_select_fill) + sql += " from stable_1 where " + sql += "%s " % random.choice(interp_where) + sql += "%s " % random.choice(fill_where) + sql += "%s " % random.choice(group_where) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + rsDn = self.restartDnodes() + tdSql.query("select 20-2 from stable_1;") + #TD-6438 + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s , " % random.choice(calc_select_fill_j) + sql += "%s ," % random.choice(calc_select_fill_j) + sql += "%s " % random.choice(calc_select_fill_j) + sql += " from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " + sql += "%s and " % random.choice(t_join_where) + sql += "%s " % random.choice(interp_where_j) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 20-2.2 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s , " % random.choice(calc_select_fill_j) + sql += "%s ," % random.choice(calc_select_fill_j) + sql += "%s " % random.choice(calc_select_fill_j) + sql += " from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " + sql += "%s and " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(interp_where_j) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 20-3 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s , " % random.choice(calc_select_fill) + sql += "%s ," % random.choice(calc_select_fill) + sql += "%s " % random.choice(calc_select_fill) + sql += " from stable_1 where " + sql += "%s " % interp_where[2] + sql += "%s " % random.choice(fill_where) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + #interp不支持 tdSql.query(sql) + + tdSql.query("select 20-4 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s , " % random.choice(calc_select_fill_j) + sql += "%s ," % random.choice(calc_select_fill_j) + sql += "%s " % random.choice(calc_select_fill_j) + sql += " from stable_1 t1, table_1 t2 where t1.ts = t2.ts and " + #sql += "%s and " % random.choice(t_join_where) + sql += "%s " % interp_where_j[random.randint(0,5)] + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 20-4.2 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s , " % random.choice(calc_select_fill_j) + sql += "%s ," % random.choice(calc_select_fill_j) + sql += "%s " % random.choice(calc_select_fill_j) + sql += " from stable_1 t1, stable_1_1 t2 where t1.ts = t2.ts and " + sql += "%s and " % random.choice(qt_u_or_where) + sql += "%s " % interp_where_j[random.randint(0,5)] + sql += "%s " % random.choice(fill_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + tdSql.query("select 20-5 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s , " % random.choice(calc_select_fill) + sql += "%s ," % random.choice(calc_select_fill) + sql += "%s " % random.choice(calc_select_fill) + sql += " from regular_table_1 where " + sql += "%s " % interp_where[1] + sql += "%s " % random.choice(fill_where) + sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + tdSql.query("select 20-6 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s , " % random.choice(calc_select_fill_j) + sql += "%s ," % random.choice(calc_select_fill_j) + sql += "%s " % random.choice(calc_select_fill_j) + sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + #sql += "%s " % random.choice(interp_where_j) + sql += "%s " % interp_where_j[random.randint(0,5)] + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + # error + #1 select * from (select * from (select * form regular_table where <\>\in\and\or order by limit )) + tdSql.query("select 1-1 from stable_1;") + for i in range(self.fornum): + sql = "select * , ts from ( select * from ( select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += ")) ;" + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + #2 select * from (select * from (select * form stable where <\>\in\and\or order by limit )) + tdSql.query("select 2-1 from stable_1;") + for i in range(self.fornum): + sql = "select * , ts from ( select * from ( select " + sql += "%s, " % random.choice(s_s_select) + sql += "%s, " % random.choice(qt_select) + sql += "ts from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += ")) ;" + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + #3 select ts ,calc from (select * form stable where <\>\in\and\or order by limit ) + #self.dropandcreateDB_random("%s" %db, 1) + tdSql.query("select 3-1 from stable_1;") + for i in range(self.fornum): + sql = "select ts , " + sql += "%s " % random.choice(calc_calculate_regular) + sql += " from ( select * from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(orders_desc_where) + sql += "%s " % random.choice(limit_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + #4 select * from (select calc form stable where <\>\in\and\or order by limit ) + tdSql.query("select 4-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(calc_select_in_ts) + sql += "from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(order_desc_where) + sql += "%s " % random.choice(limit_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #tdSql.query(sql) + + #5 select ts ,tbname from (select * form stable where <\>\in\and\or order by limit ) + tdSql.query("select 5-1 from stable_1;") + for i in range(self.fornum): + sql = "select ts , tbname , " + sql += "%s ," % random.choice(calc_calculate_regular) + sql += "%s ," % random.choice(dqt_select) + sql += "%s " % random.choice(qt_select) + sql += " from ( select * from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(orders_desc_where) + sql += "%s " % random.choice(limit_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + # tdSql.error(sql) + + #special sql + tdSql.query("select 6-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select _block_dist() from stable_1);" + # tdSql.query(sql) + # tdSql.checkRows(1) + sql = "select _block_dist() from (select * from stable_1);" + tdSql.error(sql) + sql = "select * from (select database());" + tdSql.error(sql) + sql = "select * from (select client_version());" + tdSql.error(sql) + sql = "select * from (select client_version() as version);" + tdSql.error(sql) + sql = "select * from (select server_version());" + tdSql.error(sql) + sql = "select * from (select server_version() as version);" + tdSql.error(sql) + sql = "select * from (select server_status());" + tdSql.error(sql) + sql = "select * from (select server_status() as status);" + tdSql.error(sql) + + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/query_cols_tags_and_or.py b/tests/system-test/2-query/query_cols_tags_and_or.py index 77e91aa98356a47db053909b03949a95a37ac5de..5dc5a2f1236ea1d73f3a4a7f4775ec5133adc8d1 100644 --- a/tests/system-test/2-query/query_cols_tags_and_or.py +++ b/tests/system-test/2-query/query_cols_tags_and_or.py @@ -22,18 +22,6 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def insertData(self, tb_name): - # insert_sql_list = [f'insert into {tb_name} values ("2021-01-01 12:00:00", 1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1)', - # f'insert into {tb_name} values ("2021-01-05 12:00:00", 2, 2, 1, 3, 1.1, 1.1, "binary", "nchar", true, 2)', - # f'insert into {tb_name} values ("2021-01-07 12:00:00", 1, 3, 1, 2, 1.1, 1.1, "binary", "nchar", true, 3)', - # f'insert into {tb_name} values ("2021-01-09 12:00:00", 1, 2, 4, 3, 1.1, 1.1, "binary", "nchar", true, 4)', - # f'insert into {tb_name} values ("2021-01-11 12:00:00", 1, 2, 5, 5, 1.1, 1.1, "binary", "nchar", true, 5)', - # f'insert into {tb_name} values ("2021-01-13 12:00:00", 1, 2, 1, 3, 6.6, 1.1, "binary", "nchar", true, 6)', - # f'insert into {tb_name} values ("2021-01-15 12:00:00", 1, 2, 1, 3, 1.1, 7.7, "binary", "nchar", true, 7)', - # f'insert into {tb_name} values ("2021-01-17 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary8", "nchar", true, 8)', - # f'insert into {tb_name} values ("2021-01-19 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary", "nchar9", true, 9)', - # f'insert into {tb_name} values ("2021-01-21 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary", "nchar", false, 10)', - # f'insert into {tb_name} values ("2021-01-23 12:00:00", 1, 3, 1, 3, 1.1, 1.1, Null, Null, false, 11)' - # ] insert_sql_list = [f'insert into {tb_name} values ("2021-01-01 12:00:00", 1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1, 2, 3, 4)', f'insert into {tb_name} values ("2021-01-05 12:00:00", 2, 2, 1, 3, 1.1, 1.1, "binary", "nchar", true, 2, 3, 4, 5)', f'insert into {tb_name} values ("2021-01-07 12:00:00", 1, 3, 1, 2, 1.1, 1.1, "binary", "nchar", true, 3, 4, 5, 6)', @@ -54,7 +42,6 @@ class TDTestCase: tb_name = tdCom.getLongName(8, "letters") tdSql.execute( f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned)") - # f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 int)") self.insertData(tb_name) return tb_name @@ -95,6 +82,31 @@ class TDTestCase: def queryTsCol(self, tb_name, check_elm=None): select_elm = "*" if check_elm is None else check_elm + # ts in + query_sql = f'select {select_elm} from {tb_name} where ts in ("2021-01-11 12:00:00", "2021-01-13 12:00:00")' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 6) if select_elm == "*" else False + # ts not in + query_sql = f'select {select_elm} from {tb_name} where ts not in ("2021-01-11 12:00:00", "2021-01-13 12:00:00")' + tdSql.query(query_sql) + tdSql.checkRows(9) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False + # ts not null + query_sql = f'select {select_elm} from {tb_name} where ts is not Null' + tdSql.query(query_sql) + tdSql.checkRows(11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False + # ts null + query_sql = f'select {select_elm} from {tb_name} where ts is Null' + tdSql.query(query_sql) + tdSql.checkRows(0) + # not support like not like match nmatch + tdSql.error(f'select {select_elm} from {tb_name} where ts like ("2021-01-11 12:00:00%")') + tdSql.error(f'select {select_elm} from {tb_name} where ts not like ("2021-01-11 12:00:0_")') + tdSql.error(f'select {select_elm} from {tb_name} where ts match "2021-01-11 12:00:00%"') + tdSql.error(f'select {select_elm} from {tb_name} where ts nmatch "2021-01-11 12:00:00%"') + # ts and ts query_sql = f'select {select_elm} from {tb_name} where ts > "2021-01-11 12:00:00" or ts < "2021-01-13 12:00:00"' tdSql.query(query_sql) @@ -1422,9 +1434,9 @@ class TDTestCase: tdSql.query(query_sql) tdSql.checkRows(11) tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False - query_sql = f'select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13 from {tb_name} where c9 > "binary" and c9 >= "binary8" or c9 < "binary9" and c9 <= "binary" and c9 != 2 and c9 <> 2 and c9 = 4 or c9 is not null and c9 between 2 and 4 and c9 not between 1 and 2 and c9 in (2,4) and c9 not in (1,2) or c9 match "binary[28]" or c9 nmatch "binary"' + query_sql = f'select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13 from {tb_name} where c9 > "binary" and c9 >= "binary8" or c9 < "binary9" and c9 <= "binary" and c9 != 2 and c9 <> 2 and c9 = 4 or c9 is not null and c9 between 2 and 4 and c9 not between 1 and 2 and c9 in (2,4) and c9 not in (1,2)' tdSql.query(query_sql) - tdSql.checkRows(11) + tdSql.checkRows(9) def queryFullColType(self, tb_name, check_elm=None): select_elm = "*" if check_elm is None else check_elm diff --git a/tests/system-test/2-query/union.py b/tests/system-test/2-query/union.py new file mode 100644 index 0000000000000000000000000000000000000000..5b0099c6ebe7a27c00b166707b92e36eaa638aef --- /dev/null +++ b/tests/system-test/2-query/union.py @@ -0,0 +1,412 @@ +import datetime + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +PRIMARY_COL = "ts" + +INT_COL = "c1" +BINT_COL = "c2" +SINT_COL = "c3" +TINT_COL = "c4" +FLOAT_COL = "c5" +DOUBLE_COL = "c6" +BOOL_COL = "c7" + +BINARY_COL = "c8" +NCHAR_COL = "c9" +TS_COL = "c10" + +NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] +CHAR_COL = [ BINARY_COL, NCHAR_COL, ] +BOOLEAN_COL = [ BOOL_COL, ] +TS_TYPE_COL = [ TS_COL, ] + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def __query_condition(self,tbname): + query_condition = [] + for char_col in CHAR_COL: + query_condition.extend( + ( + f"{tbname}.{char_col}", + f"upper( {tbname}.{char_col} )", + f"char_length( {tbname}.{char_col} )", + f"concat( {tbname}.{char_col}, {tbname}.{char_col} )", + f"concat_ws( '_', {tbname}.{char_col}, {tbname}.{char_col} )", + f"length( {tbname}.{char_col} )", + f"lower( {tbname}.{char_col} )", + f"ltrim( {tbname}.{char_col} )", + f"rtrim( {tbname}.{char_col} )", + f"substr( {tbname}.{char_col}, 1 )", + f"count( {tbname}.{char_col} )", + f"cast( {tbname}.{char_col} as nchar(3) )", + f"cast( {tbname}.{char_col} as nchar(8) )", + ) + ) + query_condition.extend( f"cast( {tbname}.{un_char_col} as binary(16) ) " for un_char_col in NUM_COL) + query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL ) + query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{un_char_col} as binary(32) ) " for un_char_col in NUM_COL ) + + for num_col in NUM_COL: + query_condition.extend( + ( + f"{tbname}.{num_col}", + f"ceil( {tbname}.{num_col} )", + f"abs( {tbname}.{num_col} )", + f"acos( {tbname}.{num_col} )", + f"asin( {tbname}.{num_col} )", + f"atan( {tbname}.{num_col} )", + f"cos( {tbname}.{num_col} )", + f"floor( {tbname}.{num_col} )", + f"log( {tbname}.{num_col}, {tbname}.{num_col})", + f"sin( {tbname}.{num_col} )", + f"sqrt( {tbname}.{num_col} )", + f"tan( {tbname}.{num_col} )", + f"round( {tbname}.{num_col} )", + f"max( {tbname}.{num_col} )", + f"sum( {tbname}.{num_col} )", + f"count( {tbname}.{num_col} )", + f"min( {tbname}.{num_col} )", + ) + ) + query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_2}" for num_col_2 in NUM_COL ) + query_condition.extend( f"{tbname}.{num_col} + {tbname}.{char_col} " for char_col in CHAR_COL ) + + query_condition.extend( + ( + ''' "test1234!@#$%^&*():'>= 0" + if col in CHAR_COL: + return f" where lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' " + if col in BOOLEAN_COL: + return f" where {tbname}.{col} in (false, true) " + if col in TS_TYPE_COL or col in PRIMARY_COL: + return f" where cast( {tbname}.{col} as binary(16) ) is not null " + + return "" + + + def __group_condition(self, col, having = None): + if isinstance(col, str): + if col.startswith("count"): + col = col[6:-1] + elif col.startswith("max"): + col = col[4:-1] + elif col.startswith("sum"): + col = col[4:-1] + elif col.startswith("min"): + col = col[4:-1] + return f" group by {col} having {having}" if having else f" group by {col} " + + def __single_sql(self, select_clause, from_clause, where_condition="", group_condition=""): + if isinstance(select_clause, str) and "on" not in from_clause and select_clause.split(".")[0] != from_clause.split(".")[0]: + return + return f"select {select_clause} from {from_clause} {where_condition} {group_condition}" + + + @property + def __join_tblist(self): + return [ + ["ct1", "ct2"], + ["ct1", "ct4"], + ["ct1", "t1"], + ["ct2", "ct4"], + ["ct2", "t1"], + ["ct4", "t1"], + # ["ct1", "ct2", "ct4"], + # ["ct1", "ct2", "t1"], + # ["ct1", "ct4", "t1"], + # ["ct2", "ct4", "t1"], + # ["ct1", "ct2", "ct4", "t1"], + ] + + @property + def __tb_liast(self): + return [ + "ct1", + "ct2", + "ct4", + "t1", + ] + + def sql_list(self): + sqls = [] + __join_tblist = self.__join_tblist + for join_tblist in __join_tblist: + for join_tb in join_tblist: + select_claus_list = self.__query_condition(join_tb) + for select_claus in select_claus_list: + group_claus = self.__group_condition( col=select_claus) + where_claus = self.__where_condition(query_conditon=select_claus) + having_claus = self.__group_condition( col=select_claus, having=f"{select_claus} is not null") + sqls.extend( + ( + self.__single_sql(select_claus, join_tb, where_claus, group_claus), + self.__single_sql(select_claus, join_tb, where_claus, having_claus), + self.__single_sql(select_claus, self.__join_condition(join_tblist), where_claus, having_claus), + self.__single_sql(select_claus, self.__join_condition(join_tblist, INNER=True), where_claus, having_claus), + self.__single_sql(select_claus, join_tb, where_claus), + self.__single_sql(select_claus, join_tb, having_claus), + self.__single_sql(select_claus, join_tb, group_claus), + self.__single_sql(select_claus, join_tb), + + ) + ) + __no_join_tblist = self.__tb_liast + for tb in __no_join_tblist: + select_claus_list = self.__query_condition(tb) + for select_claus in select_claus_list: + group_claus = self.__group_condition(col=select_claus) + where_claus = self.__where_condition(query_conditon=select_claus) + having_claus = self.__group_condition(col=select_claus, having=f"{select_claus} is not null") + sqls.extend( + ( + self.__single_sql(select_claus, join_tb, where_claus, group_claus), + self.__single_sql(select_claus, join_tb, where_claus, having_claus), + self.__single_sql(select_claus, join_tb, where_claus), + self.__single_sql(select_claus, join_tb, group_claus), + self.__single_sql(select_claus, join_tb, having_claus), + self.__single_sql(select_claus, join_tb), + ) + ) + + return filter(None, sqls) + # return list(filter(None, sqls)) + + def __get_type(self, col): + if tdSql.cursor.istype(col, "BOOL"): + return "BOOL" + if tdSql.cursor.istype(col, "INT"): + return "INT" + if tdSql.cursor.istype(col, "BIGINT"): + return "BIGINT" + if tdSql.cursor.istype(col, "TINYINT"): + return "TINYINT" + if tdSql.cursor.istype(col, "SMALLINT"): + return "SMALLINT" + if tdSql.cursor.istype(col, "FLOAT"): + return "FLOAT" + if tdSql.cursor.istype(col, "DOUBLE"): + return "DOUBLE" + if tdSql.cursor.istype(col, "BINARY"): + return "BINARY" + if tdSql.cursor.istype(col, "NCHAR"): + return "NCHAR" + if tdSql.cursor.istype(col, "TIMESTAMP"): + return "TIMESTAMP" + if tdSql.cursor.istype(col, "JSON"): + return "JSON" + if tdSql.cursor.istype(col, "TINYINT UNSIGNED"): + return "TINYINT UNSIGNED" + if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"): + return "SMALLINT UNSIGNED" + if tdSql.cursor.istype(col, "INT UNSIGNED"): + return "INT UNSIGNED" + if tdSql.cursor.istype(col, "BIGINT UNSIGNED"): + return "BIGINT UNSIGNED" + + def union_check(self): + sqls = self.sql_list() + for sql1 in sqls: + tdSql.query(sql1) + res1_type = self.__get_type(0) + for sql2 in sqls: + tdSql.query(sql2) + union_type = False + res2_type = self.__get_type(0) + + if res1_type in ( "BIGINT" , "NCHAR" ): + union_type = True + elif res2_type == res1_type: + union_type = True + elif res1_type == "TIMESAMP" and res2_type not in ("BINARY", "NCHAR"): + union_type = True + elif res1_type == "BINARY" and res2_type != "NCHAR": + union_type = True + + if union_type: + tdSql.query(f"{sql1} union {sql2}") + tdSql.checkCols(1) + tdSql.query(f"{sql1} union all {sql2}") + tdSql.checkCols(1) + else: + tdSql.error(f"{sql1} union {sql2}") + + def __test_error(self): + + + tdSql.error( "show tables union show tables" ) + tdSql.error( "create table errtb1 union all create table errtb2" ) + tdSql.error( "drop table ct1 union all drop table ct3" ) + tdSql.error( "select c1 from ct1 union all drop table ct3" ) + tdSql.error( "select c1 from ct1 union all '' " ) + tdSql.error( " '' union all select c1 from ct1 " ) + tdSql.error( "select c1 from ct1 union select c1 from ct2 union select c1 from ct4 ") + + def all_test(self): + self.__test_error() + self.union_check() + + + def __create_tb(self): + + tdLog.printNoPrefix("==========step1:create table") + create_stb_sql = f'''create table stb1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + ) tags (t1 int) + ''' + create_ntb_sql = f'''create table t1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + ) + ''' + tdSql.execute(create_stb_sql) + tdSql.execute(create_ntb_sql) + + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2} + + def __insert_data(self, rows): + now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + for i in range(rows): + tdSql.execute( + f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f'''insert into ct1 values + ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) + ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) + ''' + ) + + tdSql.execute( + f'''insert into ct4 values + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( + { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000} + ) + ( + { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000} + ) + ''' + ) + + tdSql.execute( + f'''insert into ct2 values + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( + { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, + { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } + ) + ( + { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, + { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } + ) + ''' + ) + + for i in range(rows): + insert_data = f'''insert into t1 values + ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, + "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) + ''' + tdSql.execute(insert_data) + tdSql.execute( + f'''insert into t1 values + ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, + "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } + ) + ( + { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, + "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } + ) + ''' + ) + + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + self.__create_tb() + + tdLog.printNoPrefix("==========step2:insert data") + self.rows = 10 + self.__insert_data(self.rows) + + tdLog.printNoPrefix("==========step3:all check") + self.all_test() + + tdDnodes.stop(1) + tdDnodes.start(1) + + tdSql.execute("use db") + + tdLog.printNoPrefix("==========step4:after wal, all check again ") + self.all_test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index 65840349bacd0f25d4eb99833c5c118ca2b523e0..c2fe25efc47064638722670304a8a0c8a869998f 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -52,7 +52,7 @@ class TDTestCase: def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) tsql.execute("use %s" %dbName) - tsql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) + tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" sql = pre_create #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) @@ -345,11 +345,11 @@ class TDTestCase: after starting consumer, create ctables ") # create and start thread parameterDict = {'cfg': '', \ - 'dbName': 'db2', \ + 'dbName': 'db3', \ 'vgroups': 1, \ 'stbName': 'stb', \ 'ctbNum': 10, \ - 'rowsPerTbl': 10000, \ + 'rowsPerTbl': 30000, \ 'batchNum': 100, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath @@ -374,22 +374,33 @@ class TDTestCase: break else: time.sleep(1) - + + tdLog.info("create stable2 for the seconde topic") + parameterDict2 = {'cfg': '', \ + 'dbName': 'db3', \ + 'vgroups': 1, \ + 'stbName': 'stb2', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 30000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict2['cfg'] = cfgPath + tdSql.execute("create stable if not exists %s.%s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%(parameterDict2['dbName'], parameterDict2['stbName'])) + tdLog.info("create topics from super table") - topicFromStb = 'topic_stb_column2' - topicFromCtb = 'topic_ctb_column2' + topicFromStb = 'topic_stb_column3' + topicFromStb2 = 'topic_stb_column32' tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName'])) - tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName'])) + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb2, parameterDict2['dbName'], parameterDict2['stbName'])) - time.sleep(1) tdSql.query("show topics") topic1 = tdSql.getData(0 , 0) topic2 = tdSql.getData(1 , 0) tdLog.info("show topics: %s, %s"%(topic1, topic2)) - if topic1 != topicFromStb and topic1 != topicFromCtb: + if topic1 != topicFromStb and topic1 != topicFromStb2: tdLog.exit("topic error1") - if topic2 != topicFromStb and topic2 != topicFromCtb: + if topic2 != topicFromStb and topic2 != topicFromStb2: tdLog.exit("topic error2") tdLog.info("create consume info table and consume result table") @@ -397,10 +408,9 @@ class TDTestCase: tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName) tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) - rowsOfNewCtb = 1000 consumerId = 0 - expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + rowsOfNewCtb - topicList = topicFromStb + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"] + topicList = topicFromStb + ',' + topicFromStb2 ifcheckdata = 0 keyList = 'group.id:cgrp1,\ enable.auto.commit:false,\ @@ -432,17 +442,13 @@ class TDTestCase: tdLog.info(shellCmd) os.system(shellCmd) - # create new child table and insert data - newCtbName = 'newctb' - tdSql.query("create table %s.%s using %s.%s tags(9999)"%(parameterDict["dbName"], newCtbName, parameterDict["dbName"], parameterDict["stbName"])) - startTs = parameterDict["startTs"] - for j in range(rowsOfNewCtb): - sql = "insert into %s.%s values (%d, %d, 'tmqrow_%d') "%(parameterDict["dbName"], newCtbName, startTs + j, j, j) - tdSql.execute(sql) - tdLog.debug("insert data into new child table ............ [OK]") + # start the second thread to create new child table and insert data + prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) + prepareEnvThread2.start() # wait for data ready prepareEnvThread.join() + prepareEnvThread2.join() tdLog.info("insert process end, and start to check consume result") while 1: @@ -457,7 +463,7 @@ class TDTestCase: tdSql.checkData(0 , 3, expectrowcnt) tdSql.query("drop topic %s"%topicFromStb) - tdSql.query("drop topic %s"%topicFromCtb) + tdSql.query("drop topic %s"%topicFromStb2) tdLog.printNoPrefix("======== test case 3 end ...... ") @@ -474,7 +480,7 @@ class TDTestCase: self.tmqCase1(cfgPath, buildPath) self.tmqCase2(cfgPath, buildPath) - #self.tmqCase3(cfgPath, buildPath) + self.tmqCase3(cfgPath, buildPath) def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index b8d3abca5c1cb8828777ff434c82ec8f3b659e30..90e6fc7da173a2d5af6d8e58cba7a6849be41e00 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -49,7 +49,38 @@ class TDTestCase: print(cur) return cur - def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg,showRow,cdbName,valgrind=0): + def initConsumerTable(self,cdbName='cdb'): + tdLog.info("create consume database, and consume info table, and consume result table") + tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) + + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + tdLog.info("consume info sql: %s"%sql) + tdSql.query(sql) + + def selectConsumeResult(self,expectRows,cdbName='cdb'): + resultList=[] + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == expectRows: + break + else: + time.sleep(5) + + for i in range(expectRows): + tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + resultList.append(tdSql.getData(i , 3)) + + return resultList + + def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): shellCmd = 'nohup ' if valgrind == 1: logFile = cfgPath + '/../log/valgrind-tmq.log' @@ -58,7 +89,7 @@ class TDTestCase: shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) @@ -87,6 +118,8 @@ class TDTestCase: pre_insert = "insert into " sql = pre_insert + t = time.time() + startTs = int(round(t * 1000)) #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) for i in range(ctbNum): sql += " %s_%d values "%(stbName,i) @@ -127,7 +160,7 @@ class TDTestCase: return def tmqCase1(self, cfgPath, buildPath): - tdLog.printNoPrefix("======== test case 1: Produce while one consume to subscribe one db") + tdLog.printNoPrefix("======== test case 1: Produce while one consume to subscribe one db, inclue 1 stb") tdLog.info("step 1: create database, stb, ctb and insert data") # create and start thread parameterDict = {'cfg': '', \ @@ -135,11 +168,13 @@ class TDTestCase: 'vgroups': 4, \ 'stbName': 'stb', \ 'ctbNum': 10, \ - 'rowsPerTbl': 100000, \ - 'batchNum': 200, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath + self.initConsumerTable() + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) @@ -149,23 +184,16 @@ class TDTestCase: topicName1 = 'topic_db1' tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) - - tdLog.info("create consume info table and consume result table") - cdbName = parameterDict["dbName"] - tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName) - tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) - consumerId = 0 expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] topicList = topicName1 ifcheckdata = 0 + ifManualCommit = 0 keyList = 'group.id:cgrp1,\ enable.auto.commit:false,\ auto.commit.interval.ms:6000,\ auto.offset.reset:earliest' - sql = "insert into %s.consumeinfo values "%cdbName - sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) - tdSql.query(sql) + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) event.wait() @@ -173,32 +201,28 @@ class TDTestCase: pollDelay = 5 showMsg = 1 showRow = 1 - self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName) + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) # wait for data ready prepareEnvThread.join() tdLog.info("insert process end, and start to check consume result") - while 1: - tdSql.query("select * from %s.consumeresult"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if tdSql.getRows() == 1: - break - else: - time.sleep(5) - - tdLog.info("consumer result: %d, %d"%(tdSql.getData(0 , 2), tdSql.getData(0 , 3))) - tdSql.checkData(0 , 1, consumerId) - # mulit rows and mulit tables in one sql, this num of msg is not sure - #tdSql.checkData(0 , 2, expectmsgcnt) - tdSql.checkData(0 , 3, expectrowcnt+1) + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") tdSql.query("drop topic %s"%topicName1) tdLog.printNoPrefix("======== test case 1 end ...... ") def tmqCase2(self, cfgPath, buildPath): - tdLog.printNoPrefix("======== test case 2: Produce while two consumers to subscribe one db") + tdLog.printNoPrefix("======== test case 2: Produce while two consumers to subscribe one db, inclue 1 stb") tdLog.info("step 1: create database, stb, ctb and insert data") # create and start thread parameterDict = {'cfg': '', \ @@ -206,11 +230,13 @@ class TDTestCase: 'vgroups': 4, \ 'stbName': 'stb', \ 'ctbNum': 10, \ - 'rowsPerTbl': 100000, \ + 'rowsPerTbl': 10000, \ 'batchNum': 100, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath + self.initConsumerTable() + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) @@ -221,27 +247,19 @@ class TDTestCase: tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) - tdLog.info("create consume info table and consume result table") - cdbName = parameterDict["dbName"] - tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName) - tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) - consumerId = 0 expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] topicList = topicName1 ifcheckdata = 0 + ifManualCommit = 0 keyList = 'group.id:cgrp1,\ enable.auto.commit:false,\ auto.commit.interval.ms:6000,\ auto.offset.reset:earliest' - sql = "insert into %s.consumeinfo values "%cdbName - sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) - tdSql.query(sql) - + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + consumerId = 1 - sql = "insert into %s.consumeinfo values "%cdbName - sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) - tdSql.query(sql) + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) event.wait() @@ -249,30 +267,20 @@ class TDTestCase: pollDelay = 5 showMsg = 1 showRow = 1 - self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName) + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) # wait for data ready prepareEnvThread.join() tdLog.info("insert process end, and start to check consume result") - while 1: - tdSql.query("select * from %s.consumeresult"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if tdSql.getRows() == 2: - break - else: - time.sleep(5) - - consumerId0 = tdSql.getData(0 , 1) - consumerId1 = tdSql.getData(1 , 1) - actConsumeRows0 = tdSql.getData(0 , 3) - actConsumeRows1 = tdSql.getData(1 , 3) - - tdLog.info("consumer %d rows: %d"%(consumerId0, actConsumeRows0)) - tdLog.info("consumer %d rows: %d"%(consumerId1, actConsumeRows1)) - - totalConsumeRows = actConsumeRows0 + actConsumeRows1 - if totalConsumeRows != expectrowcnt + 2: + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") tdSql.query("drop topic %s"%topicName1) @@ -288,11 +296,13 @@ class TDTestCase: 'vgroups': 4, \ 'stbName': 'stb', \ 'ctbNum': 10, \ - 'rowsPerTbl': 100000, \ + 'rowsPerTbl': 10000, \ 'batchNum': 100, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath + self.initConsumerTable() + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) @@ -303,7 +313,7 @@ class TDTestCase: 'vgroups': 4, \ 'stbName': 'stb2', \ 'ctbNum': 10, \ - 'rowsPerTbl': 100000, \ + 'rowsPerTbl': 10000, \ 'batchNum': 100, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath @@ -316,64 +326,376 @@ class TDTestCase: tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) - tdLog.info("create consume info table and consume result table") - cdbName = parameterDict["dbName"] - tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName) - tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) - consumerId = 0 expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"] topicList = topicName1 ifcheckdata = 0 + ifManualCommit = 0 keyList = 'group.id:cgrp1,\ enable.auto.commit:false,\ auto.commit.interval.ms:6000,\ auto.offset.reset:earliest' - sql = "insert into %s.consumeinfo values "%cdbName - sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) - tdSql.query(sql) + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) # consumerId = 1 - # sql = "insert into %s.consumeinfo values "%cdbName - # sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata) - # tdSql.query(sql) + # self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) event.wait() tdLog.info("start consume processor") pollDelay = 5 showMsg = 1 - showRow = 1 - self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow, cdbName) + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) # wait for data ready prepareEnvThread.join() prepareEnvThread2.join() tdLog.info("insert process end, and start to check consume result") - while 1: - tdSql.query("select * from %s.consumeresult"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if tdSql.getRows() == 1: - break - else: - time.sleep(5) + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicName1) + + tdLog.printNoPrefix("======== test case 3 end ...... ") + + def tmqCase4(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 4: Produce while two consumers to subscribe one db, include 2 stb") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db4', \ + 'vgroups': 4, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.initConsumerTable() + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + parameterDict2 = {'cfg': '', \ + 'dbName': 'db4', \ + 'vgroups': 4, \ + 'stbName': 'stb2', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) + prepareEnvThread2.start() - consumerId0 = tdSql.getData(0 , 1) - #consumerId1 = tdSql.getData(1 , 1) - actConsumeRows0 = tdSql.getData(0 , 3) - #actConsumeRows1 = tdSql.getData(1 , 3) + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) + + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"] + topicList = topicName1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + consumerId = 1 + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + event.wait() - tdLog.info("consumer %d rows: %d"%(consumerId0, actConsumeRows0)) - #tdLog.info("consumer %d rows: %d"%(consumerId1, actConsumeRows1)) + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) - #totalConsumeRows = actConsumeRows0 + actConsumeRows1 - if actConsumeRows0 != expectrowcnt + 1: + # wait for data ready + prepareEnvThread.join() + prepareEnvThread2.join() + + tdLog.info("insert process end, and start to check consume result") + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") tdSql.query("drop topic %s"%topicName1) - tdLog.printNoPrefix("======== test case 3 end ...... ") + tdLog.printNoPrefix("======== test case 4 end ...... ") + + def tmqCase5(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 5: Produce while two consumers to subscribe one db, firstly create one stb, after start consume create other stb") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db5', \ + 'vgroups': 4, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.initConsumerTable() + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + parameterDict2 = {'cfg': '', \ + 'dbName': 'db5', \ + 'vgroups': 4, \ + 'stbName': 'stb2', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) + + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"] + topicList = topicName1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + consumerId = 1 + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + event.wait() + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) + prepareEnvThread2.start() + + # wait for data ready + prepareEnvThread.join() + prepareEnvThread2.join() + + tdLog.info("insert process end, and start to check consume result") + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicName1) + + tdLog.printNoPrefix("======== test case 5 end ...... ") + + def tmqCase6(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 6: Produce while one consumers to subscribe tow topic, Each contains one db") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db60', \ + 'vgroups': 4, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.initConsumerTable() + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + parameterDict2 = {'cfg': '', \ + 'dbName': 'db61', \ + 'vgroups': 4, \ + 'stbName': 'stb2', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict2['dbName'], parameterDict2['vgroups'])) + + prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) + prepareEnvThread2.start() + + tdLog.info("create topics from db") + topicName1 = 'topic_db60' + topicName2 = 'topic_db61' + + tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) + tdSql.execute("create topic %s as %s" %(topicName2, parameterDict2['dbName'])) + + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"] + topicList = topicName1 + ',' + topicName2 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + #consumerId = 1 + #self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + event.wait() + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + # wait for data ready + prepareEnvThread.join() + prepareEnvThread2.join() + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicName1) + tdSql.query("drop topic %s"%topicName2) + + tdLog.printNoPrefix("======== test case 6 end ...... ") + + def tmqCase7(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 7: Produce while two consumers to subscribe tow topic, Each contains one db") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db70', \ + 'vgroups': 4, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.initConsumerTable() + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + parameterDict2 = {'cfg': '', \ + 'dbName': 'db71', \ + 'vgroups': 4, \ + 'stbName': 'stb2', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict2['dbName'], parameterDict2['vgroups'])) + + prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) + prepareEnvThread2.start() + + tdLog.info("create topics from db") + topicName1 = 'topic_db60' + topicName2 = 'topic_db61' + + tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) + tdSql.execute("create topic %s as %s" %(topicName2, parameterDict2['dbName'])) + + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"] + topicList = topicName1 + ',' + topicName2 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + consumerId = 1 + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + event.wait() + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + # wait for data ready + prepareEnvThread.join() + prepareEnvThread2.join() + + tdLog.info("insert process end, and start to check consume result") + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicName1) + tdSql.query("drop topic %s"%topicName2) + + tdLog.printNoPrefix("======== test case 7 end ...... ") def run(self): tdSql.prepare() @@ -387,8 +709,13 @@ class TDTestCase: tdLog.info("cfgPath: %s" % cfgPath) #self.tmqCase1(cfgPath, buildPath) - self.tmqCase2(cfgPath, buildPath) + #self.tmqCase2(cfgPath, buildPath) #self.tmqCase3(cfgPath, buildPath) + self.tmqCase4(cfgPath, buildPath) + self.tmqCase5(cfgPath, buildPath) + self.tmqCase6(cfgPath, buildPath) + #self.tmqCase7(cfgPath, buildPath) + def stop(self): tdSql.close() diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 2b813f83ca29b44abaf5a3e0897f9ef83f5bd341..1cb123a52f83edae3d60eae961e08cf4c505014f 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -53,6 +53,9 @@ python3 ./test.py -f 2-query/tan.py python3 ./test.py -f 2-query/arcsin.py python3 ./test.py -f 2-query/arccos.py python3 ./test.py -f 2-query/arctan.py -# python3 ./test.py -f 2-query/query_cols_tags_and_or.py +python3 ./test.py -f 2-query/query_cols_tags_and_or.py +python3 ./test.py -f 2-query/nestedQuery.py python3 ./test.py -f 7-tmq/basic5.py +python3 ./test.py -f 7-tmq/subscribeDb.py + diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 1228d6174c0e1ff672cc7d6fc6cb64729b50b280..b3dba695a7a8a5943e70e056a2ed17713f2a7142 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -37,9 +37,10 @@ typedef struct { TdThread thread; int32_t consumerId; - int32_t autoCommitIntervalMs; // 1000 ms - char autoCommit[8]; // true, false - char autoOffsetRest[16]; // none, earliest, latest + int32_t ifManualCommit; + //int32_t autoCommitIntervalMs; // 1000 ms + //char autoCommit[8]; // true, false + //char autoOffsetRest[16]; // none, earliest, latest int32_t ifCheckData; int64_t expectMsgCnt; @@ -136,9 +137,9 @@ void saveConfigToLogFile() { for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); - taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); - taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); - taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); + //taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); + //taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); + //taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); taosFprintfFile(g_fp, " Topics: "); for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); @@ -232,13 +233,18 @@ static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) while (1) { TAOS_ROW row = taos_fetch_row(msg); - if (row == NULL) break; - if (0 != g_stConfInfo.showRowFlag) { - TAOS_FIELD* fields = taos_fetch_fields(msg); - int32_t numOfFields = taos_field_count(msg); - taos_print_row(buf, row, fields, numOfFields); + + if (row == NULL) break; + + TAOS_FIELD* fields = taos_fetch_fields(msg); + int32_t numOfFields = taos_field_count(msg); + + taos_print_row(buf, row, fields, numOfFields); + + if (0 != g_stConfInfo.showRowFlag) { taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); } + totalRows++; } @@ -316,6 +322,8 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName, pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult); + taosFprintfFile(g_fp, "== save result sql: %s \n", sqlStr); + TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { pError("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); @@ -384,8 +392,12 @@ void* consumeThreadFunc(void* param) { loop_consume(pInfo); - tmq_commit(pInfo->tmq, NULL, 0); - + if (pInfo->ifManualCommit) { + taosFprintfFile(g_fp, "tmq_commit() manual commit when consume end.\n"); + pPrint("tmq_commit() manual commit when consume end.\n"); + tmq_commit(pInfo->tmq, NULL, 0); + } + err = tmq_unsubscribe(pInfo->tmq); if (err) { pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); @@ -470,9 +482,9 @@ int32_t getConsumeInfo() { int32_t* lengths = taos_fetch_lengths(pRes); // set default value - g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; - memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); - memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); + //g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; + //memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); + //memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); for (int i = 0; i < num_fields; ++i) { if (row[i] == NULL || 0 == i) { @@ -489,12 +501,8 @@ int32_t getConsumeInfo() { g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]); } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]); - } else if ((6 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { - memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, row[i], lengths[i]); - } else if ((7 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = *((int32_t*)row[i]); - } else if ((8 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { - memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, row[i], lengths[i]); + } else if ((6 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { + g_stConfInfo.stThreads[numOfThread].ifManualCommit = *((int32_t*)row[i]); } } numOfThread++;