diff --git a/Jenkinsfile b/Jenkinsfile index 22657bb262b7c6d7f2e9232fb0df21e589d479a1..bb757d6624bdc7e8d1ecd4735645f7716e0b943d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,6 +8,7 @@ def skipbuild = 0 def win_stop = 0 def scope = [] def mod = [0,1,2,3,4] +def sim_mod = [0,1,2,3] def abortPreviousBuilds() { def currentJobName = env.JOB_NAME @@ -54,10 +55,10 @@ def pre_test(){ git checkout master ''' } - else if(env.CHANGE_TARGET == '2.4'){ + else if(env.CHANGE_TARGET == '2.0'){ sh ''' cd ${WKC} - git checkout 2.4 + git checkout 2.0 ''' } else{ @@ -86,10 +87,10 @@ def pre_test(){ git checkout master ''' } - else if(env.CHANGE_TARGET == '2.4'){ + else if(env.CHANGE_TARGET == '2.0'){ sh ''' cd ${WK} - git checkout 2.4 + git checkout 2.0 ''' } else{ @@ -129,10 +130,10 @@ def pre_test_noinstall(){ git checkout master ''' } - else if(env.CHANGE_TARGET == '2.4'){ + else if(env.CHANGE_TARGET == '2.0'){ sh ''' cd ${WKC} - git checkout 2.4 + git checkout 2.0 ''' } else{ @@ -161,10 +162,10 @@ def pre_test_noinstall(){ git checkout master ''' } - else if(env.CHANGE_TARGET == '2.4'){ + else if(env.CHANGE_TARGET == '2.0'){ sh ''' cd ${WK} - git checkout 2.4 + git checkout 2.0 ''' } else{ @@ -201,10 +202,10 @@ def pre_test_mac(){ git checkout master ''' } - else if(env.CHANGE_TARGET == '2.4'){ + else if(env.CHANGE_TARGET == '2.0'){ sh ''' cd ${WKC} - git checkout 2.4 + git checkout 2.0 ''' } else{ @@ -233,10 +234,10 @@ def pre_test_mac(){ git checkout master ''' } - else if(env.CHANGE_TARGET == '2.4'){ + else if(env.CHANGE_TARGET == '2.0'){ sh ''' cd ${WK} - git checkout 2.4 + git checkout 2.0 ''' } else{ @@ -279,10 +280,10 @@ def pre_test_win(){ git checkout master ''' } - else if(env.CHANGE_TARGET == '2.4'){ + else if(env.CHANGE_TARGET == '2.0'){ bat ''' cd C:\\workspace\\TDinternal\\community - git checkout 2.4 + git checkout 2.0 ''' } else{ @@ -310,10 +311,10 @@ def pre_test_win(){ git checkout master ''' } - else if(env.CHANGE_TARGET == '2.4'){ + else if(env.CHANGE_TARGET == '2.0'){ bat ''' cd C:\\workspace\\TDinternal - git checkout 2.4 + git checkout 2.0 ''' } else{ @@ -382,7 +383,9 @@ pipeline { temp = (gitlog =~ /\((.*?)\)/) temp = temp[0].remove(1) scope = temp.split(",") + scope = ['connector','query','insert','other','tools','taosAdapter'] Collections.shuffle mod + Collections.shuffle sim_mod } } @@ -400,10 +403,10 @@ pipeline { } parallel { stage('python_1') { - agent{label " slave1 || slave6 || slave11 || slave16 "} + agent{label " slave1 || slave11 "} steps { pre_test() - timeout(time: 55, unit: 'MINUTES'){ + timeout(time: 100, unit: 'MINUTES'){ script{ scope.each { sh """ @@ -417,10 +420,10 @@ pipeline { } } stage('python_2') { - agent{label " slave2 || slave7 || slave12 || slave17 "} + agent{label " slave2 || slave12 "} steps { pre_test() - timeout(time: 55, unit: 'MINUTES'){ + timeout(time: 100, unit: 'MINUTES'){ script{ scope.each { sh """ @@ -434,7 +437,7 @@ pipeline { } } stage('python_3') { - agent{label " slave3 || slave8 || slave13 ||slave18 "} + agent{label " slave3 || slave13 "} steps { timeout(time: 105, unit: 'MINUTES'){ pre_test() @@ -451,9 +454,9 @@ pipeline { } } stage('python_4') { - agent{label " slave4 || slave9 || slave14 || slave19 "} + agent{label " slave4 || slave14 "} steps { - timeout(time: 55, unit: 'MINUTES'){ + timeout(time: 100, unit: 'MINUTES'){ pre_test() script{ scope.each { @@ -469,9 +472,9 @@ pipeline { } } stage('python_5') { - agent{label " slave5 || slave10 || slave15 || slave20 "} + agent{label " slave5 || slave15 "} steps { - timeout(time: 55, unit: 'MINUTES'){ + timeout(time: 100, unit: 'MINUTES'){ pre_test() script{ scope.each { @@ -486,35 +489,98 @@ pipeline { } } } - stage('arm64centos7') { - agent{label " arm64centos7 "} + stage('sim_1') { + agent{label " slave6 || slave16 "} steps { - pre_test_noinstall() - } + pre_test() + timeout(time: 100, unit: 'MINUTES'){ + sh """ + date + cd ${WKC}/tests + ./test-CI.sh sim 4 ${sim_mod[0]} + date""" + } + } } - stage('arm64centos8') { - agent{label " arm64centos8 "} + stage('sim_2') { + agent{label " slave7 || slave17 "} steps { - pre_test_noinstall() + pre_test() + timeout(time: 100, unit: 'MINUTES'){ + sh """ + date + cd ${WKC}/tests + ./test-CI.sh sim 4 ${sim_mod[1]} + date""" } + } } - stage('arm32bionic') { - agent{label " arm32bionic "} + stage('sim_3') { + agent{label " slave8 || slave18 "} steps { - pre_test_noinstall() + timeout(time: 105, unit: 'MINUTES'){ + pre_test() + sh """ + date + cd ${WKC}/tests + ./test-CI.sh sim 4 ${sim_mod[2]} + date""" } + } } - stage('arm64bionic') { - agent{label " arm64bionic "} + stage('sim_4') { + agent{label " slave9 || slave19 "} steps { - pre_test_noinstall() + timeout(time: 100, unit: 'MINUTES'){ + pre_test() + sh """ + date + cd ${WKC}/tests + ./test-CI.sh sim 4 ${sim_mod[3]} + date""" + } } + } - stage('arm64focal') { - agent{label " arm64focal "} + stage('other') { + agent{label " slave10 || slave20 "} steps { - pre_test_noinstall() + timeout(time: 100, unit: 'MINUTES'){ + pre_test() + timeout(time: 60, unit: 'MINUTES'){ + sh ''' + cd ${WKC}/tests/pytest + ./crash_gen.sh -a -p -t 4 -s 2000 + ''' + } + timeout(time: 60, unit: 'MINUTES'){ + sh ''' + cd ${WKC}/tests/pytest + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + ./handle_crash_gen_val_log.sh + ''' + sh ''' + cd ${WKC}/tests/pytest + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + ./handle_taosd_val_log.sh + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + ./valgrind-test.sh 2>&1 > mem-error-out.log + ./handle_val_log.sh + ''' + } + sh ''' + cd ${WKC}/tests + ./test-all.sh full unit + date + ''' } + } } stage('centos7') { agent{label " centos7 "} @@ -546,12 +612,41 @@ pipeline { pre_test_mac() } } - + stage('arm64centos7') { + agent{label " arm64centos7 "} + steps { + pre_test_noinstall() + } + } + stage('arm64centos8') { + agent{label " arm64centos8 "} + steps { + pre_test_noinstall() + } + } + stage('arm32bionic') { + agent{label " arm32bionic "} + steps { + pre_test_noinstall() + } + } + stage('arm64bionic') { + agent{label " arm64bionic "} + steps { + pre_test_noinstall() + } + } + stage('arm64focal') { + agent{label " arm64focal "} + steps { + pre_test_noinstall() + } + } stage('build'){ agent{label " wintest "} steps { pre_test() - script{ + script{ while(win_stop == 0){ sleep(1) } @@ -561,6 +656,7 @@ pipeline { stage('test'){ agent{label "win"} steps{ + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { pre_test_win() timeout(time: 20, unit: 'MINUTES'){ @@ -569,7 +665,7 @@ pipeline { .\\test-all.bat wintest ''' } - } + } script{ win_stop=1 } diff --git a/packaging/docker/dockerManifest.sh b/packaging/docker/dockerManifest.sh index 9f60b840d68577b751314e7ddecc9da98c20f8d6..71788423f6e58b2788346ef2804cd4d03ee54b02 100755 --- a/packaging/docker/dockerManifest.sh +++ b/packaging/docker/dockerManifest.sh @@ -44,30 +44,26 @@ echo "version=${version}" #docker manifest rm tdengine/tdengine #docker manifest rm tdengine/tdengine:${version} if [ "$verType" == "beta" ]; then - docker manifest inspect tdengine/tdengine-beta:latest - docker manifest inspect tdengine/tdengine-beta:${version} docker manifest create -a tdengine/tdengine-beta:${version} tdengine/tdengine-amd64-beta:${version} tdengine/tdengine-aarch64-beta:${version} tdengine/tdengine-aarch32-beta:${version} docker manifest create -a tdengine/tdengine-beta:latest tdengine/tdengine-amd64-beta:latest tdengine/tdengine-aarch64-beta:latest tdengine/tdengine-aarch32-beta:latest - sleep 30 docker manifest rm tdengine/tdengine-beta:${version} docker manifest rm tdengine/tdengine-beta:latest docker manifest create -a tdengine/tdengine-beta:${version} tdengine/tdengine-amd64-beta:${version} tdengine/tdengine-aarch64-beta:${version} tdengine/tdengine-aarch32-beta:${version} docker manifest create -a tdengine/tdengine-beta:latest tdengine/tdengine-amd64-beta:latest tdengine/tdengine-aarch64-beta:latest tdengine/tdengine-aarch32-beta:latest + docker manifest inspect tdengine/tdengine:latest + docker manifest inspect tdengine/tdengine:${version} docker login -u tdengine -p ${passWord} #replace the docker registry username and password docker manifest push tdengine/tdengine-beta:${version} docker manifest push tdengine/tdengine-beta:latest elif [ "$verType" == "stable" ]; then - docker manifest inspect tdengine/tdengine:latest - docker manifest inspect tdengine/tdengine:${version} docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest - sleep 30 docker manifest rm tdengine/tdengine:latest docker manifest rm tdengine/tdengine:${version} - docker manifest inspect tdengine/tdengine:latest - docker manifest inspect tdengine/tdengine:${version} docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest + docker manifest inspect tdengine/tdengine:latest + docker manifest inspect tdengine/tdengine:${version} docker login -u tdengine -p ${passWord} #replace the docker registry username and password docker manifest push tdengine/tdengine:${version} docker manifest push tdengine/tdengine:latest diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 7f95e3fd976c507924de3626b75a04ea5c9f6ec0..3e7728cd88c9c635ee8a937a802b39cdaa6025f6 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -42,12 +42,12 @@ if [ -d ${top_dir}/src/kit/taos-tools/packaging/deb ]; then cd ${top_dir}/src/kit/taos-tools/packaging/deb [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" - taostools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}') - taostools_install_dir="${release_dir}/${productName}-tools-${taostools_ver}" +taostools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}') +taostools_install_dir="${release_dir}/${productName}Tools-${taostools_ver}" cd ${curr_dir} else - taostools_install_dir="${release_dir}/${productName}-tools-${version}" + taostools_install_dir="${release_dir}/${productName}Tools-${version}" fi # Directories and files diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c index 9997d44ca55954b120ae4849d4f68be4d23419f7..e23bdc654e02bb3d9b34f656b0b49840c97f37e8 100644 --- a/src/balance/src/bnMain.c +++ b/src/balance/src/bnMain.c @@ -567,7 +567,7 @@ void bnCheckStatus() { while (1) { pIter = mnodeGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; - if (tsAccessSquence - pDnode->lastAccess > 3) { + if (tsAccessSquence - pDnode->lastAccess > tsOfflineInterval) { if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) { pDnode->status = TAOS_DN_STATUS_OFFLINE; pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT; diff --git a/src/client/src/tscGlobalmerge.c b/src/client/src/tscGlobalmerge.c index 5d936fe7067a9ce13a590537c2ba6162cf2a6c83..68e3bf4b8a20106d37c0dcd9c0a5e449c634ed58 100644 --- a/src/client/src/tscGlobalmerge.c +++ b/src/client/src/tscGlobalmerge.c @@ -902,7 +902,7 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) { // not belongs to the same group, return the result of current group; setInputDataBlock(pOperator, pAggInfo->binfo.pCtx, pAggInfo->pExistBlock, TSDB_ORDER_ASC); - updateOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity, pAggInfo->pExistBlock->info.rows); + updateOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity, pAggInfo->pExistBlock->info.rows, pOperator->pRuntimeEnv); { // reset output buffer for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { @@ -954,7 +954,7 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) { // not belongs to the same group, return the result of current group setInputDataBlock(pOperator, pAggInfo->binfo.pCtx, pBlock, TSDB_ORDER_ASC); - updateOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity, pBlock->info.rows * pAggInfo->resultRowFactor); + updateOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity, pBlock->info.rows * pAggInfo->resultRowFactor, pOperator->pRuntimeEnv); doExecuteFinalMerge(pOperator, pOperator->numOfOutput, pBlock); savePrevOrderColumns(pAggInfo->currentGroupColData, pAggInfo->groupColumnList, pBlock, 0, &pAggInfo->hasGroupColData); diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index b00138b4c46943933145241b3ca9e7ef47c4fcfe..c682138a354c312815060838120113e0f0f47004 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -170,6 +170,16 @@ void tscAddIntoStreamList(SSqlStream *pStream) { STscObj * pObj = pStream->pSql->pTscObj; pthread_mutex_lock(&pObj->mutex); + //check if newly added stream node is present + //in the streamList to prevent loop in the list + SSqlStream *iter = pObj->streamList; + while (iter) { + if (pStream == iter) { + pthread_mutex_unlock(&pObj->mutex); + return; + } + iter = iter->next; + } pStream->next = pObj->streamList; if (pObj->streamList) pObj->streamList->prev = pStream; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4d63438c9a5582981b86e41599b6b3a30314c9c9..c917d60eb69df2a2ead0556b1c6c015f6dc75340 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -433,7 +433,7 @@ int32_t readFromFile(char *name, uint32_t *len, void **buf) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - int fd = open(name, O_RDONLY); + int fd = open(name, O_RDONLY | O_BINARY); if (fd < 0) { tscError("open file %s failed, error:%s", name, strerror(errno)); tfree(*buf); @@ -1084,8 +1084,9 @@ static int32_t addPrimaryTsColumnForTimeWindowQuery(SQueryInfo* pQueryInfo, SSql uint64_t uid = tscExprGet(pQueryInfo, 0)->base.uid; int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; + STableMetaInfo* pTableMetaInfo = NULL; for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); if (pTableMetaInfo->pTableMeta->id.uid == uid) { tableIndex = i; break; @@ -1097,7 +1098,11 @@ static int32_t addPrimaryTsColumnForTimeWindowQuery(SQueryInfo* pQueryInfo, SSql } SSchema s = {.bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tstrncpy(s.name, aAggs[TSDB_FUNC_TS].name, sizeof(s.name)); + if (pTableMetaInfo) { + tstrncpy(s.name, pTableMetaInfo->pTableMeta->schema[PRIMARYKEY_TIMESTAMP_COL_INDEX].name, sizeof(s.name)); + } else { + tstrncpy(s.name, aAggs[TSDB_FUNC_TS].name, sizeof(s.name)); + } SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL, 0); @@ -1392,7 +1397,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* pSl const char* msg1 = "sliding value no larger than the interval value"; const char* msg2 = "sliding value can not less than 1% of interval value"; const char* msg3 = "does not support sliding when interval is natural month/year"; - const char* msg4 = "sliding not support for interp query"; + const char* msg4 = "sliding not support for interp query"; const static int32_t INTERVAL_SLIDING_FACTOR = 100; @@ -1410,7 +1415,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* pSl if (interpQuery) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } - + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -2668,6 +2673,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col const char* msg14 = "third parameter algorithm must be 'default' or 't-digest'"; const char* msg15 = "parameter is out of range [1, 1000]"; const char* msg16 = "elapsed duration should be greater than or equal to database precision"; + const char* msg17 = "elapsed/twa should not be used in nested query if inner query has group by clause"; switch (functionId) { case TSDB_FUNC_COUNT: { @@ -2727,7 +2733,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token,sizeof(pExpr->base.aliasName) - 1); - + SColumnList list = createColumnList(1, index.tableIndex, index.columnIndex); if (finalResult) { int32_t numOfOutput = tscNumOfFields(pQueryInfo); @@ -2792,6 +2798,17 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } } + //for timeline related aggregation function like elapsed and twa, groupby in subquery is not allowed + //as calculation result is meaningless by mixing different childtables(timelines) results. + if ((functionId == TSDB_FUNC_ELAPSED || functionId == TSDB_FUNC_TWA) && pQueryInfo->pUpstream != NULL) { + size_t numOfUpstreams = taosArrayGetSize(pQueryInfo->pUpstream); + for (int32_t i = 0; i < numOfUpstreams; ++i) { + SQueryInfo* pSub = taosArrayGetP(pQueryInfo->pUpstream, i); + if (pSub->groupbyExpr.numOfGroupCols > 0) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg17); + } + } + } STableComInfo info = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -2852,6 +2869,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col char val[8] = {0}; int64_t tickPerSec = 0; + char *exprToken = tcalloc(pParamElem[1].pNode->exprToken.n + 1, sizeof(char)); + memcpy(exprToken, pParamElem[1].pNode->exprToken.z, pParamElem[1].pNode->exprToken.n); + if (pParamElem[1].pNode->exprToken.type == TK_NOW || strstr(exprToken, "now")) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + tfree(exprToken); + if ((TSDB_DATA_TYPE_NULL == pParamElem[1].pNode->value.nType) || tVariantDump(&pParamElem[1].pNode->value, (char*) &tickPerSec, TSDB_DATA_TYPE_BIGINT, true) < 0) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -2866,7 +2890,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg10); } else if (tickPerSec <= 0) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg16); - } + } tscExprAddParams(&pExpr->base, (char*) &tickPerSec, TSDB_DATA_TYPE_BIGINT, LONG_BYTES); if (functionId == TSDB_FUNC_DERIVATIVE) { @@ -4906,14 +4930,14 @@ static int32_t validateNullExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t if (IS_VAR_DATA_TYPE(pSchema[index].type) || pSchema[index].type == TSDB_DATA_TYPE_JSON) { return TSDB_CODE_SUCCESS; } - + char *v = strndup(pRight->exprToken.z, pRight->exprToken.n); int32_t len = strRmquote(v, pRight->exprToken.n); if (len > 0) { uint32_t type = 0; tGetToken(v, &type); - if (type == TK_NULL) { + if (type == TK_NULL) { free(v); return invalidOperationMsg(msgBuf, msg); } @@ -5229,7 +5253,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql } } - if (pRight != NULL && (pRight->tokenId == TK_ID || pRight->tokenId == TK_ARROW)) { // join on tag columns for stable query + if (joinQuery && pRight != NULL && (pRight->tokenId == TK_ID || pRight->tokenId == TK_ARROW)) { // join on tag columns for stable query if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -6090,7 +6114,7 @@ int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t t // todo error !!!! int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { - const char rep[] = {'(', ')', '*', ',', '.', '/', '\\', '+', '-', '%', ' '}; + const char rep[] = {'(', ')', '*', ',', '.', '/', '\\', '+', '-', '%', ' ', '`'}; for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name; @@ -7265,6 +7289,7 @@ int32_t validateDNodeConfig(SMiscInfo* pOptions) { const int tokenMonitor = 3; const int tokenDebugFlag = 4; const int tokenDebugFlagEnd = 20; + const int tokenOfflineInterval = 21; const SDNodeDynConfOption cfgOptions[] = { {"resetLog", 8}, {"resetQueryCache", 15}, {"balance", 7}, {"monitor", 7}, {"debugFlag", 9}, {"monDebugFlag", 12}, {"vDebugFlag", 10}, {"mDebugFlag", 10}, @@ -7272,6 +7297,7 @@ int32_t validateDNodeConfig(SMiscInfo* pOptions) { {"uDebugFlag", 10}, {"tsdbDebugFlag", 13}, {"sDebugflag", 10}, {"rpcDebugFlag", 12}, {"dDebugFlag", 10}, {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"tmrDebugFlag", 12}, {"cqDebugFlag", 11}, + {"offlineInterval", 15}, }; SStrToken* pOptionToken = taosArrayGet(pOptions->a, 1); @@ -7303,6 +7329,14 @@ int32_t validateDNodeConfig(SMiscInfo* pOptions) { return TSDB_CODE_TSC_INVALID_OPERATION; // options value is invalid } return TSDB_CODE_SUCCESS; + } else if ((strncasecmp(cfgOptions[tokenOfflineInterval].name, pOptionToken->z, pOptionToken->n) == 0) && + (cfgOptions[tokenOfflineInterval].len == pOptionToken->n)) { + SStrToken* pValToken = taosArrayGet(pOptions->a, 2); + int32_t val = strtol(pValToken->z, NULL, 10); + if (val < 1 || val > 600) { + return TSDB_CODE_TSC_INVALID_OPERATION; // options value is invalid + } + return TSDB_CODE_SUCCESS; } else { SStrToken* pValToken = taosArrayGet(pOptions->a, 2); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 57362499a4fcaaa1500b199de8f63c07a03af898..7f8e9066af2ea05dd2cc50d8a9e156a5b44cb6cc 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1866,6 +1866,7 @@ int tscProcessRetrieveGlobalMergeRsp(SSqlObj *pSql) { bool convertJson = true; if (pQueryInfo->isStddev == true) convertJson = false; convertQueryResult(pRes, pQueryInfo, pSql->self, true, convertJson); + pRes->code = pQueryInfo->pQInfo->code; code = pRes->code; if (pRes->code == TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 322413a3cd7e637c477903b09522f60c11056885..e6027d304fa6ecc570e6ff98d0d54b898b5e1f6f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1479,6 +1479,18 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue break; } } + + // set input data order to param[1] + if(pex->base.functionId == TSDB_FUNC_FIRST || pex->base.functionId == TSDB_FUNC_FIRST_DST || + pex->base.functionId == TSDB_FUNC_LAST || pex->base.functionId == TSDB_FUNC_LAST_DST) { + // set input order + SQueryInfo* pInputQI = pSqlObjList[0]->cmd.pQueryInfo; + if(pInputQI) { + pex->base.numOfParams = 3; + pex->base.param[2].nType = TSDB_DATA_TYPE_INT; + pex->base.param[2].i64 = pInputQI->order.order; + } + } } tscDebug("0x%"PRIx64" create QInfo 0x%"PRIx64" to execute the main query while all nest queries are ready", pSql->self, pSql->self); @@ -4303,6 +4315,11 @@ void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { // create sub query to handle the sub query. SQueryInfo* pq = tscGetQueryInfo(&psub->cmd); + STableMetaInfo* pSubMeta = tscGetMetaInfo(pq, 0); + if (UTIL_TABLE_IS_SUPER_TABLE(pSubMeta) && + pq->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { + psub->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + } executeQuery(psub, pq); } diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index cde5eab48783351d4bd8c00be9008d52b5bf6561..890bed123bb1a03c93d676b1b12495c7a8b65ade 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -119,6 +119,7 @@ extern int32_t tsdbWalFlushSize; extern int8_t tsEnableBalance; extern int8_t tsAlternativeRole; extern int32_t tsBalanceInterval; +extern int32_t tsOfflineInterval; extern int32_t tsOfflineThreshold; extern int32_t tsMnodeEqualVnodeNum; extern int8_t tsEnableFlowCtrl; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 8627a3153cdac2b06cd3cf15dddefad32c39c58d..b87ed82d6963c98a95e087ddad2feb8f7590a8a7 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -164,6 +164,7 @@ int32_t tsdbWalFlushSize = TSDB_DEFAULT_WAL_FLUSH_SIZE; // MB int8_t tsEnableBalance = 1; int8_t tsAlternativeRole = 0; int32_t tsBalanceInterval = 300; // seconds +int32_t tsOfflineInterval = 3; int32_t tsOfflineThreshold = 86400 * 10; // seconds of 10 days int32_t tsMnodeEqualVnodeNum = 4; int8_t tsEnableFlowCtrl = 1; @@ -653,6 +654,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "offlineInterval"; + cfg.ptr = &tsOfflineInterval; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 1; + cfg.maxValue = 600; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + // 0-any; 1-mnode; 2-vnode cfg.option = "role"; cfg.ptr = &tsAlternativeRole; diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 420f462051687c72019d7c0697a23c940e4b8ae0..0580761de1c8768ed6fdb1c8f3ea6c7b4fa0836b 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -240,7 +240,7 @@ static void dnodeCheckDataDirOpenned(char *dir) { char filepath[256] = {0}; sprintf(filepath, "%s/.running", dir); - int fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + int fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); if (fd < 0) { dError("failed to open lock file:%s, reason: %s, quit", filepath, strerror(errno)); exit(0); diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index 22a6dc5b1993b6d15510b078ac4245909221ae78..ec09ab5d752cfbd4219787c0438c9b8bf4d1a9c4 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -266,7 +266,7 @@ static void* telemetryThread(void* param) { } static void dnodeGetEmail(char* filepath) { - int32_t fd = open(filepath, O_RDONLY); + int32_t fd = open(filepath, O_RDONLY | O_BINARY); if (fd < 0) { return; } diff --git a/src/inc/tfs.h b/src/inc/tfs.h index 11e33a3af791c3aef51c9d6ca876df2feb784473..1f16587536ddcc08770410cc34dc3b29b001eccb 100644 --- a/src/inc/tfs.h +++ b/src/inc/tfs.h @@ -70,7 +70,7 @@ typedef struct { #define TFILE_NAME(pf) ((pf)->aname) #define TFILE_REL_NAME(pf) ((pf)->rname) -#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags) +#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags | O_BINARY) #define tfsclose(fd) close(fd) #define tfsremove(pf) remove(TFILE_NAME(pf)) #define tfscopy(sf, df) taosCopy(TFILE_NAME(sf), TFILE_NAME(df)) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index a44e958be4345d4aa131cab8f616e0460624e8c1..d9c93e8cce4fd54e3417b7d335d05cfcba185c42 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -173,6 +173,7 @@ typedef void *TsdbQueryHandleT; // Use void to hide implementation details typedef struct STsdbQueryCond { STimeWindow twindow; int32_t order; // desc|asc order to iterate the data block + int64_t offset; // skip offset put down to tsdb int32_t numOfCols; SColumnInfo *colList; bool loadExternalRows; // load external rows or not @@ -391,6 +392,9 @@ void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCon int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist* pTableBlockInfo); +// obtain queryHandle attribute +int64_t tsdbSkipOffset(TsdbQueryHandleT queryHandle); + /** * get the statistics of repo usage * @param repo. point to the tsdbrepo diff --git a/src/kit/shell/src/shellCheck.c b/src/kit/shell/src/shellCheck.c index 43256719e125a712e6a52ddadaa9637498278092..dfc5d83b9fc820f7c5e08e5a26d2475f82d16040 100644 --- a/src/kit/shell/src/shellCheck.c +++ b/src/kit/shell/src/shellCheck.c @@ -131,7 +131,7 @@ static void *shellCheckThreadFp(void *arg) { char *tbname = tbNames[t]; if (tbname == NULL) break; - snprintf(sql, SHELL_SQL_LEN, "select last_row(_c0) from %s;", tbname); + snprintf(sql, SHELL_SQL_LEN, "select count(*) from %s;", tbname); TAOS_RES *pSql = taos_query(pThread->taos, sql); int32_t code = taos_errno(pSql); diff --git a/src/kit/taos-tools b/src/kit/taos-tools index 27751ba9ca17407425fb50a52cd68295794dedc3..59f00a69f36b08cea86a70a22c29b2c27ef506ae 160000 --- a/src/kit/taos-tools +++ b/src/kit/taos-tools @@ -1 +1 @@ -Subproject commit 27751ba9ca17407425fb50a52cd68295794dedc3 +Subproject commit 59f00a69f36b08cea86a70a22c29b2c27ef506ae diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index aff0411fdd777f83ccc6a882fbe91d7bc909e16b..bda4bbf3201cd0d425383304bfcffd526d244955 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -43,7 +43,7 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_ int32_t mnodeCreateVgroup(struct SMnodeMsg *pMsg); void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle); void mnodeAlterVgroup(SVgObj *pVgroup, void *ahandle); -int32_t mnodeGetAvailableVgroup(struct SMnodeMsg *pMsg, SVgObj **pVgroup, int32_t *sid); +int32_t mnodeGetAvailableVgroup(struct SMnodeMsg *pMsg, SVgObj **pVgroup, int32_t *sid, int32_t vgId); int32_t mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable, bool needCheck); void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 4f277efd34bdb1d04c227919d36fa707ca1917bb..2b49dcbcef679e8d54367a8d524657d02314b67f 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -48,6 +48,12 @@ #define CREATE_CTABLE_RETRY_TIMES 10 #define CREATE_CTABLE_RETRY_SEC 14 +// informal +#define META_SYNC_TABLE_NAME "_taos_meta_sync_table_name_taos_" +#define META_SYNC_TABLE_NAME_LEN 32 +static int32_t tsMetaSyncOption = 0; +// informal + int64_t tsCTableRid = -1; static void * tsChildTableSdb; int64_t tsSTableRid = -1; @@ -1726,6 +1732,9 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, cols++; numOfRows++; + + mDebug("stable: %s, uid: %" PRIu64, prefix, pTable->uid); + mnodeDecTableRef(pTable); } @@ -2227,9 +2236,19 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { if (pMsg->pTable == NULL) { SVgObj *pVgroup = NULL; int32_t tid = 0; - code = mnodeGetAvailableVgroup(pMsg, &pVgroup, &tid); + int32_t vgId = 0; + + if (tsMetaSyncOption) { + char *pTbName = strchr(pCreate->tableName, '.'); + if (pTbName && (pTbName = strchr(pTbName + 1, '.'))) { + if (0 == strncmp(META_SYNC_TABLE_NAME, ++pTbName, META_SYNC_TABLE_NAME_LEN)) { + vgId = atoi(pTbName + META_SYNC_TABLE_NAME_LEN); + } + } + } + code = mnodeGetAvailableVgroup(pMsg, &pVgroup, &tid, vgId); if (code != TSDB_CODE_SUCCESS) { - mDebug("msg:%p, app:%p table:%s, failed to get available vgroup, reason:%s", pMsg, pMsg->rpcMsg.ahandle, + mError("msg:%p, app:%p table:%s, failed to get available vgroup, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName, tstrerror(code)); return code; } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index fd6d60c034c702e12a5d996f5b130e54bf3c6a4f..46dc07f0f2b4db9d9a0018aa8c19fe3a8b0fcbea 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -428,10 +428,47 @@ static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) { return TSDB_CODE_SUCCESS; } -int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSid) { +int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSid, int32_t vgId) { SDbObj *pDb = pMsg->pDb; pthread_mutex_lock(&pDb->mutex); + if (vgId > 0) { + for (int32_t v = 0; v < pDb->numOfVgroups; ++v) { + SVgObj *pVgroup = pDb->vgList[v]; + if (pVgroup == NULL) { + mError("db:%s, vgroup: %d is null", pDb->name, v); + pthread_mutex_unlock(&pDb->mutex); + return TSDB_CODE_MND_APP_ERROR; + } + + if (pVgroup->vgId != (uint32_t)vgId) { // find the target vgId + continue; + } + + int32_t sid = taosAllocateId(pVgroup->idPool); + if (sid <= 0) { + int curMaxId = taosIdPoolMaxSize(pVgroup->idPool); + if ((taosUpdateIdPool(pVgroup->idPool, curMaxId + 1) < 0) || ((sid = taosAllocateId(pVgroup->idPool)) <= 0)) { + mError("msg:%p, app:%p db:%s, no enough sid in vgId:%d", pMsg, pMsg->rpcMsg.ahandle, pDb->name, + pVgroup->vgId); + pthread_mutex_unlock(&pDb->mutex); + return TSDB_CODE_MND_APP_ERROR; + } + } + mDebug("vgId:%d, alloc tid:%d", pVgroup->vgId, sid); + + *pSid = sid; + *ppVgroup = pVgroup; + pDb->vgListIndex = v; + + pthread_mutex_unlock(&pDb->mutex); + return TSDB_CODE_SUCCESS; + } + pthread_mutex_unlock(&pDb->mutex); + mError("db:%s, vgroup: %d not exist", pDb->name, vgId); + return TSDB_CODE_MND_APP_ERROR; + } + for (int32_t v = 0; v < pDb->numOfVgroups; ++v) { int vgIndex = (v + pDb->vgListIndex) % pDb->numOfVgroups; SVgObj *pVgroup = pDb->vgList[vgIndex]; diff --git a/src/os/src/detail/osRand.c b/src/os/src/detail/osRand.c index 0dda908bb35c68513dba150e8380846c36aa2893..e1d81ea5d3ed1fccd0b8b96cb8c3991475f9c714 100644 --- a/src/os/src/detail/osRand.c +++ b/src/os/src/detail/osRand.c @@ -22,7 +22,7 @@ uint32_t taosSafeRand(void) { int fd; int seed; - fd = open("/dev/urandom", 0); + fd = open("/dev/urandom", 0 | O_BINARY); if (fd < 0) { seed = (int)time(0); } else { diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index 0542407c3ba8e8d17c79f16ef0f3560e3bc10693..06c58d43067ce5941975f97c169a2718640bac2a 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -713,7 +713,7 @@ bool taosGetSystemUid(char *uid) { int fd; int len = 0; - fd = open("/proc/sys/kernel/random/uuid", 0); + fd = open("/proc/sys/kernel/random/uuid", 0 | O_BINARY); if (fd < 0) { return false; } else { diff --git a/src/os/src/detail/osTimer.c b/src/os/src/detail/osTimer.c index 618df8a8bad451984fafd022a33a799986a48422..bc5119107a312b5f281263823d766e9ce506a85a 100644 --- a/src/os/src/detail/osTimer.c +++ b/src/os/src/detail/osTimer.c @@ -20,6 +20,7 @@ #if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) || defined(_TD_DARWIN_64)) +#ifndef _ALPINE static void taosDeleteTimer(void *tharg) { timer_t *pTimer = tharg; timer_delete(*pTimer); @@ -105,4 +106,41 @@ void taosUninitTimer() { pthread_join(timerThread, NULL); } +#else + +static timer_t timerId; + +void sig_alrm_handler(union sigval sv) { + void (*callback)(int) = sv.sival_ptr; + callback(0); +} +int taosInitTimer(void (*callback)(int), int ms) { + struct sigevent evp; + memset((void *)&evp, 0, sizeof(evp)); + evp.sigev_notify = SIGEV_THREAD; + evp.sigev_notify_function = &sig_alrm_handler; + evp.sigev_signo = SIGALRM; + evp.sigev_value.sival_ptr = (void *)callback; + + struct itimerspec ts; + ts.it_value.tv_sec = 0; + ts.it_value.tv_nsec = 1000000 * MSECONDS_PER_TICK; + ts.it_interval.tv_sec = 0; + ts.it_interval.tv_nsec = 1000000 * MSECONDS_PER_TICK; + if (timer_create(CLOCK_REALTIME, &evp, &timerId)) { + uError("Failed to create timer"); + return -1; + } + + if (timer_settime(timerId, 0, &ts, NULL)) { + uError("Failed to init timer"); + return -1; + } + return 0; +} + +void taosUninitTimer() { + timer_delete(timerId); +} +#endif #endif diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index 4de93a5f0c3006969b0f8e80570d0fc915672f4e..44b563dd31741add8bafd95ddb7d7890494cc7d9 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -34,7 +34,7 @@ char* taosGetCmdlineByPID(int pid) { static char cmdline[1024]; sprintf(cmdline, "/proc/%d/cmdline", pid); - int fd = open(cmdline, O_RDONLY); + int fd = open(cmdline, O_RDONLY | O_BINARY); if (fd >= 0) { int n = read(fd, cmdline, sizeof(cmdline) - 1); if (n < 0) n = 0; diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index a2452a16b94fea060a370c86518bb36c1da45070..2c18904d2a9cc2d2dca57c406134518028daba9b 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -156,7 +156,10 @@ void httpSendErrorResp(HttpContext *pContext, int32_t errNo) { HttpServer *pServer = &tsHttpServer; SMonHttpStatus *httpStatus = monGetHttpStatusHashTableEntry(httpCode); - pServer->statusCodeErrs[httpStatus->index] += 1; + // FIXME(@huolinhe): I don't known why the errors index is overflowed, but fix it by index check + if (httpStatus->index < HTTP_STATUS_CODE_NUM) { + pServer->statusCodeErrs[httpStatus->index] += 1; + } pContext->error = true; diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c index a03bc09036d14045043704e82e22fdd177c243b2..68bd98dd5e0ed343e9a9966a8e75ffe4493a4cfb 100644 --- a/src/plugins/monitor/src/monMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -171,7 +171,6 @@ static void monSaveSystemInfo(); static void monSaveClusterInfo(); static void monSaveDnodesInfo(); static void monSaveVgroupsInfo(); -static void monSaveSlowQueryInfo(); static void monSaveDisksInfo(); static void monSaveGrantsInfo(); static void monSaveHttpReqInfo(); @@ -321,7 +320,6 @@ static void *monThreadFunc(void *param) { monSaveClusterInfo(); } monSaveVgroupsInfo(); - monSaveSlowQueryInfo(); monSaveDisksInfo(); monSaveGrantsInfo(); monSaveHttpReqInfo(); @@ -383,9 +381,9 @@ static void monBuildMonitorSql(char *sql, int32_t cmd) { tsMonitorDbName, TSDB_DEFAULT_USER); } else if (cmd == MON_CMD_CREATE_TB_SLOWQUERY) { snprintf(sql, SQL_LENGTH, - "create table if not exists %s.slowquery(ts timestamp, query_id " - "binary(%d), username binary(%d), qid binary(%d), created_time timestamp, time bigint, end_point binary(%d), sql binary(%d))", - tsMonitorDbName, QUERY_ID_LEN, TSDB_TABLE_FNAME_LEN - 1, QUERY_ID_LEN, TSDB_EP_LEN, TSDB_SLOW_QUERY_SQL_LEN); + "create table if not exists %s.slowquery(ts timestamp, username " + "binary(%d), created_time timestamp, time bigint, sql binary(%d))", + tsMonitorDbName, TSDB_TABLE_FNAME_LEN - 1, TSDB_SLOW_QUERY_SQL_LEN); } else if (cmd == MON_CMD_CREATE_TB_LOG) { snprintf(sql, SQL_LENGTH, "create table if not exists %s.log(ts timestamp, level tinyint, " @@ -460,14 +458,18 @@ static void monBuildMonitorSql(char *sql, int32_t cmd) { ", expire_time int, timeseries_used int, timeseries_total int)", tsMonitorDbName); } else if (cmd == MON_CMD_CREATE_MT_RESTFUL) { + int usedLen = 0, len = 0; int pos = snprintf(sql, SQL_LENGTH, "create table if not exists %s.restful_info(ts timestamp", tsMonitorDbName); + usedLen += pos; for (int i = 0; i < tListLen(monHttpStatusTable); ++i) { - pos += snprintf(sql + pos, SQL_LENGTH, ", `%s(%d)` int", + len = snprintf(sql + pos, SQL_LENGTH - usedLen, ", %s_%d int", monHttpStatusTable[i].name, monHttpStatusTable[i].code); + usedLen += len; + pos += len; } - snprintf(sql + pos, SQL_LENGTH, + snprintf(sql + pos, SQL_LENGTH - usedLen, ") tags (dnode_id int, dnode_ep binary(%d))", TSDB_EP_LEN); } else if (cmd == MON_CMD_CREATE_TB_RESTFUL) { @@ -1213,91 +1215,6 @@ static void monSaveVgroupsInfo() { taos_free_result(result); } -static void monSaveSlowQueryInfo() { - int64_t ts = taosGetTimestampUs(); - char * sql = tsMonitor.sql; - int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.slowquery values(%" PRId64, tsMonitorDbName, ts); - bool has_slowquery = false; - - TAOS_RES *result = taos_query(tsMonitor.conn, "show queries"); - int32_t code = taos_errno(result); - if (code != TSDB_CODE_SUCCESS) { - monError("failed to execute cmd: show queries, reason:%s", tstrerror(code)); - } - - TAOS_ROW row; - int32_t num_fields = taos_num_fields(result); - TAOS_FIELD *fields = taos_fetch_fields(result); - - int32_t charLen; - while ((row = taos_fetch_row(result))) { - for (int i = 0; i < num_fields; ++i) { - if (strcmp(fields[i].name, "query_id") == 0) { - has_slowquery = true; - charLen = monGetRowElemCharLen(fields[i], (char *)row[i]); - if (charLen < 0) { - monError("failed to save slow_query info, reason: invalid row %s len, sql:%s", (char *)row[i], tsMonitor.sql); - goto DONE; - } - pos += snprintf(sql + pos, strlen(SQL_STR_FMT) + charLen + 1, ", "SQL_STR_FMT, (char *)row[i]); - } else if (strcmp(fields[i].name, "user") == 0) { - charLen = monGetRowElemCharLen(fields[i], (char *)row[i]); - if (charLen < 0) { - monError("failed to save slow_query info, reason: invalid row %s len, sql:%s", (char *)row[i], tsMonitor.sql); - goto DONE; - } - pos += snprintf(sql + pos, strlen(SQL_STR_FMT) + charLen + 1, ", "SQL_STR_FMT, (char *)row[i]); - } else if (strcmp(fields[i].name, "qid") == 0) { - charLen = monGetRowElemCharLen(fields[i], (char *)row[i]); - if (charLen < 0) { - monError("failed to save slow_query info, reason: invalid row %s len, sql:%s", (char *)row[i], tsMonitor.sql); - goto DONE; - } - pos += snprintf(sql + pos, strlen(SQL_STR_FMT) + charLen + 1, ", "SQL_STR_FMT, (char *)row[i]); - } else if (strcmp(fields[i].name, "created_time") == 0) { - int64_t create_time = *(int64_t *)row[i]; - create_time = convertTimePrecision(create_time, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO); - pos += snprintf(sql + pos, SQL_LENGTH, ", %" PRId64 "", create_time); - } else if (strcmp(fields[i].name, "time") == 0) { - pos += snprintf(sql + pos, SQL_LENGTH, ", %" PRId64 "", *(int64_t *)row[i]); - } else if (strcmp(fields[i].name, "ep") == 0) { - charLen = monGetRowElemCharLen(fields[i], (char *)row[i]); - if (charLen < 0) { - monError("failed to save slow_query info, reason: invalid row %s len, sql:%s", (char *)row[i], tsMonitor.sql); - goto DONE; - } - pos += snprintf(sql + pos, strlen(SQL_STR_FMT) + charLen + 1, ", "SQL_STR_FMT, (char *)row[i]); - } else if (strcmp(fields[i].name, "sql") == 0) { - charLen = monGetRowElemCharLen(fields[i], (char *)row[i]); - if (charLen < 0) { - monError("failed to save slow_query info, reason: invalid row %s len, sql:%s", (char *)row[i], tsMonitor.sql); - goto DONE; - } - pos += snprintf(sql + pos, strlen(SQL_STR_FMT) + charLen + 2, ", "SQL_STR_FMT")", (char *)row[i]); - } - } - } - - monDebug("save slow query, sql:%s", sql); - if (!has_slowquery) { - goto DONE; - } - void *res = taos_query(tsMonitor.conn, tsMonitor.sql); - code = taos_errno(res); - taos_free_result(res); - - if (code != 0) { - monError("failed to save slowquery info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql); - } else { - monIncSubmitReqCnt(); - monDebug("successfully to save slowquery info, sql:%s", tsMonitor.sql); - } - -DONE: - taos_free_result(result); - return; -} - static void monSaveDisksInfo() { int64_t ts = taosGetTimestampUs(); char * sql = tsMonitor.sql; diff --git a/src/plugins/taosadapter b/src/plugins/taosadapter index 273b5219f8bcc604e43beebc6f1f95abed85170a..47fb0b3e627ddadf1ca983c1d75b9a4e44cd98fd 160000 --- a/src/plugins/taosadapter +++ b/src/plugins/taosadapter @@ -1 +1 @@ -Subproject commit 273b5219f8bcc604e43beebc6f1f95abed85170a +Subproject commit 47fb0b3e627ddadf1ca983c1d75b9a4e44cd98fd diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index ba277b23018a58e3ed29122761aa65506c94078a..f399cbc7e12bd7f54b34bb03e792f8b9023870ec 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -237,6 +237,7 @@ typedef struct SQueryAttr { bool createFilterOperator; // if filter operator is needed bool multigroupResult; // multigroup result can exist in one SSDataBlock bool needSort; // need sort rowRes + bool skipOffset; // can skip offset if true int32_t interBufSize; // intermediate buffer sizse int32_t havingNum; // having expr number @@ -659,7 +660,7 @@ void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFil void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput); void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); -void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows); +void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows, SQueryRuntimeEnv* runtimeEnv); void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity); void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 6b8e31b181559c3d2e92cb52c5b50d4261c66611..9e80a4fb62d5e20bc0771714a2dfb82f66dae8d9 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -1620,33 +1620,65 @@ static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* // todo opt for null block static void first_function(SQLFunctionCtx *pCtx) { - if (pCtx->order == TSDB_ORDER_DESC) { - return; - } - + SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); int32_t notNullElems = 0; - - // handle the null value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - memcpy(pCtx->pOutput, data, pCtx->inputBytes); - if (pCtx->ptsList != NULL) { - TSKEY k = GET_TS_DATA(pCtx, i); - DO_UPDATE_TAG_COLUMNS(pCtx, k); + int32_t step = 1; + int32_t i = 0; + bool inputAsc = true; + + // input data come from sub query, input data order equal to sub query order + if(pCtx->numOfParams == 3) { + if(pCtx->param[2].nType == TSDB_DATA_TYPE_INT && pCtx->param[2].i64 == TSDB_ORDER_DESC) { + step = -1; + i = pCtx->size - 1; + inputAsc = false; + } + } else if (pCtx->order == TSDB_ORDER_DESC) { + return ; + } + + if(pCtx->order == TSDB_ORDER_ASC && inputAsc) { + for (int32_t m = 0; m < pCtx->size; ++m, i+=step) { + char *data = GET_INPUT_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; + } + + memcpy(pCtx->pOutput, data, pCtx->inputBytes); + if (pCtx->ptsList != NULL) { + TSKEY k = GET_TS_DATA(pCtx, i); + DO_UPDATE_TAG_COLUMNS(pCtx, k); + } + + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); + pInfo->hasResult = DATA_SET_FLAG; + pInfo->complete = true; + + notNullElems++; + break; } + } else { // desc order + for (int32_t m = 0; m < pCtx->size; ++m, i+=step) { + char *data = GET_INPUT_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { + continue; + } - SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->complete = true; - - notNullElems++; - break; + TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; + + char* buf = GET_ROWCELL_INTERBUF(pResInfo); + if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) > ts) { + pResInfo->hasResult = DATA_SET_FLAG; + memcpy(pCtx->pOutput, data, pCtx->inputBytes); + + *(TSKEY*)buf = ts; + DO_UPDATE_TAG_COLUMNS(pCtx, ts); + } + + notNullElems++; + break; + } } - SET_VAL(pCtx, notNullElems, 1); } @@ -1730,16 +1762,23 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { * least one data in this block that is not null.(TODO opt for this case) */ static void last_function(SQLFunctionCtx *pCtx) { - if (pCtx->order != pCtx->param[0].i64) { + SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); + int32_t notNullElems = 0; + int32_t step = -1; + int32_t i = pCtx->size - 1; + + // input data come from sub query, input data order equal to sub query order + if(pCtx->numOfParams == 3) { + if(pCtx->param[2].nType == TSDB_DATA_TYPE_INT && pCtx->param[2].i64 == TSDB_ORDER_DESC) { + step = 1; + i = 0; + } + } else if (pCtx->order != pCtx->param[0].i64) { return; } - SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; if (pCtx->order == TSDB_ORDER_DESC) { - - for (int32_t i = pCtx->size - 1; i >= 0; --i) { + for (int32_t m = pCtx->size - 1; m >= 0; --m, i += step) { char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { continue; @@ -1756,7 +1795,7 @@ static void last_function(SQLFunctionCtx *pCtx) { break; } } else { // ascending order - for (int32_t i = pCtx->size - 1; i >= 0; --i) { + for (int32_t m = pCtx->size - 1; m >= 0; --m, i += step) { char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { continue; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6346e743081a6594fcc9e8d8001ae18e3f90ac92..310c82ad04e6ee0e41a93b12f8e23c26d7692259 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1432,7 +1432,7 @@ static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBloc int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); if (pBlock->pDataBlock == NULL){ - tscError("pBlock->pDataBlock == NULL"); + qError("window border interpolation: pBlock->pDataBlock == NULL"); return; } SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0); @@ -3586,7 +3586,7 @@ void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, i initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); } -void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows) { +void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows, SQueryRuntimeEnv* runtimeEnv) { SSDataBlock* pDataBlock = pBInfo->pRes; int32_t newSize = pDataBlock->info.rows + numOfInputRows + 5; // extra output buffer @@ -3594,7 +3594,7 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - char* p = realloc(pColInfo->pData, newSize * pColInfo->info.bytes); + char* p = realloc(pColInfo->pData, ((size_t)newSize) * pColInfo->info.bytes); if (p != NULL) { pColInfo->pData = p; @@ -3602,7 +3602,10 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf pBInfo->pCtx[i].pOutput = pColInfo->pData; (*bufCapacity) = newSize; } else { - // longjmp + size_t allocateSize = ((size_t)(newSize)) * pColInfo->info.bytes; + qError("can not allocate %zu bytes for output. Rows: %d, colBytes %d", + allocateSize, newSize, pColInfo->info.bytes); + longjmp(runtimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } } } @@ -3610,7 +3613,7 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - pBInfo->pCtx[i].pOutput = pColInfo->pData + pColInfo->info.bytes * pDataBlock->info.rows; + pBInfo->pCtx[i].pOutput = pColInfo->pData + (size_t)pColInfo->info.bytes * pDataBlock->info.rows; // set the correct pointer after the memory buffer reallocated. int32_t functionId = pBInfo->pCtx[i].functionId; @@ -4902,6 +4905,11 @@ STsdbQueryCond createTsdbQueryCond(SQueryAttr* pQueryAttr, STimeWindow* win) { .loadExternalRows = false, }; + // set offset with + if(pQueryAttr->skipOffset) { + cond.offset = pQueryAttr->limit.offset; + } + TIME_WINDOW_COPY(cond.twindow, *win); return cond; } @@ -5604,6 +5612,18 @@ static int32_t getTableScanOrder(STableScanInfo* pTableScanInfo) { return pTableScanInfo->order; } +// check all SQLFunctionCtx is completed +static bool allCtxCompleted(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx) { + // only one false, return false + for(int32_t i = 0; i < pOperator->numOfOutput; i++) { + if(pCtx[i].resultInfo == NULL) + return false; + if(!pCtx[i].resultInfo->complete) + return false; + } + return true; +} + // this is a blocking operator static SSDataBlock* doAggregate(void* param, bool* newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; @@ -5642,6 +5662,9 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); doAggregateImpl(pOperator, pQueryAttr->window.skey, pInfo->pCtx, pBlock); + // if all pCtx is completed, then query should be over + if(allCtxCompleted(pOperator, pInfo->pCtx)) + break; } doSetOperatorCompleted(pOperator); @@ -5752,7 +5775,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); + updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows, pOperator->pRuntimeEnv); projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); if (pTableQueryInfo != NULL) { @@ -5818,7 +5841,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); + updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows, pOperator->pRuntimeEnv); projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); if (pTableQueryInfo != NULL) { @@ -5855,19 +5878,38 @@ static SSDataBlock* doLimit(void* param, bool* newgroup) { return NULL; } + bool move = false; + int32_t skip = 0; + int32_t remain = 0; + int64_t srows = tsdbSkipOffset(pRuntimeEnv->pQueryHandle); + if (pRuntimeEnv->currentOffset == 0) { break; + } + else if(srows > 0) { + if(pRuntimeEnv->currentOffset - srows >= pBlock->info.rows) { + pRuntimeEnv->currentOffset -= pBlock->info.rows; + } else { + move = true; + skip = (int32_t)(pRuntimeEnv->currentOffset - srows); + remain = (int32_t)(pBlock->info.rows - skip); + } } else if (pRuntimeEnv->currentOffset >= pBlock->info.rows) { pRuntimeEnv->currentOffset -= pBlock->info.rows; } else { - int32_t remain = (int32_t)(pBlock->info.rows - pRuntimeEnv->currentOffset); + move = true; + skip = (int32_t)pRuntimeEnv->currentOffset; + remain = (int32_t)(pBlock->info.rows - pRuntimeEnv->currentOffset); + } + + // need move + if(move) { pBlock->info.rows = remain; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); int16_t bytes = pColInfoData->info.bytes; - memmove(pColInfoData->pData, pColInfoData->pData + bytes * pRuntimeEnv->currentOffset, remain * bytes); + memmove(pColInfoData->pData, pColInfoData->pData + skip * bytes, remain * bytes); } pRuntimeEnv->currentOffset = 0; @@ -6315,7 +6357,7 @@ static void doTimeEveryImpl(SOperatorInfo* pOperator, SQLFunctionCtx *pCtx, SSDa break; } - updateOutputBuf(&pEveryInfo->binfo, &pEveryInfo->bufCapacity, 0); + updateOutputBuf(&pEveryInfo->binfo, &pEveryInfo->bufCapacity, 0, pOperator->pRuntimeEnv); } } } @@ -6335,7 +6377,7 @@ static SSDataBlock* doTimeEvery(void* param, bool* newgroup) { pRes->info.rows = 0; if (!pEveryInfo->groupDone) { - updateOutputBuf(&pEveryInfo->binfo, &pEveryInfo->bufCapacity, 0); + updateOutputBuf(&pEveryInfo->binfo, &pEveryInfo->bufCapacity, 0, pOperator->pRuntimeEnv); doTimeEveryImpl(pOperator, pInfo->pCtx, pEveryInfo->lastBlock, false); if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) { copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); @@ -6371,7 +6413,7 @@ static SSDataBlock* doTimeEvery(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pEveryInfo->binfo, &pEveryInfo->bufCapacity, pBlock->info.rows); + updateOutputBuf(&pEveryInfo->binfo, &pEveryInfo->bufCapacity, pBlock->info.rows, pOperator->pRuntimeEnv); doTimeEveryImpl(pOperator, pInfo->pCtx, pBlock, *newgroup); if (pEveryInfo->groupDone && pOperator->upstream[0]->notify) { @@ -6397,7 +6439,7 @@ static SSDataBlock* doTimeEvery(void* param, bool* newgroup) { if (!pEveryInfo->groupDone) { pEveryInfo->allDone = true; - updateOutputBuf(&pEveryInfo->binfo, &pEveryInfo->bufCapacity, 0); + updateOutputBuf(&pEveryInfo->binfo, &pEveryInfo->bufCapacity, 0, pOperator->pRuntimeEnv); doTimeEveryImpl(pOperator, pInfo->pCtx, NULL, false); if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) { break; @@ -6418,7 +6460,7 @@ static SSDataBlock* doTimeEvery(void* param, bool* newgroup) { // Return result of the previous group in the firstly. if (*newgroup) { if (!pEveryInfo->groupDone) { - updateOutputBuf(&pEveryInfo->binfo, &pEveryInfo->bufCapacity, 0); + updateOutputBuf(&pEveryInfo->binfo, &pEveryInfo->bufCapacity, 0, pOperator->pRuntimeEnv); doTimeEveryImpl(pOperator, pInfo->pCtx, NULL, false); if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) { pEveryInfo->existDataBlock = pBlock; @@ -6454,7 +6496,7 @@ static SSDataBlock* doTimeEvery(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pEveryInfo->binfo, &pEveryInfo->bufCapacity, pBlock->info.rows); + updateOutputBuf(&pEveryInfo->binfo, &pEveryInfo->bufCapacity, pBlock->info.rows, pOperator->pRuntimeEnv); pEveryInfo->groupDone = false; diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index b0015e39b96e0754377abece6e12045b0f36a901..9694dac7db152d1b5851629fe349655002c474e8 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -101,7 +101,6 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); } } else if (pFillInfo->type == TSDB_FILL_LINEAR) { - // TODO : linear interpolation supports NULL value if (prev != NULL && !outOfBound) { for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; @@ -121,6 +120,10 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData bool exceedMax = false, exceedMin = false; point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->col.offset}; point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes}; + if (isNull(point1.val, type) || isNull(point2.val, type)) { + setNull(val1, pCol->col.type, bytes); + continue; + } point = (SPoint){.key = pFillInfo->currentKey, .val = val1}; taosGetLinearInterpolationVal(&point, type, &point1, &point2, type, &exceedMax, &exceedMin); } diff --git a/src/rpc/test/rserver.c b/src/rpc/test/rserver.c index 64960db0446413ebce1978b7fe310b6a34c34f1c..767b756badcbd3e2ffdf7908a19aa61e86ac8f1b 100644 --- a/src/rpc/test/rserver.c +++ b/src/rpc/test/rserver.c @@ -172,7 +172,7 @@ int main(int argc, char *argv[]) { tInfo("RPC server is running, ctrl-c to exit"); if (commit) { - dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); + dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); if (dataFd<0) tInfo("failed to open data file, reason:%s", strerror(errno)); } diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c index 4598e16a9d05be29d11612755a079ce0a228a2ff..13010783962273f8ae5c5f68bb16e8480a8dacf9 100644 --- a/src/sync/test/syncServer.c +++ b/src/sync/test/syncServer.c @@ -43,7 +43,7 @@ int writeIntoWal(SWalHead *pHead) { char walName[280]; snprintf(walName, sizeof(walName), "%s/wal/wal.%d", path, walNum); (void)remove(walName); - dataFd = open(walName, O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); + dataFd = open(walName, O_CREAT | O_WRONLY | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); if (dataFd < 0) { uInfo("failed to open wal file:%s(%s)", walName, strerror(errno)); return -1; diff --git a/src/tfs/src/tfs.c b/src/tfs/src/tfs.c index b3aabe177bd4c34151cbe2778825bed6262679ab..63266f8d92c0051306542c683851cdfb6ed50275 100644 --- a/src/tfs/src/tfs.c +++ b/src/tfs/src/tfs.c @@ -498,7 +498,11 @@ static int tfsFormatDir(char *idir, char *odir) { } char tmp[PATH_MAX] = {0}; +#ifdef WINDOWS + if (_fullpath(tmp,wep.we_wordv[0], PATH_MAX) == NULL) { +#else if (realpath(wep.we_wordv[0], tmp) == NULL) { +#endif terrno = TAOS_SYSTEM_ERROR(errno); wordfree(&wep); return -1; diff --git a/src/tsdb/inc/tsdbFile.h b/src/tsdb/inc/tsdbFile.h index dfef13b51ecc4692f80cc6dbd937e70911228cf8..6d1e0cf2461a28dbcf481c7dc93d651551c0453d 100644 --- a/src/tsdb/inc/tsdbFile.h +++ b/src/tsdb/inc/tsdbFile.h @@ -89,7 +89,7 @@ static FORCE_INLINE void tsdbSetMFileInfo(SMFile* pMFile, SMFInfo* pInfo) { pMFi static FORCE_INLINE int tsdbOpenMFile(SMFile* pMFile, int flags) { ASSERT(TSDB_FILE_CLOSED(pMFile)); - pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), flags); + pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), flags | O_BINARY); if (pMFile->fd < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -204,7 +204,7 @@ static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFi static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { ASSERT(!TSDB_FILE_OPENED(pDFile)); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags); + pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags | O_BINARY); if (pDFile->fd < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index be734ce0cccad6827cba4e2c27d0be478af92af3..cf4d190ff3511ed25cc49fc34296e2dcd3dfb3b4 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -39,6 +39,9 @@ .tid = (_checkInfo)->tableId.tid, \ .uid = (_checkInfo)->tableId.uid}) +// limit offset start optimization for rows read over this value +#define OFFSET_SKIP_THRESHOLD 5000 + enum { TSDB_QUERY_TYPE_ALL = 1, TSDB_QUERY_TYPE_LAST = 2, @@ -117,6 +120,9 @@ typedef struct STsdbQueryHandle { STsdbRepo* pTsdb; SQueryFilePos cur; // current position int16_t order; + int64_t offset; // limit offset + int64_t srows; // skip offset rows + int64_t frows; // forbid skip offset rows STimeWindow window; // the primary query time window that applies to all queries SDataStatis* statis; // query level statistics, only one table block statistics info exists at any time int32_t numOfBlocks; @@ -155,6 +161,11 @@ typedef struct STableGroupSupporter { STSchema* pTagSchema; } STableGroupSupporter; +typedef struct SRange { + int32_t from; + int32_t to; +} SRange; + static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle); @@ -413,6 +424,9 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC } pQueryHandle->order = pCond->order; + pQueryHandle->offset = pCond->offset; + pQueryHandle->srows = 0; + pQueryHandle->frows = 0; pQueryHandle->pTsdb = tsdb; pQueryHandle->type = TSDB_QUERY_TYPE_ALL; pQueryHandle->cur.fid = INT32_MIN; @@ -529,6 +543,9 @@ void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond) { } pQueryHandle->order = pCond->order; + pQueryHandle->offset = pCond->offset; + pQueryHandle->srows = 0; + pQueryHandle->frows = 0; pQueryHandle->window = pCond->twindow; pQueryHandle->type = TSDB_QUERY_TYPE_ALL; pQueryHandle->cur.fid = -1; @@ -596,6 +613,12 @@ void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCon static int32_t lazyLoadCacheLast(STsdbQueryHandle* pQueryHandle) { STsdbRepo* pRepo = pQueryHandle->pTsdb; + if (!pQueryHandle->pTableCheckInfo) { + tsdbError("%p table check info is NULL", pQueryHandle); + terrno = TSDB_CODE_QRY_APP_ERROR; + return -1; + } + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); int32_t code = 0; for (size_t i = 0; i < numOfTables; ++i) { @@ -628,7 +651,9 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable return NULL; } - lazyLoadCacheLast(pQueryHandle); + if (lazyLoadCacheLast(pQueryHandle) != TSDB_CODE_SUCCESS) { + return NULL; + } int32_t code = checkForCachedLastRow(pQueryHandle, groupList); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 @@ -650,7 +675,9 @@ TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STab return NULL; } - lazyLoadCacheLast(pQueryHandle); + if (lazyLoadCacheLast(pQueryHandle) != TSDB_CODE_SUCCESS) { + return NULL; + } int32_t code = checkForCachedLast(pQueryHandle); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 @@ -1063,63 +1090,302 @@ static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY s return midSlot; } -static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int32_t* numOfBlocks) { - int32_t code = 0; +// array :1 2 3 5 7 -2 (8 9) skip 4 and 6 +int32_t memMoveByArray(SBlock *blocks, SArray *pArray) { + // pArray is NULL or size is zero , no need block to move + if(pArray == NULL) + return 0; + size_t count = taosArrayGetSize(pArray); + if(count == 0) + return 0; - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, index); - pCheckInfo->numOfBlocks = 0; + // memmove + int32_t num = 0; + SRange* ranges = (SRange*)TARRAY_GET_START(pArray); + for(size_t i = 0; i < count; i++) { + int32_t step = ranges[i].to - ranges[i].from + 1; + memmove(blocks + num, blocks + ranges[i].from, sizeof(SBlock) * step); + num += step; + } - if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { - code = terrno; - return code; + return num; +} + +// if block data in memory return false else true +bool blockNoItemInMem(STsdbQueryHandle* q, SBlock* pBlock) { + if(q->pMemRef == NULL) { + return false; } - SBlockIdx* compIndex = pQueryHandle->rhelper.pBlkIdx; + // mem + if(q->pMemRef->snapshot.mem) { + SMemTable* mem = q->pMemRef->snapshot.mem; + if(timeIntersect(mem->keyFirst, mem->keyLast, pBlock->keyFirst, pBlock->keyLast)) + return false; + } + // imem + if(q->pMemRef->snapshot.imem) { + SMemTable* imem = q->pMemRef->snapshot.imem; + if(timeIntersect(imem->keyFirst, imem->keyLast, pBlock->keyFirst, pBlock->keyLast)) + return false; + } - // no data block in this file, try next file - if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId.uid) { - return 0; // no data blocks in the file belongs to pCheckInfo->pTable + return true; +} + +#define MAYBE_IN_MEMORY_ROWS 4000 // approximately the capacity of one block +// skip blocks . return value is skip blocks number, skip rows reduce from *pOffset +static int32_t offsetSkipBlock(STsdbQueryHandle* q, SBlockInfo* pBlockInfo, int64_t skey, int64_t ekey, + int32_t sblock, int32_t eblock, SArray** ppArray, bool order) { + int32_t num = 0; + SBlock* blocks = pBlockInfo->blocks; + SArray* pArray = NULL; + SRange range; + range.from = -1; + + // + // ASC + // + if(order) { + for(int32_t i = sblock; i < eblock; i++) { + bool skip = false; + SBlock* pBlock = &blocks[i]; + if(i == sblock && skey > pBlock->keyFirst) { + q->frows += pBlock->numOfRows; // some rows time < s + } else { + // check can skip + if(q->srows + q->frows + pBlock->numOfRows + MAYBE_IN_MEMORY_ROWS < q->offset) { // approximately calculate + if(blockNoItemInMem(q, pBlock)) { + // can skip + q->srows += pBlock->numOfRows; + skip = true; + } else { + q->frows += pBlock->numOfRows; // maybe have some row in memroy + } + } else { + // the remainder be put to pArray + if(pArray == NULL) + pArray = taosArrayInit(1, sizeof(SRange)); + if(range.from == -1) { + range.from = i; + } else { + if(range.to + 1 != i) { + // add the previous + taosArrayPush(pArray, &range); + range.from = i; + } + } + range.to = eblock - 1; + taosArrayPush(pArray, &range); + range.from = -1; + break; + } + } + + if(skip) { + num ++; + } else { + // can't skip, append block index to pArray + if(pArray == NULL) + pArray = taosArrayInit(10, sizeof(SRange)); + if(range.from == -1) { + range.from = i; + } else { + if(range.to + 1 != i) { + // add the previous + taosArrayPush(pArray, &range); + range.from = i; + } + } + range.to = i; + } + } + // end append + if(range.from != -1) { + if(pArray == NULL) + pArray = taosArrayInit(1, sizeof(SRange)); + taosArrayPush(pArray, &range); + } + + // ASC return + *ppArray = pArray; + return num; + } + + // DES + for(int32_t i = eblock - 1; i >= sblock; i--) { + bool skip = false; + SBlock* pBlock = &blocks[i]; + if(i == eblock - 1 && ekey < pBlock->keyLast) { + q->frows += pBlock->numOfRows; // some rows time > e + } else { + // check can skip + if(q->srows + q->frows + pBlock->numOfRows + MAYBE_IN_MEMORY_ROWS < q->offset) { // approximately calculate + if(blockNoItemInMem(q, pBlock)) { + // can skip + q->srows += pBlock->numOfRows; + skip = true; + } else { + q->frows += pBlock->numOfRows; // maybe have some row in memroy + } + } else { + // the remainder be put to pArray + if(pArray == NULL) + pArray = taosArrayInit(1, sizeof(SRange)); + if(range.from == -1) { + range.from = i; + } else { + if(range.to - 1 != i) { + // add the previous + taosArrayPush(pArray, &range); + range.from = i; + } + } + range.to = 0; + taosArrayPush(pArray, &range); + range.from = -1; + break; + } + } + + if(skip) { + num ++; + } else { + // can't skip, append block index to pArray + if(pArray == NULL) + pArray = taosArrayInit(10, sizeof(SRange)); + if(range.from == -1) { + range.from = i; + } else { + if(range.to + 1 != i) { + // add the previous + taosArrayPush(pArray, &range); + range.from = i; + } + } + range.to = i; + } } - assert(compIndex->len > 0); + // end append + if(range.from != -1) { + if(pArray == NULL) + pArray = taosArrayInit(1, sizeof(SRange)); + taosArrayPush(pArray, &range); + } + if(pArray == NULL) + return num; - if (tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void**)(&pCheckInfo->pCompInfo), - (uint32_t*)(&pCheckInfo->compSize)) < 0) { - return terrno; + // reverse array + size_t count = taosArrayGetSize(pArray); + SRange* ranges = TARRAY_GET_START(pArray); + SArray* pArray1 = taosArrayInit(count, sizeof(SRange)); + + size_t i = count - 1; + while(i >= 0) { + range.from = ranges[i].to; + range.to = ranges[i].from; + taosArrayPush(pArray1, &range); + if(i == 0) + break; + i --; } - SBlockInfo* pCompInfo = pCheckInfo->pCompInfo; - TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; + *ppArray = pArray1; + taosArrayDestroy(&pArray); + return num; +} - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { +// shrink blocks by condition of query +static void shrinkBlocksByQuery(STsdbQueryHandle *pQueryHandle, STableCheckInfo *pCheckInfo) { + SBlockInfo *pCompInfo = pCheckInfo->pCompInfo; + SBlockIdx *compIndex = pQueryHandle->rhelper.pBlkIdx; + bool order = ASCENDING_TRAVERSE(pQueryHandle->order); + + if (order) { assert(pCheckInfo->lastKey <= pQueryHandle->window.ekey && pQueryHandle->window.skey <= pQueryHandle->window.ekey); } else { assert(pCheckInfo->lastKey >= pQueryHandle->window.ekey && pQueryHandle->window.skey >= pQueryHandle->window.ekey); } + TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey); e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey); // discard the unqualified data block based on the query time window int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); - int32_t end = start; - if (s > pCompInfo->blocks[start].keyLast) { - return 0; + return ; } - // todo speedup the procedure of located end block + int32_t end = start; + // locate e index of blocks -> end while (end < (int32_t)compIndex->numOfBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { end += 1; } - pCheckInfo->numOfBlocks = (end - start); + // calc offset can skip blocks number + int32_t nSkip = 0; + SArray *pArray = NULL; + if(pQueryHandle->offset > 0) { + nSkip = offsetSkipBlock(pQueryHandle, pCompInfo, s, e, start, end, &pArray, order); + } + + if(nSkip > 0) { // have offset and can skip + pCheckInfo->numOfBlocks = memMoveByArray(pCompInfo->blocks, pArray); + } else { // no offset + pCheckInfo->numOfBlocks = end - start; + if(start > 0) + memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SBlock)); + } + + if(pArray) + taosArrayDestroy(&pArray); +} - if (start > 0) { - memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SBlock)); +// load one table (tsd_index point to) need load blocks info and put into pCheckInfo->pCompInfo->blocks +static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t tsd_index, int32_t* numOfBlocks) { + // + // ONE PART. Load all blocks info from one table of tsd_index + // + int32_t code = 0; + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, tsd_index); + pCheckInfo->numOfBlocks = 0; + if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { + code = terrno; + return code; } + SBlockIdx* compIndex = pQueryHandle->rhelper.pBlkIdx; + // no data block in this file, try next file + if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId.uid) { + return 0; // no data blocks in the file belongs to pCheckInfo->pTable + } + + if (pCheckInfo->compSize < (int32_t)compIndex->len) { + assert(compIndex->len > 0); + char* t = realloc(pCheckInfo->pCompInfo, compIndex->len); + if (t == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_TDB_OUT_OF_MEMORY; + return code; + } + + pCheckInfo->pCompInfo = (SBlockInfo*)t; + pCheckInfo->compSize = compIndex->len; + } + + if (tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void**)(&pCheckInfo->pCompInfo), + (uint32_t*)(&pCheckInfo->compSize)) < 0) { + return terrno; + } + + // + // TWO PART. shrink no need blocks from all blocks by condition of query + // + shrinkBlocksByQuery(pQueryHandle, pCheckInfo); (*numOfBlocks) += pCheckInfo->numOfBlocks; + return 0; } @@ -4302,4 +4568,11 @@ end: return string; } - +// obtain queryHandle attribute +int64_t tsdbSkipOffset(TsdbQueryHandleT queryHandle) { + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; + if (pQueryHandle) { + return pQueryHandle->srows; + } + return 0; +} diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index c52fbf208f6fbf0384ecf66650919c4d12ae352e..fd9a340a25a752b18ab07a8fbb2691038af3b71b 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -20,7 +20,7 @@ extern "C" { #endif -#define TSDB_CFG_MAX_NUM 130 +#define TSDB_CFG_MAX_NUM 131 #define TSDB_CFG_PRINT_LEN 23 #define TSDB_CFG_OPTION_LEN 24 #define TSDB_CFG_VALUE_LEN 41 diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index dd943e8cc45837c814680c9e63b720ddc0c80010..cbf7006f8ffd88037ed320735e8fa4f099865d74 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -58,6 +58,13 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *tar memcpy(target, context.digest, TSDB_KEY_LEN); } +// +// TSKEY util +// + +// if time area(s1,e1) intersect with time area(s2,e2) then return true else return false +bool timeIntersect(TSKEY s1, TSKEY e1, TSKEY s2, TSKEY e2); + #ifdef __cplusplus } #endif diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 0ab7edd65a6adaf5b84fc9acd6cde4bb34f10548..a4dc184df55ff5f806c3f3602ce90e22cd941c2a 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -178,7 +178,7 @@ static void *taosThreadToOpenNewFile(void *param) { umask(0); - int32_t fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t fd = open(name, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); if (fd < 0) { tsLogObj.openInProgress = 0; tsLogObj.lines = tsLogObj.maxLines - 1000; @@ -239,7 +239,7 @@ void taosResetLog() { } static bool taosCheckFileIsOpen(char *logFileName) { - int32_t fd = open(logFileName, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t fd = open(logFileName, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO | O_BINARY); if (fd < 0) { if (errno == ENOENT) { return false; @@ -327,7 +327,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { pthread_mutex_init(&tsLogObj.logMutex, NULL); umask(0); - tsLogObj.logHandle->fd = open(fileName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + tsLogObj.logHandle->fd = open(fileName, O_WRONLY | O_CREAT | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); if (tsLogObj.logHandle->fd < 0) { printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno)); @@ -362,6 +362,9 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { fflush(stdout); return; } + if (flags == NULL || format == NULL) { + return; + } va_list argpointer; char buffer[MAX_LOGLINE_BUFFER_SIZE] = { 0 }; diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index b691abc5b9f6f828edcc46ec3a5989baa083f443..193ad3263cfa502d2eae6507cf4e12d6033c8a8c 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -92,7 +92,7 @@ static void *taosThreadToOpenNewNote(void *param) { umask(0); - int32_t fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t fd = open(name, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); if (fd < 0) { return NULL; } @@ -132,7 +132,7 @@ static int32_t taosOpenNewNote(SNoteObj *pNote) { } static bool taosCheckNoteIsOpen(char *noteName, SNoteObj *pNote) { - int32_t fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t fd = open(noteName, O_WRONLY | O_CREAT | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); if (fd < 0) { fprintf(stderr, "failed to open note:%s reason:%s\n", noteName, strerror(errno)); return true; @@ -207,7 +207,7 @@ static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxN pthread_mutex_init(&pNote->mutex, NULL); umask(0); - pNote->fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + pNote->fd = open(noteName, O_WRONLY | O_CREAT | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); if (pNote->fd < 0) { fprintf(stderr, "failed to open note file:%s reason:%s\n", noteName, strerror(errno)); diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 02498e222212fada5b7a9f39fbcfe5c76494a651..4a72697f790d770bfa744e91d165f0ad244ecbf6 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -549,3 +549,16 @@ FORCE_INLINE double taos_align_get_double(const char* pBuf) { memcpy(&dv, pBuf, sizeof(dv)); // in ARM, return *((const double*)(pBuf)) may cause problem return dv; } + +// +// TSKEY util +// + +// if time area(s1,e1) intersect with time area(s2,e2) then return true else return false +bool timeIntersect(TSKEY s1, TSKEY e1, TSKEY s2, TSKEY e2) { + // s1,e1 and s2,e2 have 7 scenarios, 5 is intersection, 2 is no intersection, so we pick up 2. + if(e2 < s1 || s2 > e1) + return false; + else + return true; +} \ No newline at end of file diff --git a/tests/develop-test/0-others/TD-12435.py b/tests/develop-test/0-others/TD-12435.py new file mode 100644 index 0000000000000000000000000000000000000000..33d44dab2832804c5d1cc9c24dcf0b58b0d03614 --- /dev/null +++ b/tests/develop-test/0-others/TD-12435.py @@ -0,0 +1,48 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, db_test.stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +import json + + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TD-12435] fix ` identifier in table column name if using create table as subquery + ''' + return + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("============== STEP 1 ===== prepare data & validate json string") + tdSql.execute("create table if not exists st(ts timestamp, dataInt int)") + tdSql.execute("create table st_from_sub as select avg(`dataInt`) from st interval(1m)") + tdSql.query("describe st_from_sub") + tdSql.checkData(1, 0, 'avg__dataInt__') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/develop-test/2-query/session_two_stage.py b/tests/develop-test/2-query/session_two_stage.py index ca17814c8e31a2f7e9aca3712655cb50f6a0f0b8..723919233c722eefbf1629146de1d8d7cc914f8b 100644 --- a/tests/develop-test/2-query/session_two_stage.py +++ b/tests/develop-test/2-query/session_two_stage.py @@ -13,7 +13,7 @@ from posixpath import split import sys -import os +import os from util.log import * from util.cases import * @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + self.ts = 1420041600000 # 2015-01-01 00:00:00 this is begin time for first record self.num = 10 @@ -49,8 +49,8 @@ class TDTestCase: ''' case1 : [TD-12344] : fix session window for super table two stage query - ''' - return + ''' + return def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -75,13 +75,13 @@ class TDTestCase: projPath = selfPath[:selfPath.find("community")] else: projPath = selfPath[:selfPath.find("tests")] - + cfgPath = projPath + "/sim/dnode1/cfg " return cfgPath - - + + def run(self): tdSql.prepare() tdSql.execute("create database if not exists testdb keep 36500;") @@ -95,9 +95,9 @@ class TDTestCase: cfg_path = self.getcfgPath() print(cfg_path) tdSql.query('select elapsed(ts,10s) from st where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" session(ts,1d) group by tbname;') # session not support super table - tdSql.checkRows(10) - - + tdSql.checkRows(10) + + def stop(self): tdSql.close() diff --git a/tests/develop-test/2-query/timeline_agg_func_groupby.py b/tests/develop-test/2-query/timeline_agg_func_groupby.py new file mode 100644 index 0000000000000000000000000000000000000000..39776875bb1e5667887af7a4c320adedf6bd7cd8 --- /dev/null +++ b/tests/develop-test/2-query/timeline_agg_func_groupby.py @@ -0,0 +1,77 @@ +################################################################### +# Copyright (c) 2021 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TD-12614] : Functions related to timeline should not support inner query group by tbname + ''' + return + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1420041600000 # 2015-01-01 00:00:00 this is begin time for first record + self.num = 10 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def run(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db") + tdSql.execute('use db') + + #Prepare data + tdSql.execute("create stable st (ts timestamp , id int , value double) tags(hostname binary(10) ,ind int);") + for i in range(self.num): + tdSql.execute("insert into tb%s using st tags('host_%s' , %d) values (%d , %d , %f );"%(str(i),str(i),i*10,self.ts+100*i,i*2,i+10.00)) + tdSql.execute("insert into tb%s using st tags('host_%s' , %d) values (%d , %d , %f );"%(str(i),str(i),i*10,self.ts+200*i,i*2,i+10.00)) + tdSql.execute("insert into tb%s using st tags('host_%s' , %d) values (%d , %d , %f );"%(str(i),str(i),i*10,self.ts+300*i,i*2,i+10.00)) + tdSql.execute("insert into tb%s using st tags('host_%s' , %d) values (%d , %d , %f );"%(str(i),str(i),i*10,self.ts+10000*i,i*2,i+10.00)) + + #execute query + tdSql.error(' select elapsed(ts) from (select csum(value) from st group by tbname );') + tdSql.error(' select elapsed(ts) from (select diff(value) from st group by tbname );') + tdSql.error(' select twa(value) from (select csum(value) value from st group by tbname );') + tdSql.error(' select twa(value) from (select diff(value) value from st group by tbname );') + + tdSql.execute('drop database db') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/develop-test/2-query/ts_2016.py b/tests/develop-test/2-query/ts_2016.py new file mode 100644 index 0000000000000000000000000000000000000000..ecebf53ed3d4afa753ae6f563b63c62f1fd58b21 --- /dev/null +++ b/tests/develop-test/2-query/ts_2016.py @@ -0,0 +1,62 @@ +################################################################### +# Copyright (c) 2021 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TS-2016]fix select * from (select * from empty_stable) + ''' + return + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self._conn = conn + + def run(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists td12229") + tdSql.execute("create database if not exists td12229") + tdSql.execute('use td12229') + + tdSql.execute('create stable st(ts timestamp , value int ) tags (ind int)') + tdSql.execute('insert into tb1 using st tags(1) values(now ,1)') + tdSql.execute('insert into tb1 using st tags(1) values(now+1s ,2)') + tdSql.execute('insert into tb1 using st tags(1) values(now+2s ,3)') + tdSql.execute('create stable ste(ts timestamp , value int ) tags (ind int)') + tdSql.query('select * from st') + tdSql.checkRows(3) + tdSql.query('select * from (select * from ste)') + tdSql.checkRows(0) + tdSql.query('select * from st union all select * from ste') + tdSql.checkRows(3) + tdSql.query('select * from ste union all select * from st') + tdSql.checkRows(3) + tdSql.query('select elapsed(ts) from ste group by tbname union all select elapsed(ts) from st group by tbname;') + tdSql.checkRows(1) + tdSql.query('select elapsed(ts) from st group by tbname union all select elapsed(ts) from ste group by tbname;') + tdSql.checkRows(1) + tdSql.execute('drop database td12229') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/develop-test/fulltest-others.sh b/tests/develop-test/fulltest-others.sh index bb0bb585b5323b45d43b01404093b97babca3ab7..b9e056a67b455bbb4b2c6518f7b9b8665618713d 100755 --- a/tests/develop-test/fulltest-others.sh +++ b/tests/develop-test/fulltest-others.sh @@ -1 +1,2 @@ -python3 ./test.py -f 0-others/json_tag.py \ No newline at end of file +python3 ./test.py -f 0-others/json_tag.py +python3 ./test.py -f 0-others/TD-12435.py \ No newline at end of file diff --git a/tests/develop-test/fulltest-query.sh b/tests/develop-test/fulltest-query.sh index b5147d20a399e6e19bcb7d84985a83a187429780..36c68e8d00d7dda89601ed2ef70b09d8cccf9b1a 100755 --- a/tests/develop-test/fulltest-query.sh +++ b/tests/develop-test/fulltest-query.sh @@ -1,3 +1,5 @@ python3 ./test.py -f 2-query/ts_hidden_column.py python3 ./test.py -f 2-query/union-order.py python3 ./test.py -f 2-query/session_two_stage.py +python3 ./test.py -f 2-query/timeline_agg_func_groupby.py +python3 ./test.py -f 2-query/ts_2016.py diff --git a/tests/pytest/fulltest-insert.sh b/tests/pytest/fulltest-insert.sh index 153bc072dba128fa8f5635e26aba0d30066b9c9a..495c6844b0186331b33416ec9eee7d0b4fbda79f 100755 --- a/tests/pytest/fulltest-insert.sh +++ b/tests/pytest/fulltest-insert.sh @@ -1,7 +1,6 @@ #!/bin/bash ulimit -c unlimited #======================p1-start=============== - python3 testCompress.py python3 testNoCompress.py python3 ./test.py -f import_merge/importBlock1HO.py @@ -130,9 +129,4 @@ python3 ./test.py -f update/merge_commit_last.py python3 ./test.py -f update/update_options.py python3 ./test.py -f update/merge_commit_data-0.py python3 ./test.py -f wal/addOldWalTest.py -python3 ./test.py -f wal/sdbComp.py - - - - - +# python3 ./test.py -f wal/sdbComp.py \ No newline at end of file diff --git a/tests/pytest/fulltest-others.sh b/tests/pytest/fulltest-others.sh index a081833ddb323ad1becfc24f48fdaaebac26b328..7d3db012d1521154cc9bfdf46f90aa213eddaeba 100755 --- a/tests/pytest/fulltest-others.sh +++ b/tests/pytest/fulltest-others.sh @@ -1,45 +1,34 @@ #!/bin/bash ulimit -c unlimited #======================p1-start=============== - #python3 ./test.py -f dbmgmt/database-name-boundary.py python3 test.py -f dbmgmt/nanoSecondCheck.py - # python3 ./test.py -f tsdb/tsdbComp.py - # user python3 ./test.py -f user/user_create.py python3 ./test.py -f user/pass_len.py - #======================p1-end=============== #======================p2-start=============== - # perfbenchmark python3 ./test.py -f perfbenchmark/bug3433.py #python3 ./test.py -f perfbenchmark/bug3589.py #python3 ./test.py -f perfbenchmark/taosdemoInsert.py - #alter table python3 ./test.py -f alter/alter_table_crash.py python3 ./test.py -f alter/alterTabAddTagWithNULL.py python3 ./test.py -f alter/alterTimestampColDataProcess.py - #======================p2-end=============== #======================p3-start=============== - python3 ./test.py -f alter/alter_table.py python3 ./test.py -f alter/alter_debugFlag.py python3 ./test.py -f alter/alter_keep.py python3 ./test.py -f alter/alter_cacheLastRow.py python3 ./test.py -f alter/alter_create_exception.py python3 ./test.py -f alter/alterColMultiTimes.py - #======================p3-end=============== #======================p4-start=============== - python3 ./test.py -f account/account_create.py - # client python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py @@ -50,12 +39,10 @@ python3 ./test.py -f client/taoshellCheckCase.py # python3 ./test.py -f client/change_time_1_2.py python3 client/twoClients.py python3 testMinTablesPerVnode.py - # topic python3 ./test.py -f topic/topicQuery.py #======================p4-end=============== #======================p5-start=============== python3 ./test.py -f ../system-test/0-management/1-stable/create_col_tag.py python3 ./test.py -f ../develop-test/0-management/3-tag/json_tag.py - -#======================p5-end=============== +#======================p5-end=============== \ No newline at end of file diff --git a/tests/pytest/fulltest-query.sh b/tests/pytest/fulltest-query.sh index b36694017c405991271340c91d21da7ca2e1b21b..5ad0f850b355bba1ab01843d7012b0ad487f761b 100755 --- a/tests/pytest/fulltest-query.sh +++ b/tests/pytest/fulltest-query.sh @@ -1,14 +1,11 @@ #!/bin/bash ulimit -c unlimited #======================p1-start=============== - # timezone python3 ./test.py -f TimeZone/TestCaseTimeZone.py - #stable python3 ./test.py -f stable/insert.py python3 ./test.py -f stable/query_after_reset.py - #table python3 ./test.py -f table/alter_wal0.py python3 ./test.py -f table/column_name.py @@ -22,7 +19,6 @@ python3 ./test.py -f table/boundary.py #python3 ./test.py -f table/create.py python3 ./test.py -f table/del_stable.py python3 ./test.py -f table/create_db_from_normal_db.py - # tag python3 ./test.py -f tag_lite/filter.py python3 ./test.py -f tag_lite/create-tags-boundary.py @@ -38,10 +34,8 @@ python3 ./test.py -f tag_lite/bool_binary.py python3 ./test.py -f tag_lite/bool_int.py python3 ./test.py -f tag_lite/bool.py python3 ./test.py -f tag_lite/change.py - #======================p1-end=============== #======================p2-start=============== - python3 ./test.py -f tag_lite/column.py python3 ./test.py -f tag_lite/commit.py python3 ./test.py -f tag_lite/create.py @@ -65,10 +59,8 @@ python3 ./test.py -f tag_lite/unsignedTinyint.py python3 ./test.py -f tag_lite/alter_tag.py python3 ./test.py -f tag_lite/drop_auto_create.py python3 ./test.py -f tag_lite/json_tag_extra.py - #======================p2-end=============== #======================p3-start=============== - #query python3 ./test.py -f query/distinctOneColTb.py python3 ./test.py -f query/filter.py @@ -118,10 +110,8 @@ python3 ./test.py -f query/subqueryFilter.py python3 ./test.py -f query/nestedQuery/queryInterval.py python3 ./test.py -f query/queryStateWindow.py # python3 ./test.py -f query/nestedQuery/queryWithOrderLimit.py - #======================p3-end=============== #======================p4-start=============== - python3 ./test.py -f query/nestquery_last_row.py python3 ./test.py -f query/nestedQuery/nestedQuery.py python3 ./test.py -f query/nestedQuery/nestedQuery_datacheck.py @@ -145,7 +135,6 @@ python3 ./test.py -f query/query.py python3 ./test.py -f query/queryDiffColsTagsAndOr.py python3 ./test.py -f query/queryGroupTbname.py python3 ./test.py -f query/queryRegex.py - #stream python3 ./test.py -f stream/metric_1.py python3 ./test.py -f stream/metric_n.py @@ -154,23 +143,19 @@ python3 ./test.py -f stream/stream1.py python3 ./test.py -f stream/stream2.py #python3 ./test.py -f stream/parser.py python3 ./test.py -f stream/history.py -python3 ./test.py -f stream/sys.py +#python3 ./test.py -f stream/sys.py python3 ./test.py -f stream/table_1.py python3 ./test.py -f stream/table_n.py python3 ./test.py -f stream/showStreamExecTimeisNull.py python3 ./test.py -f stream/cqSupportBefore1970.py - python3 ./test.py -f query/queryGroupbyWithInterval.py python3 queryCount.py - # subscribe python3 test.py -f subscribe/singlemeter.py #python3 test.py -f subscribe/stability.py python3 test.py -f subscribe/supertable.py - #======================p4-end=============== #======================p5-start=============== - # functions python3 ./test.py -f functions/all_null_value.py python3 ./test.py -f functions/function_avg.py -r 1 @@ -208,12 +193,6 @@ python3 ./test.py -f functions/function_mavg.py python3 ./test.py -f functions/function_csum.py python3 ./test.py -f functions/function_percentile2.py python3 ./test.py -f functions/variable_httpDbNameMandatory.py - - - ######## system-test #python3 ./test.py -f ../system-test/2-query/9-others/TD-11389.py # this case will run when this bug fix TD-11389 - - #======================p5-end=============== - diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 9160d34a8aa38c1c41be9cb54accc2cb76bcd80c..c9cf6a2af4241d17d9a9536374a25d1c3add604b 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -229,7 +229,8 @@ python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertAllType.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertShell.py #query -python3 test.py -f query/distinctOneColTb.py +python3 ./test.py -f query/queryBase.py +python3 ./test.py -f query/distinctOneColTb.py python3 ./test.py -f query/filter.py python3 ./test.py -f query/filterCombo.py python3 ./test.py -f query/queryNormal.py @@ -286,6 +287,8 @@ python3 ./test.py -f query/queryCnameDisplay.py python3 test.py -f query/nestedQuery/queryWithSpread.py python3 ./test.py -f query/bug6586.py # python3 ./test.py -f query/bug5903.py +python3 ./test.py -f query/queryLimit.py +python3 ./test.py -f query/queryPriKey.py #stream python3 ./test.py -f stream/metric_1.py diff --git a/tests/pytest/functions/function_derivative.py b/tests/pytest/functions/function_derivative.py index a2a458ea290b13ed462d8dcd47a8af16e3af0f82..3696dc24010cdbff6d4e139a4224a23469403041 100644 --- a/tests/pytest/functions/function_derivative.py +++ b/tests/pytest/functions/function_derivative.py @@ -140,6 +140,9 @@ class TDTestCase: tdSql.error("select derivative(col, 1s, 1) from tb2") tdSql.error("select derivative(col, 10s, 0) from tb2") tdSql.error("select derivative(col, 999ms, 0) from tb2") + tdSql.error("select derivative(col, now, 0) from tb2") #TD-11983 now not allowed in second param + tdSql.error("select derivative(col, now+3d-8h+6m, 0) from tb2") #TD-11983 now not allowed in second param + tdSql.error("select derivative(col, 3d-8h+now+6m, 0) from tb2") #TD-11983 now not allowed in second param tdSql.error("select derivative(col, 10s, 1) from stb") tdSql.error("select derivative(col, 10s, 1) from stb group by col") @@ -150,6 +153,9 @@ class TDTestCase: tdSql.error("select derivative(col, 10y, 0) from stb group by tbname") #TD-10399, DB error: syntax error near '10y, 0) from stb group by tbname;' tdSql.error("select derivative(col, -106752d, 0) from stb group by tbname") #TD-10398 overflow tips tdSql.error("select derivative(col, 106751991168d, 0) from stb group by tbname") #TD-10398 overflow tips + tdSql.error("select derivative(col, now, 1) from stb") #TD-11983 now not allowed in second param + tdSql.error("select derivative(col, now+3d-8h+6m, 1) from stb") #TD-11983 now not allowed in second param + tdSql.error("select derivative(col, 3d-8h+now+6m, 1) from stb") #TD-11983 now not allowed in second param def run(self): tdSql.prepare() diff --git a/tests/pytest/functions/function_elapsed_case.py b/tests/pytest/functions/function_elapsed_case.py index 50fbb0fe3244ec214e040f43962321a28ed31d9b..025354f2c3e31d1483f339e0e4f23bbda4c1e997 100644 --- a/tests/pytest/functions/function_elapsed_case.py +++ b/tests/pytest/functions/function_elapsed_case.py @@ -345,7 +345,9 @@ class ElapsedCase: tdSql.error("select elapsed(*) from t1 where ts > '2021-11-22 00:00:00' and ts < '2021-11-23 00:00:00'") tdSql.error("select elapsed(ts, '1s') from t1 where ts > '2021-11-22 00:00:00' and ts < '2021-11-23 00:00:00'") tdSql.error("select elapsed(ts, i) from t1 where ts > '2021-11-22 00:00:00' and ts < '2021-11-23 00:00:00'") - #tdSql.error("select elapsed(ts, now) from t1 where ts > '2021-11-22 00:00:00' and ts < '2021-11-23 00:00:00'") + tdSql.error("select elapsed(ts, now) from t1 where ts > '2021-11-22 00:00:00' and ts < '2021-11-23 00:00:00'") + tdSql.error("select elapsed(ts, now-7d+2h-3m+2s) from t1 where ts > '2021-11-22 00:00:00' and ts < '2021-11-23 00:00:00'") + tdSql.error("select elapsed(ts, 7d+2h+now+3m+2s) from t1 where ts > '2021-11-22 00:00:00' and ts < '2021-11-23 00:00:00'") tdSql.error("select elapsed(ts, ts) from t1 where ts > '2021-11-22 00:00:00' and ts < '2021-11-23 00:00:00'") tdSql.error("select elapsed(ts + 1) from t1 where ts > '2021-11-22 00:00:00' and ts < '2021-11-23 00:00:00'") tdSql.error("select elapsed(ts, 1b) from t1 where ts > '2021-11-22 00:00:00' and ts < '2021-11-23 00:00:00'") diff --git a/tests/pytest/query/queryBase.py b/tests/pytest/query/queryBase.py new file mode 100644 index 0000000000000000000000000000000000000000..af174eea11202923abdabd98a3deea33d43eb2f8 --- /dev/null +++ b/tests/pytest/query/queryBase.py @@ -0,0 +1,163 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +# +# query base function test case +# + +import sys + +from numpy.lib.function_base import insert +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + +# constant define +WAITS = 5 # wait seconds + +class TDTestCase: + # + # --------------- main frame ------------------- + # + + def caseDescription(self): + ''' + Query moudle base api or keyword test case: + case1: api first() last() + case2: none + ''' + return + + # init + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + tdSql.prepare() + self.create_tables(); + self.ts = 1500000000000 + + + # run case + def run(self): + # insert data + self.insert_data("t1", self.ts, 1*10000, 30000, 0); + self.insert_data("t2", self.ts, 2*10000, 30000, 100000); + self.insert_data("t3", self.ts, 3*10000, 30000, 200000); + # test base case + self.case_first() + tdLog.debug(" QUERYBASE first() api ............ [OK]") + # test advance case + self.case_last() + tdLog.debug(" QUERYBASE last() api ............ [OK]") + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + # + # --------------- case ------------------- + # + + # create table + def create_tables(self): + # super table + tdSql.execute("create table st(ts timestamp, i1 int) tags(area int)"); + # child table + tdSql.execute("create table t1 using st tags(1)"); + tdSql.execute("create table t2 using st tags(2)"); + tdSql.execute("create table t3 using st tags(3)"); + return + + # insert data1 + def insert_data(self, tbname, ts_start, count, batch_num, base): + pre_insert = "insert into %s values"%tbname + sql = pre_insert + tdLog.debug("doing insert table %s rows=%d ..."%(tbname, count)) + for i in range(count): + sql += " (%d,%d)"%(ts_start + i*1000, base + i) + if i >0 and i%batch_num == 0: + tdSql.execute(sql) + sql = pre_insert + # end sql + if sql != pre_insert: + tdSql.execute(sql) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + # first case base + def case_first(self): + # + # last base function + # + + # base t1 table + sql = "select first(*) from t1 where ts>='2017-07-14 12:40:00' order by ts asc;" + tdSql.waitedQuery(sql, 1, WAITS) + tdSql.checkData(0, 1, 7200) + sql = "select first(*) from t1 where ts>='2017-07-14 12:40:00' order by ts desc;" # desc + tdSql.waitedQuery(sql, 1, WAITS) + tdSql.checkData(0, 1, 7200) + # super table st + sql = "select first(*) from st where ts>='2017-07-14 11:40:00' and ts<='2017-07-14 12:40:00' and tbname in('t1') order by ts;" + tdSql.waitedQuery(sql, 1, WAITS) + tdSql.checkData(0, 1, 3600) + sql = "select first(*) from st where ts>='2017-07-14 11:40:00' and ts<='2017-07-14 12:40:00' and tbname in('t1') order by ts desc;" # desc + tdSql.waitedQuery(sql, 1, WAITS) + tdSql.checkData(0, 1, 3600) + # sub query + sql = "select first(*) from ( select sum(i1) from st where ts>='2017-07-14 11:40:00' and ts<'2017-07-14 12:40:00' interval(10m) order by ts asc );" + tdSql.waitedQuery(sql, 1, WAITS) + tdSql.checkData(0, 1, 187019100) + sql = "select first(*) from ( select sum(i1) from st where ts>='2017-07-14 11:40:00' and ts<'2017-07-14 12:40:00' interval(10m) order by ts desc );" # desc + tdSql.waitedQuery(sql, 1, WAITS) + tdSql.checkData(0, 1, 187019100) + return + + # last case + def case_last(self): + # + # last base test + # + + # base t1 table + sql = "select last(*) from t1 where ts<='2017-07-14 12:40:00' order by ts asc;" + tdSql.waitedQuery(sql, 1, WAITS) + tdSql.checkData(0, 1, 7200) + sql = "select last(*) from t1 where ts<='2017-07-14 12:40:00' order by ts desc;" # desc + tdSql.waitedQuery(sql, 1, WAITS) + tdSql.checkData(0, 1, 7200) + # super table st + sql = "select last(*) from st where ts>='2017-07-14 11:40:00' and ts<='2017-07-14 12:40:00' and tbname in('t1') order by ts;" + tdSql.waitedQuery(sql, 1, WAITS) + tdSql.checkData(0, 1, 7200) + sql = "select last(*) from st where ts>='2017-07-14 11:40:00' and ts<='2017-07-14 12:40:00' and tbname in('t1') order by ts desc;" # desc + tdSql.waitedQuery(sql, 1, WAITS) + tdSql.checkData(0, 1, 7200) + + # sub query + sql = "select last(*) from ( select sum(i1) from st where ts>='2017-07-14 11:40:00' and ts<'2017-07-14 12:40:00' interval(10m) order by ts asc );" + tdSql.waitedQuery(sql, 1, WAITS) + tdSql.checkData(0, 1, 192419100) + sql = "select last(*) from ( select sum(i1) from st where ts>='2017-07-14 11:40:00' and ts<'2017-07-14 12:40:00' interval(10m) order by ts desc );" # desc + tdSql.waitedQuery(sql, 1, WAITS) + tdSql.checkData(0, 1, 192419100) + + +# +# add case with filename +# +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/queryLimit.py b/tests/pytest/query/queryLimit.py new file mode 100644 index 0000000000000000000000000000000000000000..b7761ddf2a5594637140ae2b4748df1b1df157f5 --- /dev/null +++ b/tests/pytest/query/queryLimit.py @@ -0,0 +1,194 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys + +from numpy.lib.function_base import insert +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + +# constant define +WAITS = 5 # wait seconds + +class TDTestCase: + # + # --------------- main frame ------------------- + # + + def caseDescription(self): + ''' + limit and offset keyword function test cases; + case1: limit offset base function test + case2: limit offset advance test + ''' + return + + # init + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + tdSql.prepare() + self.create_tables(); + self.ts = 1500000000000 + + + # run case + def run(self): + # insert data + self.insert_data("t1", self.ts, 300*10000, 30000); + # test base case + self.test_case1() + tdLog.debug(" LIMIT test_case1 ............ [OK]") + # test advance case + self.test_case2() + tdLog.debug(" LIMIT test_case2 ............ [OK]") + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + # + # --------------- case ------------------- + # + + # create table + def create_tables(self): + # super table + tdSql.execute("create table st(ts timestamp, i1 int) tags(area int)"); + # child table + tdSql.execute("create table t1 using st tags(1)"); + tdSql.execute("create table t2 using st tags(2)"); + tdSql.execute("create table t3 using st tags(3)"); + return + + # insert data1 + def insert_data(self, tbname, ts_start, count, batch_num): + pre_insert = "insert into %s values"%tbname + sql = pre_insert + tdLog.debug("doing insert table %s rows=%d ..."%(tbname, count)) + for i in range(count): + sql += " (%d,%d)"%(ts_start + i*1000, i) + if i >0 and i%batch_num == 0: + tdSql.execute(sql) + sql = pre_insert + # end sql + if sql != pre_insert: + tdSql.execute(sql) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + # test case1 base + def test_case1(self): + # + # limit base function + # + # base no where + sql = "select * from t1 limit 10" + tdSql.waitedQuery(sql, 10, WAITS) + tdSql.checkData(0, 1, 0) + tdSql.checkData(9, 1, 9) + sql = "select * from t1 order by ts desc limit 10" # desc + tdSql.waitedQuery(sql, 10, WAITS) + tdSql.checkData(0, 1, 2999999) + tdSql.checkData(9, 1, 2999990) + + # have where + sql = "select * from t1 where ts>='2017-07-14 10:40:01' and ts<'2017-07-14 10:40:06' limit 10" + tdSql.waitedQuery(sql, 5, WAITS) + tdSql.checkData(0, 1, 1) + tdSql.checkData(4, 1, 5) + sql = "select * from t1 where ts>='2017-08-18 03:59:52' and ts<'2017-08-18 03:59:57' order by ts desc limit 10" # desc + tdSql.waitedQuery(sql, 5, WAITS) + tdSql.checkData(0, 1, 2999996) + tdSql.checkData(4, 1, 2999992) + + # + # offset base function + # + # no where + sql = "select * from t1 limit 10 offset 5" + tdSql.waitedQuery(sql, 10, WAITS) + tdSql.checkData(0, 1, 5) + tdSql.checkData(9, 1, 14) + sql = "select * from t1 order by ts desc limit 10 offset 5" # desc + tdSql.waitedQuery(sql, 10, WAITS) + tdSql.checkData(0, 1, 2999994) + tdSql.checkData(9, 1, 2999985) + + # have where only ts + sql = "select * from t1 where ts>='2017-07-14 10:40:10' and ts<'2017-07-14 10:40:20' limit 10 offset 5" + tdSql.waitedQuery(sql, 5, WAITS) + tdSql.checkData(0, 1, 15) + tdSql.checkData(4, 1, 19) + sql = "select * from t1 where ts>='2017-08-18 03:59:52' and ts<'2017-08-18 03:59:57' order by ts desc limit 10 offset 4" # desc + tdSql.waitedQuery(sql, 1, WAITS) + tdSql.checkData(0, 1, 2999992) + + # have where with other column condition + sql = "select * from t1 where i1>=1 and i1<11 limit 10 offset 5" + tdSql.waitedQuery(sql, 5, WAITS) + tdSql.checkData(0, 1, 6) + tdSql.checkData(4, 1, 10) + sql = "select * from t1 where i1>=300000 and i1<=500000 order by ts desc limit 10 offset 100000" # desc + tdSql.waitedQuery(sql, 10, WAITS) + tdSql.checkData(0, 1, 400000) + tdSql.checkData(9, 1, 399991) + + # have where with ts and other column condition + sql = "select * from t1 where ts>='2017-07-14 10:40:10' and ts<'2017-07-14 10:40:50' and i1>=20 and i1<=25 limit 10 offset 5" + tdSql.waitedQuery(sql, 1, WAITS) + tdSql.checkData(0, 1, 25) + + return + + # test advance + def test_case2(self): + # + # OFFSET merge file data with memory data + # + + # offset + sql = "select * from t1 limit 10 offset 72000" + tdSql.waitedQuery(sql, 10, WAITS) + tdSql.checkData(0, 1, 72000) + + # each insert one row into NO.0 NO.2 NO.7 blocks + sql = "insert into t1 values (%d, 0) (%d, 2) (%d, 7)"%(self.ts+1, self.ts + 2*3300*1000+1, self.ts + 7*3300*1000+1) + tdSql.execute(sql) + # query result + sql = "select * from t1 limit 10 offset 72000" + tdSql.waitedQuery(sql, 10, WAITS) + tdSql.checkData(0, 1, 72000 - 3) + + # have where + sql = "select * from t1 where ts>='2017-07-14 10:40:10' and ts<'2017-07-22 18:40:10' limit 10 offset 72000" + tdSql.waitedQuery(sql, 10, WAITS) + tdSql.checkData(0, 1, 72000 - 3 + 10 + 1) + + # have where desc + sql = "select * from t1 where ts<'2017-07-14 20:40:00' order by ts desc limit 15 offset 36000" + tdSql.waitedQuery(sql, 3, WAITS) + tdSql.checkData(0, 1, 1) + + +# +# add case with filename +# +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/queryPriKey.py b/tests/pytest/query/queryPriKey.py new file mode 100644 index 0000000000000000000000000000000000000000..c2a68b23ed681fef68c59f487af32c913a2abdfe --- /dev/null +++ b/tests/pytest/query/queryPriKey.py @@ -0,0 +1,54 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + tdSql.execute("drop database if exists tdb") + tdSql.execute("create database if not exists tdb keep 3650") + tdSql.execute("use tdb") + + tdSql.execute( + "create table stb1 (time timestamp, c1 int) TAGS (t1 int)" + ) + + tdSql.execute( + "insert into t1 using stb1 tags(1) values (now - 1m, 1)" + ) + tdSql.execute( + "insert into t1 using stb1 tags(1) values (now - 2m, 2)" + ) + tdSql.execute( + "insert into t1 using stb1 tags(1) values (now - 3m, 3)" + ) + + res = tdSql.getColNameList("select count(*) from t1 interval(1m)") + assert res[0] == 'time' + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/queryTbnameUpperLower.py b/tests/pytest/query/queryTbnameUpperLower.py index 147ec04793c3708258fc08bfadc8c12637a3df80..ec30f1089052ff8f1102aa0df03dcd57e4833697 100644 --- a/tests/pytest/query/queryTbnameUpperLower.py +++ b/tests/pytest/query/queryTbnameUpperLower.py @@ -26,6 +26,8 @@ class TDTestCase: ''' tdCom.cleanTb() table_name = tdCom.getLongName(8, "letters_mixed") + while table_name.islower(): + table_name = tdCom.getLongName(8, "letters_mixed") table_name_sub = f'{table_name}_sub' tb_name_lower = table_name_sub.lower() tb_name_upper = table_name_sub.upper() diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py index 34acbb2c0112b56cee6a637b9e1fbd5ddb42ddf7..f6928dffefde2420969492c2160456297d99e8bf 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py @@ -12,7 +12,7 @@ # -*- coding: utf-8 -*- import sys -import os +import os, time from util.log import * from util.cases import * from util.sql import * @@ -107,6 +107,7 @@ class TDTestCase: # insert by csv files and timetamp is long int , strings in ts and # cols + os.system( "%staosBenchmark -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabasecsv.json -y " % binPath) @@ -117,9 +118,11 @@ class TDTestCase: tdSql.checkData(0, 0, 10000) tdSql.query("describe stb0") tdSql.checkDataType(3, 1, "TIMESTAMP") + tdSql.query( - "select count(*) from stb0 where ts > \"2021-07-01 00:00:00.490000000\"") + "select count(*) from nsdbcsv.stb0 where ts > \"2021-07-01 00:00:00.490000000\"") tdSql.checkData(0, 0, 5000) + tdSql.query("select count(*) from stb0 where ts < 1626918583000000000") tdSql.checkData(0, 0, 10000) @@ -134,31 +137,9 @@ class TDTestCase: binPath) tdSql.query("select count(*) from test.meters") tdSql.checkData(0, 0, 600) - # check taosdemo -s - - sqls_ls = [ - 'drop database if exists nsdbsql;', - 'create database nsdbsql precision "ns" keep 3600 days 6 update 1;', - 'use nsdbsql;', - 'CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int);', - 'CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2);', - 'INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32);', - 'INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 85, 32, 0.76);'] - - with open("./taosdemoTestNanoCreateDB.sql", mode="a") as sql_files: - for sql in sqls_ls: - sql_files.write(sql + "\n") - sql_files.close() - - sleep(10) - - os.system("%staosBenchmark -s taosdemoTestNanoCreateDB.sql -y " % binPath) - tdSql.query("select count(*) from nsdbsql.meters") - tdSql.checkData(0, 0, 2) os.system("rm -rf ./res.txt") os.system("rm -rf ./*.py.sql") - os.system("rm -rf ./taosdemoTestNanoCreateDB.sql") def stop(self): tdSql.close() diff --git a/tests/pytest/tools/taosdemoAllTest/sml/insert-sml-json-alltype.json b/tests/pytest/tools/taosdemoAllTest/sml/insert-sml-json-alltype.json index 1b255a37f19b584211430b2f13e8754faedd5577..66885ebab89f7221830e66d642ca17b99de0e397 100644 --- a/tests/pytest/tools/taosdemoAllTest/sml/insert-sml-json-alltype.json +++ b/tests/pytest/tools/taosdemoAllTest/sml/insert-sml-json-alltype.json @@ -56,7 +56,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "INT"}], - "tags": [{"type": "INT", "count":1}] + "tags": [{"type": "INT", "count":6}] }, { "name": "stb1", @@ -81,8 +81,8 @@ "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", - "columns": [{"type": "TINYINT", "count":1}], - "tags": [{"type": "TINYINT", "count":1}] + "columns": [{"type": "TINYINT", "count":6}], + "tags": [{"type": "TINYINT", "count":6}] }, { "name": "stb2", @@ -108,7 +108,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "BIGINT"}], - "tags": [{"type": "BIGINT", "count":1}] + "tags": [{"type": "BIGINT", "count":6}] }, { "name": "stb3", @@ -134,7 +134,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "SMALLINT"}], - "tags": [{"type": "SMALLINT", "count":1}] + "tags": [{"type": "SMALLINT", "count":6}] }, { "name": "stb4", @@ -160,7 +160,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "FLOAT"}], - "tags": [{"type": "FLOAT", "count":1}] + "tags": [{"type": "FLOAT", "count":6}] }, { "name": "stb5", @@ -186,7 +186,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "DOUBLE"}], - "tags": [{"type": "DOUBLE", "count":1}] + "tags": [{"type": "DOUBLE", "count":6}] }, { "name": "stb6", @@ -212,7 +212,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "UINT"}], - "tags": [{"type": "UINT", "count":1}] + "tags": [{"type": "UINT", "count":6}] }, { "name": "stb7", @@ -237,8 +237,8 @@ "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", - "columns": [ {"type": "BOOL"}], - "tags": [{"type": "BOOL", "count":1}] + "columns": [ {"type": "INT"}], + "tags": [{"type": "INT", "count":3}] }, { "name": "stb8", @@ -263,8 +263,8 @@ "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", - "columns": [{"type": "NCHAR","len": 16, "count":1}], - "tags": [{"type": "NCHAR", "count":1}] + "columns": [{"type": "NCHAR","len": 16, "count":6}], + "tags": [{"type": "NCHAR", "count":6}] }, { "name": "stb9", @@ -289,8 +289,8 @@ "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", - "columns": [{"type": "BINARY", "len": 16, "count":1}], - "tags": [{"type": "BINARY", "count":1}] + "columns": [{"type": "BINARY", "len": 16, "count":6}], + "tags": [{"type": "BINARY", "count":6}] }, { "name": "stb10", @@ -316,7 +316,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "UBIGINT"}], - "tags": [{"type": "UBIGINT", "count":1}] + "tags": [{"type": "UBIGINT", "count":6}] }, { "name": "stb11", @@ -342,7 +342,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "UTINYINT"}], - "tags": [{"type": "UTINYINT", "count":1}] + "tags": [{"type": "UTINYINT", "count":3}] }, { "name": "stb12", @@ -368,7 +368,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [ {"type": "USMALLINT"}], - "tags": [{"type": "USMALLINT", "count":1}] + "tags": [{"type": "USMALLINT", "count":6}] }] }] } diff --git a/tests/pytest/tools/taosdemoAllTest/sml/insert-sml-telnet-alltype.json b/tests/pytest/tools/taosdemoAllTest/sml/insert-sml-telnet-alltype.json index 983a3009db68e95fecf3f8eda91f0aa3f41aff37..a786e93696e8b13b39d45a9c4c8ef1aae829fef8 100644 --- a/tests/pytest/tools/taosdemoAllTest/sml/insert-sml-telnet-alltype.json +++ b/tests/pytest/tools/taosdemoAllTest/sml/insert-sml-telnet-alltype.json @@ -56,7 +56,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "INT"}], - "tags": [{"type": "INT", "count":1}] + "tags": [{"type": "INT", "count":6}] }, { "name": "stb1", @@ -82,7 +82,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "UINT"}], - "tags": [{"type": "UINT", "count":1}] + "tags": [{"type": "UINT", "count":6}] }, { "name": "stb2", @@ -107,8 +107,8 @@ "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", - "columns": [{"type": "TINYINT", "count":1}], - "tags": [{"type": "TINYINT", "count":1}] + "columns": [{"type": "TINYINT", "count":6}], + "tags": [{"type": "TINYINT", "count":6}] }, { "name": "stb3", @@ -134,7 +134,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "BIGINT"}], - "tags": [{"type": "BIGINT", "count":1}] + "tags": [{"type": "BIGINT", "count":6}] }, { "name": "stb4", @@ -160,7 +160,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "SMALLINT"}], - "tags": [{"type": "SMALLINT", "count":1}] + "tags": [{"type": "SMALLINT", "count":6}] }, { "name": "stb5", @@ -186,7 +186,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "FLOAT"}], - "tags": [{"type": "FLOAT", "count":1}] + "tags": [{"type": "FLOAT", "count":6}] }, { "name": "stb6", @@ -212,7 +212,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "DOUBLE"}], - "tags": [{"type": "DOUBLE", "count":1}] + "tags": [{"type": "DOUBLE", "count":6}] }, { "name": "stb7", @@ -237,8 +237,8 @@ "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", - "columns": [ {"type": "BOOL"}], - "tags": [{"type": "BOOL", "count":1}] + "columns": [ {"type": "int"}], + "tags": [{"type": "int", "count":6}] }, { "name": "stb8", @@ -263,8 +263,8 @@ "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", - "columns": [{"type": "NCHAR","len": 16, "count":1}], - "tags": [{"type": "NCHAR", "count":1}] + "columns": [{"type": "NCHAR","len": 16, "count":6}], + "tags": [{"type": "NCHAR", "count":6}] }, { "name": "stb9", @@ -289,8 +289,8 @@ "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", - "columns": [{"type": "BINARY", "len": 16, "count":1}], - "tags": [{"type": "BINARY", "count":1}] + "columns": [{"type": "BINARY", "len": 16, "count":6}], + "tags": [{"type": "BINARY", "count":6}] }, { "name": "stb10", @@ -316,7 +316,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "UBIGINT"}], - "tags": [{"type": "UBIGINT", "count":1}] + "tags": [{"type": "UBIGINT", "count":6}] }, { "name": "stb11", @@ -342,7 +342,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "UTINYINT"}], - "tags": [{"type": "UTINYINT", "count":1}] + "tags": [{"type": "UTINYINT", "count":6}] }, { "name": "stb12", @@ -368,7 +368,7 @@ "sample_file": "./sample.csv", "tags_file": "", "columns": [ {"type": "USMALLINT"}], - "tags": [{"type": "USMALLINT", "count":1}] + "tags": [{"type": "USMALLINT", "count":6}] }] }] } diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJsonStmt.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJsonStmt.py index 05ccce79101b5bec1b541bd0436b86fc0151492c..6a5a3f767f1c5787680d75ee8cb98ee284a44741 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJsonStmt.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJsonStmt.py @@ -268,16 +268,16 @@ class TDTestCase: tdSql.checkData(0, 0, 10) # insert: sample json - os.system("%staosBenchmark -f tools/taosdemoAllTest/stmt/insert-sample-stmt.json -y " % binPath) - tdSql.execute("use dbtest123") - tdSql.query("select c2 from stb0") - tdSql.checkData(0, 0, 2147483647) - tdSql.query("select * from stb1 where t1=-127") - tdSql.checkRows(20) - tdSql.query("select * from stb1 where t2=127") - tdSql.checkRows(10) - tdSql.query("select * from stb1 where t2=126") - tdSql.checkRows(10) + #os.system("%staosBenchmark -f tools/taosdemoAllTest/stmt/insert-sample-stmt.json -y " % binPath) + #tdSql.execute("use dbtest123") + #tdSql.query("select c2 from stb0") + #tdSql.checkData(0, 0, 2147483647) + #tdSql.query("select * from stb1 where t1=-127") + #tdSql.checkRows(20) + #tdSql.query("select * from stb1 where t2=127") + #tdSql.checkRows(10) + #tdSql.query("select * from stb1 where t2=126") + #tdSql.checkRows(10) # insert: test interlace parament os.system("%staosBenchmark -f tools/taosdemoAllTest/stmt/insert-interlace-row-stmt.json -y " % binPath) diff --git a/tests/pytest/tools/taosdumpTest2.py b/tests/pytest/tools/taosdumpTest2.py index c60fb42266c6d23ad9aeabc9bf9f48ac5feec17b..85142e95af851159e6e7e595c8bd778be19f3e44 100644 --- a/tests/pytest/tools/taosdumpTest2.py +++ b/tests/pytest/tools/taosdumpTest2.py @@ -27,6 +27,7 @@ class TDTestCase: self.ts = 1601481600000 self.numberOfTables = 1 self.numberOfRecords = 15000 + self.tmpdir = "tmp" def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -67,16 +68,22 @@ class TDTestCase: tdLog.info("taosdump found in %s" % buildPath) binPath = buildPath + "/build/bin/" - os.system("rm /tmp/*.sql") + if not os.path.exists(self.tmpdir): + os.makedirs(self.tmpdir) + else: + print("directory exists") + os.system("rm -rf %s" % self.tmpdir) + os.makedirs(self.tmpdir) + os.system( - "%staosdump --databases db -o /tmp -B 16384 -L 1048576" % - binPath) + "%staosdump --databases db -o %s -B 16384 " % + (binPath, self.tmpdir)) tdSql.execute("drop database db") tdSql.query("show databases") tdSql.checkRows(0) - os.system("%staosdump -i /tmp" % binPath) + os.system("%staosdump -i %s" % (binPath, self.tmpdir)) tdSql.query("show databases") tdSql.checkRows(1) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 9a8f602901507bc4fc31d3902461394446a3067b..6bbf5a24cf3ae2c50e8a30636962e815cd5b15c0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -1,8 +1,6 @@ cd ../../../debug; cmake .. cd ../../../debug; make - #======================b1-start=============== - ./test.sh -f general/field/2.sim ./test.sh -f general/field/3.sim ./test.sh -f general/field/4.sim @@ -14,8 +12,6 @@ cd ../../../debug; make ./test.sh -f general/field/single.sim ./test.sh -f general/field/smallint.sim ./test.sh -f general/field/tinyint.sim - - # ./test.sh -f general/http/autocreate.sim # ./test.sh -f general/http/chunked.sim # ./test.sh -f general/http/gzip.sim @@ -27,7 +23,6 @@ cd ../../../debug; make # ./test.sh -f general/http/telegraf.sim # ./test.sh -f general/http/grafana_bug.sim # ./test.sh -f general/http/grafana.sim - ./test.sh -f general/insert/basic.sim ./test.sh -f general/insert/insert_drop.sim ./test.sh -f general/insert/query_block1_memory.sim @@ -37,7 +32,6 @@ cd ../../../debug; make ./test.sh -f general/insert/query_file_memory.sim ./test.sh -f general/insert/query_multi_file.sim ./test.sh -f general/insert/tcp.sim - ./test.sh -f general/parser/alter.sim ./test.sh -f general/parser/alter1.sim ./test.sh -f general/parser/alter_stable.sim @@ -90,30 +84,22 @@ cd ../../../debug; make ./test.sh -f general/db/nosuchfile.sim ./test.sh -f general/parser/function.sim ./test.sh -f unique/cluster/vgroup100.sim - # ./test.sh -f unique/http/admin.sim # ./test.sh -f unique/http/opentsdb.sim - ./test.sh -f unique/import/replica2.sim ./test.sh -f unique/import/replica3.sim - ./test.sh -f general/alter/cached_schema_after_alter.sim - #======================b1-end=============== #======================b2-start=============== - - #./test.sh -f general/wal/sync.sim ./test.sh -f general/wal/kill.sim ./test.sh -f general/wal/maxtables.sim - ./test.sh -f general/user/authority.sim ./test.sh -f general/user/monitor.sim ./test.sh -f general/user/pass_alter.sim ./test.sh -f general/user/pass_len.sim ./test.sh -f general/user/user_create.sim ./test.sh -f general/user/user_len.sim - ./test.sh -f general/vector/metrics_field.sim ./test.sh -f general/vector/metrics_mix.sim ./test.sh -f general/vector/metrics_query.sim @@ -125,7 +111,6 @@ cd ../../../debug; make ./test.sh -f general/vector/table_mix.sim ./test.sh -f general/vector/table_query.sim ./test.sh -f general/vector/table_time.sim - ./test.sh -f unique/account/account_create.sim ./test.sh -f unique/account/account_delete.sim ./test.sh -f unique/account/account_len.sim @@ -137,24 +122,17 @@ cd ../../../debug; make ./test.sh -f unique/account/usage.sim ./test.sh -f unique/account/user_create.sim ./test.sh -f unique/account/user_len.sim - ./test.sh -f unique/big/maxvnodes.sim ./test.sh -f unique/big/tcp.sim - ./test.sh -f unique/cluster/alter.sim ./test.sh -f unique/cluster/cache.sim #./test.sh -f unique/http/admin.sim #./test.sh -f unique/http/opentsdb.sim - ./test.sh -f unique/import/replica2.sim ./test.sh -f unique/import/replica3.sim - ./test.sh -f general/alter/cached_schema_after_alter.sim - - #======================b2-end=============== #======================b3-start=============== - ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim #./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim @@ -175,7 +153,6 @@ cd ../../../debug; make ./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim ./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim ./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim - ./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim ./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim ./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim @@ -189,19 +166,16 @@ cd ../../../debug; make ./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim ./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim ./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim - ./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim ./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim ./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim ./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim ./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim ./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim - ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim ./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim - ./test.sh -f unique/stable/balance_replica1.sim ./test.sh -f unique/stable/dnode2_stop.sim ./test.sh -f unique/stable/dnode2.sim @@ -210,11 +184,8 @@ cd ../../../debug; make ./test.sh -f unique/stable/replica2_vnode3.sim ./test.sh -f unique/stable/replica3_dnode6.sim ./test.sh -f unique/stable/replica3_vnode3.sim - #======================b3-end=============== #======================b4-start=============== - - ./test.sh -f general/alter/count.sim ./test.sh -f general/alter/dnode.sim ./test.sh -f general/alter/import.sim @@ -222,22 +193,17 @@ cd ../../../debug; make ./test.sh -f general/alter/insert2.sim ./test.sh -f general/alter/metrics.sim ./test.sh -f general/alter/table.sim - ./test.sh -f general/cache/new_metrics.sim ./test.sh -f general/cache/restart_metrics.sim ./test.sh -f general/cache/restart_table.sim - ./test.sh -f general/connection/connection.sim - ./test.sh -f general/column/commit.sim ./test.sh -f general/column/metrics.sim ./test.sh -f general/column/table.sim - ./test.sh -f general/compress/commitlog.sim ./test.sh -f general/compress/compress.sim ./test.sh -f general/compress/compress2.sim ./test.sh -f general/compress/uncompress.sim - ./test.sh -f general/stable/disk.sim ./test.sh -f general/stable/dnode3.sim ./test.sh -f general/stable/metrics.sim @@ -245,7 +211,6 @@ cd ../../../debug; make ./test.sh -f general/stable/show.sim ./test.sh -f general/stable/values.sim ./test.sh -f general/stable/vnode3.sim - ./test.sh -f unique/column/replica3.sim ./test.sh -f issue/TD-2713.sim ./test.sh -f general/parser/select_distinct_tag.sim @@ -253,10 +218,8 @@ cd ../../../debug; make ./test.sh -f issue/TD-2677.sim ./test.sh -f issue/TD-2680.sim ./test.sh -f unique/dnode/lossdata.sim - #======================b4-end=============== #======================b5-start=============== - ./test.sh -f unique/dnode/alternativeRole.sim ./test.sh -f unique/dnode/balance1.sim ./test.sh -f unique/dnode/balance2.sim @@ -264,7 +227,6 @@ cd ../../../debug; make ./test.sh -f unique/dnode/balancex.sim ./test.sh -f unique/dnode/offline1.sim ./test.sh -f unique/dnode/offline2.sim - ./test.sh -f general/stream/metrics_del.sim ./test.sh -f general/stream/metrics_replica1_vnoden.sim ./test.sh -f general/stream/restart_stream.sim @@ -272,22 +234,18 @@ cd ../../../debug; make ./test.sh -f general/stream/stream_restart.sim ./test.sh -f general/stream/table_del.sim ./test.sh -f general/stream/table_replica1_vnoden.sim - ./test.sh -f general/connection/test_old_data.sim ./test.sh -f unique/dnode/datatrans_3node.sim ./test.sh -f unique/dnode/datatrans_3node_2.sim ./test.sh -f general/db/alter_tables_d2.sim ./test.sh -f general/db/alter_tables_v1.sim ./test.sh -f general/db/alter_tables_v4.sim - #======================b5-end=============== #======================b6-start=============== - ./test.sh -f unique/dnode/reason.sim ./test.sh -f unique/dnode/remove1.sim ./test.sh -f unique/dnode/remove2.sim ./test.sh -f unique/dnode/vnode_clean.sim - ./test.sh -f unique/db/commit.sim ./test.sh -f unique/db/delete.sim ./test.sh -f unique/db/delete_part.sim @@ -298,14 +256,12 @@ cd ../../../debug; make ./test.sh -f unique/db/replica_reduce32.sim ./test.sh -f unique/db/replica_reduce31.sim ./test.sh -f unique/db/replica_part.sim - ./test.sh -f unique/vnode/many.sim ./test.sh -f unique/vnode/replica2_basic2.sim ./test.sh -f unique/vnode/replica2_repeat.sim ./test.sh -f unique/vnode/replica3_basic.sim ./test.sh -f unique/vnode/replica3_repeat.sim ./test.sh -f unique/vnode/replica3_vgroup.sim - ./test.sh -f unique/dnode/monitor.sim ./test.sh -f unique/dnode/monitor_bug.sim ./test.sh -f unique/dnode/simple.sim @@ -315,7 +271,6 @@ cd ../../../debug; make ./test.sh -f unique/dnode/offline3.sim ./test.sh -f general/wal/kill.sim ./test.sh -f general/wal/maxtables.sim - ./test.sh -f general/import/basic.sim ./test.sh -f general/import/commit.sim ./test.sh -f general/import/large.sim @@ -323,10 +278,8 @@ cd ../../../debug; make ./test.sh -f unique/cluster/balance1.sim ./test.sh -f unique/cluster/balance2.sim ./test.sh -f unique/cluster/balance3.sim - #======================b6-end=============== #======================b7-start=============== - ./test.sh -f general/compute/avg.sim ./test.sh -f general/compute/bottom.sim ./test.sh -f general/compute/count.sim @@ -343,7 +296,6 @@ cd ../../../debug; make ./test.sh -f general/compute/stddev.sim ./test.sh -f general/compute/sum.sim ./test.sh -f general/compute/top.sim - ./test.sh -f general/db/alter_option.sim ./test.sh -f general/db/alter_vgroups.sim ./test.sh -f general/db/basic.sim @@ -392,7 +344,6 @@ cd ../../../debug; make ./test.sh -f general/table/tinyint.sim ./test.sh -f general/table/vgroup.sim ./test.sh -f general/table/createmulti.sim - ./test.sh -f unique/mnode/mgmt20.sim ./test.sh -f unique/mnode/mgmt21.sim ./test.sh -f unique/mnode/mgmt22.sim @@ -403,7 +354,6 @@ cd ../../../debug; make ./test.sh -f unique/mnode/mgmt33.sim ./test.sh -f unique/mnode/mgmt34.sim ./test.sh -f unique/mnode/mgmtr2.sim - #./test.sh -f unique/arbitrator/insert_duplicationTs.sim ./test.sh -f general/parser/join_manyblocks.sim ./test.sh -f general/parser/stableOp.sim @@ -415,9 +365,7 @@ cd ../../../debug; make ./test.sh -f general/parser/last_cache.sim ./test.sh -f unique/big/balance.sim ./test.sh -f general/parser/nestquery.sim - ./test.sh -f general/parser/udf.sim ./test.sh -f general/parser/udf_dll.sim ./test.sh -f general/parser/udf_dll_stable.sim - -#======================b7-end=============== +#======================b7-end=============== \ No newline at end of file diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim index e5f2928748896a2aaa811ddc76bfb16b9626bf1d..e3623c7c629d671eedc7b6a416b9e77e6445c4ff 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim @@ -51,7 +51,7 @@ sleep 1000 sql connect sleep 1000 sql create dnode $hostname2 -sleep 1000 +sleep 3000 print ============== step2: create database with replica 2, and create table, insert data $totalTableNum = 10 diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim index 8d063020e73be449bc95463e966d9081b0cd5be5..c88e26d7eb19a533be84f646321e103480b2d10a 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim @@ -51,7 +51,7 @@ sleep 1000 sql connect sleep 1000 sql create dnode $hostname2 -sleep 1000 +sleep 2000 print ============== step2: create database with replica 2, and create table, insert data $totalTableNum = 10 diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim index 8f837b7e477ab801b296b32ddcf9a5c683c351f0..ed3f9b8274c204727a08c163596316ed17808d6b 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim @@ -51,7 +51,7 @@ sleep 1000 sql connect sleep 1000 sql create dnode $hostname2 -sleep 1000 +sleep 2000 print ============== step2: create database with replica 2, and create table, insert data $totalTableNum = 10 diff --git a/tests/test-CI.sh b/tests/test-CI.sh index c458be0aa184d6d0a3831554d4974a4b98662cfe..b9fd8aa89f6fe08fd17786eb8f42aa2ee9cc149c 100755 --- a/tests/test-CI.sh +++ b/tests/test-CI.sh @@ -51,7 +51,52 @@ function dohavecore(){ fi fi } +function runSimCaseOneByOnefq { + end=`sed -n '$=' jenkins/basic.txt` + for ((i=1;i<=$end;i++)) ; do + if [[ $(($i%$1)) -eq $3 ]];then + line=`sed -n "$i"p jenkins/basic.txt` + if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then + case=`echo $line | grep sim$ |awk '{print $NF}'` + start_time=`date +%s` + date +%F\ %T | tee -a out.log + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + echo -n $case + ./test.sh -f $case > case.log 2>&1 && \ + ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) + else + echo -n $case + ./test.sh -f $case > ../../sim/case.log 2>&1 && \ + ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) + fi + + out_log=`tail -1 out.log ` + if [[ $out_log =~ 'failed' ]];then + rm case.log + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cp -r ../../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"` + else + cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` + fi + dohavecore $2 1 + if [[ $2 == 1 ]];then + exit 8 + fi + fi + end_time=`date +%s` + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log + dohavecore $2 1 + fi + fi + done + rm -rf ../../../sim/case.log + rm -rf ../../sim/case.log +} function runPyCaseOneByOne { while read -r line; do @@ -124,11 +169,9 @@ function runPyCaseOneByOnefq() { else echo $line if [[ $line =~ ^bash.* ]]; then - # $line > case.log 2>&1 || cat case.log && exit 8 - # cat case.log $line > case.log 2>&1 + cat case.log if [ $? -ne 0 ];then - cat case.log exit 8 fi fi @@ -175,7 +218,6 @@ if [ "${OS}" == "Linux" ]; then fi -echo "### run Python test case ###" cd $tests_dir @@ -206,8 +248,13 @@ if [ "$1" == "full" ]; then runPyCaseOneByOne fulltest-other.sh runPyCaseOneByOne fulltest-insert.sh runPyCaseOneByOne fulltest-connector.sh +elif [ "$1" == "sim" ]; then + echo "### run sim $2 test ###" + cd $tests_dir/script + runSimCaseOneByOnefq $2 1 $3 else echo "### run $1 $2 test ###" + if [ "$1" != "query" ] && [ "$1" != "taosAdapter" ] && [ "$1" != "other" ] && [ "$1" != "tools" ] && [ "$1" != "insert" ] && [ "$1" != "connector" ] ;then echo " wrong option:$1 must one of [query,other,tools,insert,connector,taosAdapter]" exit 8