Merge branch 'master' of github.com:taosdata/TDengine into test/chr

.gitmodules

 [submodule "src/connector/go"]
 	path = src/connector/go
-	url = https://github.com/taosdata/driver-go
+	url = git@github.com:taosdata/driver-go.git
 [submodule "src/connector/grafanaplugin"]
 	path = src/connector/grafanaplugin
-	url = https://github.com/taosdata/grafanaplugin
+	url = git@github.com:taosdata/grafanaplugin.git
 [submodule "src/connector/hivemq-tdengine-extension"]
 	path = src/connector/hivemq-tdengine-extension
-	url = https://github.com/huskar-t/hivemq-tdengine-extension.git
+	url = git@github.com:taosdata/hivemq-tdengine-extension.git
 [submodule "tests/examples/rust"]
 	path = tests/examples/rust
 	url = https://github.com/songtianyi/tdengine-rust-bindings.git

cmake/version.inc

@@ -4,7 +4,7 @@ PROJECT(TDengine)
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "2.0.20.2")
+  SET(TD_VER_NUMBER "2.0.20.5")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

packaging/deb/makedeb.sh

@@ -58,7 +58,12 @@ cp ${compile_dir}/build/lib/${libfile}              ${pkg_dir}${install_home_pat
 cp ${compile_dir}/../src/inc/taos.h                 ${pkg_dir}${install_home_path}/include
 cp ${compile_dir}/../src/inc/taoserror.h            ${pkg_dir}${install_home_path}/include
 cp -r ${top_dir}/tests/examples/*                   ${pkg_dir}${install_home_path}/examples
-cp -r ${top_dir}/src/connector/grafanaplugin        ${pkg_dir}${install_home_path}/connector
+if [ -d "${top_dir}/src/connector/grafanaplugin/dist" ]; then
+  cp -r ${top_dir}/src/connector/grafanaplugin/dist   ${pkg_dir}${install_home_path}/connector/grafanaplugin
+else
+  echo "grafanaplugin bundled directory not found!"
+  exit 1
+fi
 cp -r ${top_dir}/src/connector/python               ${pkg_dir}${install_home_path}/connector
 cp -r ${top_dir}/src/connector/go                   ${pkg_dir}${install_home_path}/connector
 cp -r ${top_dir}/src/connector/nodejs               ${pkg_dir}${install_home_path}/connector

packaging/rpm/tdengine.spec

@@ -66,7 +66,12 @@ cp %{_compiledir}/build/bin/taosdump                %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/lib/${libfile}              %{buildroot}%{homepath}/driver
 cp %{_compiledir}/../src/inc/taos.h                 %{buildroot}%{homepath}/include
 cp %{_compiledir}/../src/inc/taoserror.h            %{buildroot}%{homepath}/include
-cp -r %{_compiledir}/../src/connector/grafanaplugin %{buildroot}%{homepath}/connector
+if [ -d %{_compiledir}/../src/connector/grafanaplugin/dist ]; then
+    cp -r %{_compiledir}/../src/connector/grafanaplugin/dist %{buildroot}%{homepath}/connector/grafanaplugin
+else
+    echo grafanaplugin bundled directory not found!
+    exit 1
+fi
 cp -r %{_compiledir}/../src/connector/python        %{buildroot}%{homepath}/connector
 cp -r %{_compiledir}/../src/connector/go            %{buildroot}%{homepath}/connector
 cp -r %{_compiledir}/../src/connector/nodejs        %{buildroot}%{homepath}/connector

packaging/tools/make_install.sh

@@ -243,9 +243,17 @@ function install_data() {
 }
 
 function install_connector() {
-    ${csudo} cp -rf ${source_dir}/src/connector/grafanaplugin ${install_main_dir}/connector
+    if [ -d "${source_dir}/src/connector/grafanaplugin/dist" ]; then
+        ${csudo} cp -rf ${source_dir}/src/connector/grafanaplugin/dist ${install_main_dir}/connector/grafanaplugin
+    else
+        echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!"
+    fi
+    if find ${source_dir}/src/connector/go -mindepth 1 -maxdepth 1 | read; then
+        ${csudo} cp -r ${source_dir}/src/connector/go ${install_main_dir}/connector
+    else
+        echo "WARNING: go connector not found, please check if want to use it!"
+    fi
     ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector
-    ${csudo} cp -rf ${source_dir}/src/connector/go ${install_main_dir}/connector
         
     ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null 
 }

packaging/tools/makeclient.sh

@@ -117,10 +117,18 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
   if [ "$osType" != "Darwin" ]; then
     cp ${build_dir}/lib/*.jar      ${install_dir}/connector ||:
   fi
-  cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
-  cp -r ${connector_dir}/python        ${install_dir}/connector/
-  cp -r ${connector_dir}/go            ${install_dir}/connector
-  cp -r ${connector_dir}/nodejs        ${install_dir}/connector
+  if [ -d "${connector_dir}/grafanaplugin/dist" ]; then
+    cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin
+  else
+    echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!"
+  fi
+  if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then
+    cp -r ${connector_dir}/go ${install_dir}/connector
+  else
+    echo "WARNING: go connector not found, please check if want to use it!"
+  fi
+  cp -r ${connector_dir}/python             ${install_dir}/connector
+  cp -r ${connector_dir}/nodejs             ${install_dir}/connector
 fi
 # Copy release note
 # cp ${script_dir}/release_note ${install_dir}

packaging/tools/makeclient_power.sh

@@ -144,9 +144,17 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
   if [ "$osType" != "Darwin" ]; then
     cp ${build_dir}/lib/*.jar      ${install_dir}/connector ||:
   fi
-  cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
-  cp -r ${connector_dir}/python  ${install_dir}/connector/
-  cp -r ${connector_dir}/go      ${install_dir}/connector 
+  if [ -d "${connector_dir}/grafanaplugin/dist" ]; then
+    cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin
+  else
+    echo "WARNING: grafanaplugin bunlded dir not found, please check if want to use it!"
+  fi
+  if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then
+    cp -r ${connector_dir}/go ${install_dir}/connector
+  else
+    echo "WARNING: go connector not found, please check if want to use it!"
+  fi
+  cp -r ${connector_dir}/python  ${install_dir}/connector
   
   sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/connector/python/linux/python2/taos/cinterface.py
   sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/connector/python/linux/python3/taos/cinterface.py

packaging/tools/makepkg.sh

@@ -114,6 +114,25 @@ mkdir -p ${install_dir}/examples
 examples_dir="${top_dir}/tests/examples"
   cp -r ${examples_dir}/c      ${install_dir}/examples
 if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
+  if [ -d ${examples_dir}/JDBC/connectionPools/target ]; then
+    rm -rf ${examples_dir}/JDBC/connectionPools/target
+  fi
+  if [ -d ${examples_dir}/JDBC/JDBCDemo/target ]; then
+    rm -rf ${examples_dir}/JDBC/JDBCDemo/target
+  fi
+  if [ -d ${examples_dir}/JDBC/mybatisplus-demo/target ]; then
+    rm -rf ${examples_dir}/JDBC/mybatisplus-demo/target
+  fi
+  if [ -d ${examples_dir}/JDBC/springbootdemo/target ]; then
+    rm -rf ${examples_dir}/JDBC/springbootdemo/target
+  fi
+  if [ -d ${examples_dir}/JDBC/SpringJdbcTemplate/target ]; then
+    rm -rf ${examples_dir}/JDBC/SpringJdbcTemplate/target
+  fi
+  if [ -d ${examples_dir}/JDBC/taosdemo/target ]; then
+    rm -rf ${examples_dir}/JDBC/taosdemo/target
+  fi
+  
   cp -r ${examples_dir}/JDBC   ${install_dir}/examples
   cp -r ${examples_dir}/matlab ${install_dir}/examples
   cp -r ${examples_dir}/python ${install_dir}/examples
@@ -131,9 +150,17 @@ connector_dir="${code_dir}/connector"
 mkdir -p ${install_dir}/connector
 if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
   cp ${build_dir}/lib/*.jar            ${install_dir}/connector ||:
-  cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
-  cp -r ${connector_dir}/python        ${install_dir}/connector/
-  cp -r ${connector_dir}/go            ${install_dir}/connector
+  if [ -d "${connector_dir}/grafanaplugin/dist" ]; then
+    cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin
+  else
+    echo "WARNING: grafanaplugin bundled dir not found, please check if you want to use it!"
+  fi
+  if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then
+    cp -r ${connector_dir}/go ${install_dir}/connector
+  else
+    echo "WARNING: go connector not found, please check if want to use it!"
+  fi
+  cp -r ${connector_dir}/python        ${install_dir}/connector
   cp -r ${connector_dir}/nodejs        ${install_dir}/connector
 fi
 # Copy release note

packaging/tools/makepkg_power.sh

@@ -166,9 +166,18 @@ connector_dir="${code_dir}/connector"
 mkdir -p ${install_dir}/connector
 if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
   cp ${build_dir}/lib/*.jar      ${install_dir}/connector ||:
-  cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
+
+  if [ -d "${connector_dir}/grafanaplugin/dist" ]; then
+    cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin
+  else
+    echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!"
+  fi
+  if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then
+    cp -r ${connector_dir}/go ${install_dir}/connector
+  else
+    echo "WARNING: go connector not found, please check if want to use it!"
+  fi
   cp -r ${connector_dir}/python  ${install_dir}/connector/
-  cp -r ${connector_dir}/go      ${install_dir}/connector
   
   sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/connector/python/linux/python2/taos/cinterface.py
   sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/connector/python/linux/python3/taos/cinterface.py

snap/snapcraft.yaml

 name: tdengine
 base: core18
-version: '2.0.20.2'
+version: '2.0.20.5'
 icon: snap/gui/t-dengine.svg
 summary: an open-source big data platform designed and optimized for IoT. 
 description: |
@@ -72,7 +72,7 @@ parts:
       - usr/bin/taosd
       - usr/bin/taos
       - usr/bin/taosdemo
-      - usr/lib/libtaos.so.2.0.20.2
+      - usr/lib/libtaos.so.2.0.20.5
       - usr/lib/libtaos.so.1
       - usr/lib/libtaos.so
 

go @ 8ce6d865

-Subproject commit 7a26c432f8b4203e42344ff3290b9b9b01b983d5
+Subproject commit 8ce6d86558afc8c0b50c10f990fd2b4270cf06fc

src/kit/taosdemo/taosdemo.c

@@ -391,7 +391,7 @@ typedef struct SuperQueryInfo_S {
   uint64_t     sqlCount;
   char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
   char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
-  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
+  int          resubAfterConsume;
   TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
 
   char*        childTblName;
@@ -1114,21 +1114,21 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
 
 static void appendResultBufToFile(char *resultBuf, char *resultFile)
 {
-  FILE *fp = NULL;
-  if (resultFile[0] != 0) {
-    fp = fopen(resultFile, "at");
-    if (fp == NULL) {
-      errorPrint(
-              "%s() LN%d, failed to open result file: %s, result will not save to file\n",
-              __func__, __LINE__, resultFile);
-      return;
+    FILE *fp = NULL;
+    if (resultFile[0] != 0) {
+        fp = fopen(resultFile, "at");
+        if (fp == NULL) {
+            errorPrint(
+                    "%s() LN%d, failed to open result file: %s, result will not save to file\n",
+                    __func__, __LINE__, resultFile);
+            return;
+        }
+        fprintf(fp, "%s", resultBuf);
+        tmfclose(fp);
     }
-  }
-
-  fprintf(fp, "%s", resultBuf);
-  tmfclose(fp);
 }
 
+
 static void appendResultToFile(TAOS_RES *res, char* resultFile) {
   TAOS_ROW    row = NULL;
   int         num_rows = 0;
@@ -4447,6 +4447,18 @@ static bool getMetaFromQueryJsonFile(cJSON* root) {
       g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
     }
 
+    cJSON* superResubAfterConsume =
+            cJSON_GetObjectItem(superQuery, "resubAfterConsume");
+    if (superResubAfterConsume
+            && superResubAfterConsume->type == cJSON_Number) {
+        g_queryInfo.superQueryInfo.resubAfterConsume =
+            superResubAfterConsume->valueint;
+    } else if (!superResubAfterConsume) {
+        //printf("failed to read json, subscribe interval no found\n");
+        ////goto PARSE_OVER;
+        g_queryInfo.superQueryInfo.resubAfterConsume = 1;
+    }
+
     // sqls
     cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls");
     if (!subsqls) {
@@ -4478,18 +4490,6 @@ static bool getMetaFromQueryJsonFile(cJSON* root) {
         tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring,
             MAX_QUERY_SQL_LENGTH);
 
-        cJSON* superResubAfterConsume =
-            cJSON_GetObjectItem(sql, "resubAfterConsume");
-        if (superResubAfterConsume
-                && superResubAfterConsume->type == cJSON_Number) {
-            g_queryInfo.superQueryInfo.resubAfterConsume[j] =
-                superResubAfterConsume->valueint;
-        } else if (!superResubAfterConsume) {
-            //printf("failed to read json, subscribe interval no found\n");
-            //goto PARSE_OVER;
-            g_queryInfo.superQueryInfo.resubAfterConsume[j] = 1;
-        }
-
         cJSON *result = cJSON_GetObjectItem(sql, "result");
         if (result != NULL && result->type == cJSON_String
             && result->valuestring != NULL){
@@ -6612,15 +6612,6 @@ static void *superSubscribe(void *sarg) {
       exit(-1);
   }
 
-  if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
-      errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n",
-              g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables,
-              g_queryInfo.superQueryInfo.sqlCount,
-              pThreadInfo->ntables,
-              MAX_QUERY_SQL_COUNT);
-      exit(-1);
-  }
-
   if (pThreadInfo->taos == NULL) {
     TAOS * taos = NULL;
     taos = taos_connect(g_queryInfo.host,
@@ -6688,6 +6679,8 @@ static void *superSubscribe(void *sarg) {
   }
   TAOS_RES* res = NULL;
 
+  uint64_t st = 0, et = 0;
+
   while(1) {
     for (uint64_t i = pThreadInfo->start_table_from;
             i <= pThreadInfo->end_table_to; i++) {
@@ -6696,7 +6689,12 @@ static void *superSubscribe(void *sarg) {
           continue;
       }
 
+      st = taosGetTimestampMs();
+      performancePrint("st: %"PRIu64" et: %"PRIu64" st-et: %"PRIu64"\n", st, et, (st - et));
       res = taos_consume(tsub[tsubSeq]);
+      et = taosGetTimestampMs();
+      performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st));
+
       if (res) {
           if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
               sprintf(pThreadInfo->fp, "%s-%d",
@@ -6714,7 +6712,7 @@ static void *superSubscribe(void *sarg) {
 
           if ((g_queryInfo.superQueryInfo.subscribeKeepProgress)
                   && (consumed[tsubSeq] >=
-                      g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
+                      g_queryInfo.superQueryInfo.resubAfterConsume)) {
               printf("keepProgress:%d, resub super table query: %"PRIu64"\n",
                       g_queryInfo.superQueryInfo.subscribeKeepProgress,
                       pThreadInfo->querySeq);

src/sync/src/syncMain.c

@@ -709,7 +709,7 @@ static void syncChooseMaster(SSyncNode *pNode) {
 }
 
 static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
-  int32_t onlineNum = 0;
+  int32_t onlineNum = 0, arbOnlineNum = 0;
   int32_t masterIndex = -1;
   int32_t replica = pNode->replica;
 
@@ -723,13 +723,15 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
   SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
   if (pArb && pArb->role != TAOS_SYNC_ROLE_OFFLINE) {
     onlineNum++;
+    ++arbOnlineNum;
     replica = pNode->replica + 1;
   }
 
   if (onlineNum <= replica * 0.5) {
     if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) {
-       if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && onlineNum >= 1) {
+      if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && ((replica > 2 && onlineNum - arbOnlineNum > 1) || pNode->replica < 3)) {
          sInfo("vgId:%d, self keep work as master, online:%d replica:%d", pNode->vgId, onlineNum, replica);
+	 masterIndex = pNode->selfIndex;
        } else {
         nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
         sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
@@ -1002,6 +1004,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) {
   if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
     // nodeVersion = pHead->version;
     code = (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
+    syncConfirmForward(pNode->rid, pHead->version, code, false);
   } else {
     if (nodeSStatus != TAOS_SYNC_STATUS_INIT) {
       code = syncSaveIntoBuffer(pPeer, pHead);
@@ -1404,7 +1407,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) {
       pthread_mutex_lock(&pNode->mutex);
       for (int32_t i = 0; i < pSyncFwds->fwds; ++i) {
         SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % SYNC_MAX_FWDS;
-        if (ABS(time - pFwdInfo->time) < 2000) break;
+        if (ABS(time - pFwdInfo->time) < 10000) break;
 
         sDebug("vgId:%d, forward info expired, hver:%" PRIu64 " curtime:%" PRIu64 " savetime:%" PRIu64, pNode->vgId,
                pFwdInfo->version, time, pFwdInfo->time);

src/util/src/tcache.c

@@ -613,7 +613,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) {
 
   // todo memory leak if there are object with refcount greater than 0 in hash table?
   taosHashCleanup(pCacheObj->pHashTable);
-  taosTrashcanEmpty(pCacheObj, true);
+  taosTrashcanEmpty(pCacheObj, false);
 
   __cache_lock_destroy(pCacheObj);
   

tests/perftest-scripts/perftest-query.sh

@@ -64,18 +64,25 @@ function runQueryPerfTest {
 	[ -f $PERFORMANCE_TEST_REPORT ] && rm $PERFORMANCE_TEST_REPORT
 	nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/taosperf/ > /dev/null 2>&1 &
 	echoInfo "Wait TDengine to start"
-	sleep 300
+	sleep 60
 	echoInfo "Run Performance Test"	
 	cd $WORK_DIR/TDengine/tests/pytest
 	
 	python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
 
+	mkdir -p /var/lib/perf/
+	mkdir -p /var/log/perf/
+	rm -rf /var/lib/perf/*
+	rm -rf /var/log/perf/*
+	nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/perf/ > /dev/null 2>&1 &	
+	echoInfo "Wait TDengine to start"
+	sleep 10
+	echoInfo "Run Performance Test"	
+	cd $WORK_DIR/TDengine/tests/pytest
+
 	python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
 	
 	python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
-
-	#python3 perfbenchmark/joinPerformance.py  | tee -a $PERFORMANCE_TEST_REPORT
-	
 }
 
 

tests/pytest/fulltest.sh

@@ -183,7 +183,7 @@ python3 ./test.py -f stable/query_after_reset.py
 # perfbenchmark
 python3 ./test.py -f perfbenchmark/bug3433.py
 #python3 ./test.py -f perfbenchmark/bug3589.py
-
+python3 ./test.py -f perfbenchmark/taosdemoInsert.py
 
 #query
 python3 ./test.py -f query/filter.py

tests/pytest/insert/insertFromCSVPerformance.py

@@ -31,7 +31,7 @@ class insertFromCSVPerformace:
         self.host = "127.0.0.1"
         self.user = "root"
         self.password = "taosdata"
-        self.config = "/etc/taosperf"
+        self.config = "/etc/perf"
         self.conn = taos.connect(
             self.host,
             self.user,

tests/pytest/perfbenchmark/taosdemoInsert.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import taos
+import sys
+import os
+import json
+import argparse
+import subprocess
+import datetime
+import re
+
+
+from multiprocessing import cpu_count
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.dnodes import TDDnode
+
+class Taosdemo:
+    def __init__(self, clearCache, dbName, keep):
+        self.clearCache = clearCache
+        self.dbname = dbName
+        self.drop = "yes"
+        self.keep = keep
+        self.host = "127.0.0.1"
+        self.user = "root"
+        self.password = "taosdata"
+        # self.config = "/etc/taosperf"
+        # self.conn = taos.connect(
+        #     self.host,
+        #     self.user,
+        #     self.password,
+        #     self.config)
+
+    # env config
+    def getBuildPath(self) -> str:
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/debug/build/bin")]
+                    break
+        return buildPath
+
+    def getExeToolsDir(self) -> str:
+        self.debugdir = self.getBuildPath() + "/debug/build/bin"
+        return self.debugdir
+
+    def getCfgDir(self) -> str:
+        self.config = self.getBuildPath() + "/sim/dnode1/cfg"
+        return self.config
+
+    # taodemo insert file config
+    def dbinfocfg(self) -> dict:
+        return {
+            "name": self.dbname,
+            "drop": self.drop,
+            "replica": 1,
+            "days": 10,
+            "cache": 16,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": self.keep,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp": 2,
+            "walLevel": 1,
+            "cachelast": 0,
+            "quorum": 1,
+            "fsync": 3000,
+            "update": 0
+        }
+
+    def type_check(func):
+        def wrapper(self, **kwargs):
+            num_types = ["int", "float", "bigint", "tinyint", "smallint", "double"]
+            str_types = ["binary", "nchar"]
+            for k ,v in kwargs.items():
+                if k.lower() not in num_types and k.lower() not in str_types:
+                    return f"args {k} type error, not allowed"
+                elif not isinstance(v, (int, list, tuple)):
+                    return f"value {v} type error, not allowed"
+                elif k.lower() in num_types and not isinstance(v, int):
+                    return f"arg {v} takes 1 positional argument must be type int "
+                elif isinstance(v, (list,tuple)) and len(v) > 2:
+                    return f"arg {v} takes from 1 to 2 positional arguments but more than 2 were given "
+                elif isinstance(v,(list,tuple))  and [ False for _ in v if not isinstance(_, int) ]:
+                    return f"arg {v} takes from 1 to 2 positional arguments must be type int "
+                else:
+                    pass
+            return func(self, **kwargs)
+        return wrapper
+
+    @type_check
+    def column_tag_count(self, **column_tag) -> list :
+        init_column_tag = []
+        for k, v in column_tag.items():
+            if re.search(k, "int, float, bigint, tinyint, smallint, double", re.IGNORECASE):
+                init_column_tag.append({"type": k, "count": v})
+            elif re.search(k, "binary, nchar", re.IGNORECASE):
+                if isinstance(v, int):
+                    init_column_tag.append({"type": k, "count": v, "len":8})
+                elif len(v) == 1:
+                    init_column_tag.append({"type": k, "count": v[0], "len": 8})
+                else:
+                    init_column_tag.append({"type": k, "count": v[0], "len": v[1]})
+        return init_column_tag
+
+    def stbcfg(self, stb: str, child_tab_count: int, rows: int, prechildtab: str, columns: dict, tags: dict) -> dict:
+        return {
+            "name": stb,
+            "child_table_exists": "no",
+            "childtable_count": child_tab_count,
+            "childtable_prefix": prechildtab,
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 10,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": rows,
+            "childtable_limit": 0,
+            "childtable_offset": 0,
+            "rows_per_tbl": 1,
+            "max_sql_len": 65480,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 10,
+            "start_timestamp": f"{datetime.datetime.now():%F %X}",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": self.column_tag_count(**columns),
+            "tags": self.column_tag_count(**tags)
+        }
+
+    def schemecfg(self,intcount=1,floatcount=0,bcount=0,tcount=0,scount=0,doublecount=0,binarycount=0,ncharcount=0):
+        return  {
+            "INT": intcount,
+            "FLOAT": floatcount,
+            "BIGINT": bcount,
+            "TINYINT": tcount,
+            "SMALLINT": scount,
+            "DOUBLE": doublecount,
+            "BINARY": binarycount,
+            "NCHAR": ncharcount
+        }
+
+    def insertcfg(self,db: dict, stbs: list) -> dict:
+        return {
+            "filetype": "insert",
+            "cfgdir": self.config,
+            "host": self.host,
+            "port": 6030,
+            "user": self.user,
+            "password": self.password,
+            "thread_count": cpu_count(),
+            "thread_count_create_tbl": cpu_count(),
+            "result_file": "/tmp/insert_res.txt",
+            "confirm_parameter_prompt": "no",
+            "insert_interval": 0,
+            "num_of_records_per_req": 100,
+            "max_sql_len": 1024000,
+            "databases": [{
+                "dbinfo": db,
+                "super_tables": stbs
+            }]
+        }
+
+    def createinsertfile(self,db: dict, stbs: list) -> str:
+        date = datetime.datetime.now()
+        file_create_table = f"/tmp/insert_{date:%F-%H%M}.json"
+
+        with open(file_create_table, 'w') as f:
+            json.dump(self.insertcfg(db, stbs), f)
+
+        return file_create_table
+
+    # taosdemo query file config
+    def querysqls(self, sql: str) -> list:
+        return [{"sql":sql,"result":""}]
+
+    def querycfg(self, sql: str) -> dict:
+        return {
+            "filetype": "query",
+            "cfgdir": self.config,
+            "host": self.host,
+            "port": 6030,
+            "user": self.user,
+            "password": self.password,
+            "confirm_parameter_prompt": "yes",
+            "query_times": 10,
+            "query_mode": "taosc",
+            "databases": self.dbname,
+            "specified_table_query": {
+                "query_interval": 0,
+                "concurrent": cpu_count(),
+                "sqls": self.querysqls(sql)
+            }
+        }
+
+    def createqueryfile(self, sql: str):
+        date = datetime.datetime.now()
+        file_query_table = f"/tmp/query_{date:%F-%H%M}.json"
+
+        with open(file_query_table,"w") as f:
+            json.dump(self.querycfg(sql), f)
+
+        return file_query_table
+
+    # Execute taosdemo, and delete temporary files when finished
+    def taosdemotable(self, filepath: str, resultfile="/dev/null"):
+        taosdemopath = self.getBuildPath() + "/debug/build/bin"
+        with open(filepath,"r") as f:
+            filetype = json.load(f)["filetype"]
+            if filetype == "insert":
+                taosdemo_table_cmd = f"{taosdemopath}/taosdemo -f {filepath}  > {resultfile} 2>&1"
+            else:
+                taosdemo_table_cmd = f"yes | {taosdemopath}/taosdemo -f {filepath}  > {resultfile} 2>&1"
+        try:
+            _ = subprocess.check_output(taosdemo_table_cmd, shell=True).decode("utf-8")
+        except subprocess.CalledProcessError as e:
+            _ = e.output
+
+    def droptmpfile(self, filepath: str):
+        drop_file_cmd = f"[ -f {filepath} ] && rm -f  {filepath}"
+        try:
+            _ = subprocess.check_output(drop_file_cmd, shell=True).decode("utf-8")
+        except subprocess.CalledProcessError as e:
+            _ = e.output
+
+    # TODO:需要完成TD-4153的数据插入和客户端请求的性能查询。
+    def td4153insert(self):
+
+        tdLog.printNoPrefix("========== start to create table and insert data ==========")
+        self.dbname = "td4153"
+        db = self.dbinfocfg()
+        stblist = []
+
+        columntype = self.schemecfg(intcount=1, ncharcount=100)
+        tagtype = self.schemecfg(intcount=1)
+        stbname = "stb1"
+        prechild = "t1"
+        stable = self.stbcfg(
+            stb=stbname,
+            prechildtab=prechild,
+            child_tab_count=2,
+            rows=10000,
+            columns=columntype,
+            tags=tagtype
+        )
+        stblist.append(stable)
+        insertfile = self.createinsertfile(db=db, stbs=stblist)
+
+        nmon_file = f"/tmp/insert_{datetime.datetime.now():%F-%H%M}.nmon"
+        cmd = f"nmon -s5 -F {nmon_file} -m /tmp/"
+        try:
+            _ = subprocess.check_output(cmd, shell=True).decode("utf-8")
+        except subprocess.CalledProcessError as e:
+            _ = e.output
+
+        self.taosdemotable(insertfile)
+        self.droptmpfile(insertfile)
+        self.droptmpfile("/tmp/insert_res.txt")
+
+        # In order to prevent too many performance files from being generated, the nmon file is deleted.
+        # and the delete statement can be cancelled during the actual test.
+        self.droptmpfile(nmon_file)
+
+        cmd = f"ps -ef|grep -w nmon| grep -v grep | awk '{{print $2}}'"
+        try:
+            time.sleep(10)
+            _ = subprocess.check_output(cmd,shell=True).decode("utf-8")
+        except BaseException as e:
+            raise e
+
+    def td4153query(self):
+        tdLog.printNoPrefix("========== start to query operation ==========")
+
+        sqls = {
+            "select_all": "select * from stb1",
+            "select_join": "select * from t10, t11 where t10.ts=t11.ts"
+        }
+        for type, sql in sqls.items():
+            result_file = f"/tmp/queryResult_{type}.log"
+            query_file = self.createqueryfile(sql)
+            try:
+                self.taosdemotable(query_file, resultfile=result_file)
+            except subprocess.CalledProcessError as e:
+                out_put = e.output
+            if result_file:
+                print(f"execute rows {type.split('_')[1]} sql, the sql is: {sql}")
+                max_sql_time_cmd = f'''
+                grep -o Spent.*s {result_file} |awk 'NR==1{{max=$2;next}}{{max=max>$2?max:$2}}END{{print "Max=",max,"s"}}'
+                 '''
+                max_sql_time = subprocess.check_output(max_sql_time_cmd, shell=True).decode("UTF-8")
+                print(f"{type.split('_')[1]} rows sql time : {max_sql_time}")
+
+                min_sql_time_cmd = f'''
+                grep -o Spent.*s {result_file} |awk 'NR==1{{min=$2;next}}{{min=min<$2?min:$2}}END{{print "Min=",min,"s"}}'
+                '''
+                min_sql_time = subprocess.check_output(min_sql_time_cmd, shell=True).decode("UTF-8")
+                print(f"{type.split('_')[1]} rows sql time : {min_sql_time}")
+
+                avg_sql_time_cmd = f'''
+                grep -o Spent.*s {result_file} |awk '{{sum+=$2}}END{{print "Average=",sum/NR,"s"}}'
+                '''
+                avg_sql_time = subprocess.check_output(avg_sql_time_cmd, shell=True).decode("UTF-8")
+                print(f"{type.split('_')[1]} rows sql time : {avg_sql_time}")
+
+            self.droptmpfile(query_file)
+            self.droptmpfile(result_file)
+
+        drop_query_tmt_file_cmd = " find ./ -name 'querySystemInfo-*'  -type f -exec rm {} \; "
+        try:
+            _ = subprocess.check_output(drop_query_tmt_file_cmd, shell=True).decode("utf-8")
+        except subprocess.CalledProcessError as e:
+            _ = e.output
+        pass
+
+    def td4153(self):
+        self.td4153insert()
+        self.td4153query()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-r',
+        '--remove-cache',
+        action='store_true',
+        default=False,
+        help='clear cache before query (default: False)')
+    parser.add_argument(
+        '-d',
+        '--database-name',
+        action='store',
+        default='db',
+        type=str,
+        help='Database name to be created (default: db)')
+    parser.add_argument(
+        '-k',
+        '--keep-time',
+        action='store',
+        default=3650,
+        type=int,
+        help='Database keep parameters (default: 3650)')
+
+    args = parser.parse_args()
+    taosdemo = Taosdemo(args.remove_cache, args.database_name, args.keep_time)
+    # taosdemo.conn = taos.connect(
+    #     taosdemo.host,
+    #     taosdemo.user,
+    #     taosdemo.password,
+    #     taosdemo.config
+    # )
+
+    debugdir = taosdemo.getExeToolsDir()
+    cfgdir = taosdemo.getCfgDir()
+    cmd = f"{debugdir}/taosd -c {cfgdir} >/dev/null 2>&1 &"
+    try:
+        _ = subprocess.check_output(cmd, shell=True).decode("utf-8")
+    except subprocess.CalledProcessError as e:
+        _ = e.output
+
+    if taosdemo.clearCache:
+        # must be root permission
+        subprocess.check_output("echo 3 > /proc/sys/vm/drop_caches", shell=True).decode("utf-8")
+
+    taosdemo.td4153()

tests/pytest/tools/taosdemoPerformance.py

@@ -24,7 +24,7 @@ class taosdemoPerformace:
         self.host = "127.0.0.1"
         self.user = "root"
         self.password = "taosdata"
-        self.config = "/etc/taosperf"
+        self.config = "/etc/perf"
         self.conn = taos.connect(
             self.host,
             self.user,
@@ -77,7 +77,7 @@ class taosdemoPerformace:
 
         insert_data = {
             "filetype": "insert",
-            "cfgdir": "/etc/taosperf",
+            "cfgdir": "/etc/perf",
             "host": "127.0.0.1",
             "port": 6030,
             "user": "root",

tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim

@@ -158,7 +158,7 @@ if $dnode4Vtatus != offline then
   sleep 2000
   goto wait_dnode4_vgroup_offline
 endi
-if $dnode3Vtatus != master then
+if $dnode3Vtatus != unsynced then
   sleep 2000
   goto wait_dnode4_vgroup_offline
 endi