Merge branch '3.0' of https://github.com/taosdata/TDengine into enh/row_optimize

tests/pytest/util/dnodes.py

@@ -116,6 +116,7 @@ class TDDnode:
         self.deployed = 0
         self.testCluster = False
         self.valgrind = 0
+        self.asan = False
         self.remoteIP = ""
         self.cfgDict = {
             "monitor": "0",
@@ -158,6 +159,11 @@ class TDDnode:
     def setValgrind(self, value):
         self.valgrind = value
 
+    def setAsan(self, value):
+        self.asan = value
+        if value:
+            self.execPath = os.path.abspath(self.path + "/tests/script/sh/exec.sh")        
+
     def getDataSize(self):
         totalSize = 0
 
@@ -383,8 +389,14 @@ class TDDnode:
                 cmd = "mintty -h never %s -c %s" % (
                     binPath, self.cfgDir)
             else:
-                cmd = "nohup %s -c %s > /dev/null 2>&1 & " % (
-                    binPath, self.cfgDir)
+                if self.asan:
+                    asanDir = "%s/sim/asan/dnode%d.asan" % (
+                        self.path, self.index)
+                    cmd = "nohup %s -c %s > /dev/null 2> %s & " % (
+                        binPath, self.cfgDir, asanDir)
+                else:
+                    cmd = "nohup %s -c %s > /dev/null 2>&1 & " % (
+                        binPath, self.cfgDir)
         else:
             valgrindCmdline = "valgrind --log-file=\"%s/../log/valgrind.log\"  --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"%self.cfgDir
 
@@ -444,8 +456,14 @@ class TDDnode:
             tdLog.exit("dnode:%d is not deployed" % (self.index))
 
         if self.valgrind == 0:
-            cmd = "nohup %s -c %s > /dev/null 2>&1 & " % (
-                binPath, self.cfgDir)
+            if self.asan:
+               asanDir = "%s/sim/asan/dnode%d.asan" % (
+                   self.path, self.index)
+               cmd = "nohup %s -c %s > /dev/null 2> %s & " % (
+                   binPath, self.cfgDir, asanDir)
+            else:
+                cmd = "nohup %s -c %s > /dev/null 2>&1 & " % (
+                    binPath, self.cfgDir)
         else:
             valgrindCmdline = "valgrind  --log-file=\"%s/../log/valgrind.log\"  --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"%self.cfgDir
 
@@ -464,6 +482,12 @@ class TDDnode:
         tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
 
     def stop(self):
+        if self.asan:
+            stopCmd = "%s -s stop -n dnode%d" % (self.execPath, self.index)
+            tdLog.info("execute script: " + stopCmd)
+            os.system(stopCmd)
+            return
+
         if (not self.remoteIP == ""):
             self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].stop()"%(self.index-1,self.index-1))
             tdLog.info("stop dnode%d"%self.index)
@@ -501,6 +525,12 @@ class TDDnode:
 
 
     def stoptaosd(self):
+        if self.asan:
+            stopCmd = "%s -s stop -n dnode%d" % (self.execPath, self.index)
+            tdLog.info("execute script: " + stopCmd)
+            os.system(stopCmd)
+            return
+
         if (not self.remoteIP == ""):
             self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].stop()"%(self.index-1,self.index-1))
             tdLog.info("stop dnode%d"%self.index)
@@ -534,6 +564,13 @@ class TDDnode:
             tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index))
 
     def forcestop(self):
+        if self.asan:
+            stopCmd = "%s -s stop -n dnode%d -x SIGKILL" + \
+                (self.execPath, self.index)
+            tdLog.info("execute script: " + stopCmd)
+            os.system(stopCmd)
+            return
+
         if (not self.remoteIP == ""):
             self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].forcestop()"%(self.index-1,self.index-1))
             return
@@ -606,6 +643,7 @@ class TDDnodes:
         self.simDeployed = False
         self.testCluster = False
         self.valgrind = 0
+        self.asan = False
         self.killValgrind = 1
 
     def init(self, path, remoteIP = ""):
@@ -629,6 +667,13 @@ class TDDnodes:
     def setValgrind(self, value):
         self.valgrind = value
 
+    def setAsan(self, value):
+        self.asan = value
+        if value:
+            self.stopDnodesPath = os.path.abspath(self.path + "/tests/script/sh/stop_dnodes.sh")
+            self.stopDnodesSigintPath = os.path.abspath(self.path + "/tests/script/sh/sigint_stop_dnodes.sh")
+            tdLog.info("run in address sanitizer mode")
+
     def setKillValgrind(self, value):
         self.killValgrind = value
 
@@ -642,6 +687,7 @@ class TDDnodes:
         self.check(index)
         self.dnodes[index - 1].setTestCluster(self.testCluster)
         self.dnodes[index - 1].setValgrind(self.valgrind)
+        self.dnodes[index - 1].setAsan(self.asan)
         self.dnodes[index - 1].deploy(updatecfgDict)
 
     def cfg(self, index, option, value):
@@ -692,8 +738,22 @@ class TDDnodes:
         if index < 1 or index > 10:
             tdLog.exit("index:%d should on a scale of [1, 10]" % (index))
 
+    def StopAllSigint(self):
+        tdLog.info("stop all dnodes sigint")
+        if self.asan:
+            tdLog.info("execute script: %s" % self.stopDnodesSigintPath)
+            os.system(self.stopDnodesSigintPath)
+            tdLog.info("execute finished")
+            return
+
     def stopAll(self):
         tdLog.info("stop all dnodes")
+        if self.asan:
+            tdLog.info("execute script: %s" % self.stopDnodesPath)
+            os.system(self.stopDnodesPath)
+            tdLog.info("execute finished")
+            return
+
         if (not self.dnodes[0].remoteIP == ""):
             self.dnodes[0].remoteExec(self.dnodes[0].cfgDict, "for i in range(len(tdDnodes.dnodes)):\n    tdDnodes.dnodes[i].running=1\ntdDnodes.stopAll()")
             return

tests/pytest/util/log.py

@@ -33,13 +33,13 @@ class TDLog:
         print("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err))
 
     def success(self, info):
-        printf("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info))
+        print("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info))
 
     def notice(self, err):
-        printf("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err))
+        print("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err))
 
     def exit(self, err):
-        printf("\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err))
+        print("\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err))
         sys.exit(1)
 
     def printNoPrefix(self, info):

tests/script/sh/checkAsan.sh

@@ -15,7 +15,7 @@ else
 fi
 
 TAOS_DIR=`pwd`
-LOG_DIR=$TAOS_DIR/sim/tsim/asan
+LOG_DIR=$TAOS_DIR/sim/asan
 
 error_num=`cat ${LOG_DIR}/*.asan | grep "ERROR" | wc -l`
 memory_leak=`cat ${LOG_DIR}/*.asan | grep "Direct leak" | wc -l`

tests/script/sh/exec.sh

@@ -80,7 +80,7 @@ LOG_DIR=$NODE_DIR/log
 DATA_DIR=$NODE_DIR/data
 MGMT_DIR=$NODE_DIR/data/mgmt
 TSDB_DIR=$NODE_DIR/data/tsdb
-ASAN_DIR=$SIM_DIR/tsim/asan
+ASAN_DIR=$SIM_DIR/asan
 TAOS_CFG=$NODE_DIR/cfg/taos.cfg
 
 echo ------------ $EXEC_OPTON $NODE_NAME

tests/script/sh/sigint_stop_dnodes.sh

+#!/bin/sh
+
+set +e
+#set -x
+
+export LD_PRELOAD=
+UNAME_BIN=`which uname`
+OS_TYPE=`$UNAME_BIN`
+
+PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+while [ -n "$PID" ]; do
+  #echo "Killing taosd processes " $PID
+  kill $PID
+
+  PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+done

tests/script/sh/stop_dnodes.sh

 #!/bin/sh
 
+set +e
+#set -x
+
 UNAME_BIN=`which uname`
 OS_TYPE=`$UNAME_BIN`
 
+export LD_PRELOAD=
 PID=`ps -ef|grep /usr/bin/taosd | grep -v grep | awk '{print $2}'`
 if [ -n "$PID" ]; then
   echo systemctl stop taosd
@@ -22,16 +26,3 @@ while [ -n "$PID" ]; do
   fi
   PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
 done
-
-PID=`ps -ef|grep -w tarbitrator | grep -v grep | awk '{print $2}'`
-while [ -n "$PID" ]; do
-  echo kill -9 $PID
-  pkill -9 tarbitrator
-  if [ "$OS_TYPE" != "Darwin" ]; then
-    fuser -k -n tcp 6040
-  else
-    lsof -nti:6040 | xargs kill -9
-  fi
-  PID=`ps -ef|grep -w tarbitrator | grep -v grep | awk '{print $2}'`
-done
-

tests/script/test.sh

@@ -66,16 +66,13 @@ else
 fi
 
 declare -x BUILD_DIR=$TOP_DIR/$BIN_DIR
-
 declare -x SIM_DIR=$TOP_DIR/sim
-
 PROGRAM=$BUILD_DIR/build/bin/tsim
-
 PRG_DIR=$SIM_DIR/tsim
 CFG_DIR=$PRG_DIR/cfg
 LOG_DIR=$PRG_DIR/log
 DATA_DIR=$PRG_DIR/data
-ASAN_DIR=$PRG_DIR/asan
+ASAN_DIR=$SIM_DIR/asan
 
 chmod -R 777 $PRG_DIR
 echo "------------------------------------------------------------------------"
@@ -141,11 +138,14 @@ if [ -n "$FILE_NAME" ]; then
     echo "AsanDir:" $ASAN_DIR/tsim.asan
     eval $PROGRAM -c $CFG_DIR -f $FILE_NAME 2> $ASAN_DIR/tsim.asan
     result=$?
-    echo "Execute result: " $result
+    echo "Execute result:" $result
 
     if [ $result -eq 0 ]; then
       $CODE_DIR/sh/checkAsan.sh
     else
+      echo "TSIM has asan errors"
+      sleep 1
+      $CODE_DIR/sh/checkAsan.sh
       exit 1
     fi
   fi

tests/system-test/test.py

@@ -73,8 +73,9 @@ if __name__ == "__main__":
     createDnodeNums = 1
     restful = False
     replicaVar = 1
+    asan = False
     independentMnode = True
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:i:', [
+    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:i:a', [
         'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate','replicaVar','independentMnode'])
     for key, value in opts:
         if key in ['-h', '--help']:
@@ -99,6 +100,7 @@ if __name__ == "__main__":
             tdLog.printNoPrefix('-D taosadapter update cfg dict ')
             tdLog.printNoPrefix('-n the number of replicas')
             tdLog.printNoPrefix('-i independentMnode Mnode')
+            tdLog.printNoPrefix('-a address sanitizer mode')
 
             sys.exit(0)
 
@@ -167,6 +169,9 @@ if __name__ == "__main__":
         if key in ['-R', '--restful']:
             restful = True
 
+        if key in ['-a', '--asan']:
+            asan = True
+
         if key in ['-D', '--adaptercfgupdate']:
             try:
                 adaptercfgupdate = eval(base64.b64decode(value.encode()).decode())
@@ -387,6 +392,7 @@ if __name__ == "__main__":
         tdDnodes.init(deployPath, masterIp)
         tdDnodes.setTestCluster(testCluster)
         tdDnodes.setValgrind(valgrind)
+        tdDnodes.setAsan(asan)
         tdDnodes.stopAll()
         is_test_framework = 0
         key_word = 'tdCases.addLinux'
@@ -547,4 +553,7 @@ if __name__ == "__main__":
 
     if conn is not None:
         conn.close()
+    if asan:
+        tdDnodes.StopAllSigint()
+        tdLog.info("address sanitizer mode finished")
     sys.exit(0)

tests/system-test/test.sh

+#!/bin/bash
+
+##################################################
+#
+# Do simulation test
+#
+##################################################
+
+set +e
+#set -x
+
+UNAME_BIN=`which uname`
+OS_TYPE=`$UNAME_BIN`
+
+cd .
+
+# Get responsible directories
+CODE_DIR=`dirname $0`
+CODE_DIR=`pwd`
+
+IN_TDINTERNAL="community"
+if [[ "$CODE_DIR" == *"$IN_TDINTERNAL"* ]]; then
+  cd ../../..
+else
+  cd ../../
+fi
+
+TOP_DIR=`pwd`
+TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1`
+
+if [[ "$OS_TYPE" != "Darwin" ]]; then
+  cut_opt="--field="
+else
+  cut_opt="-f "
+fi
+
+if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then
+  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3`
+else
+  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2`
+fi
+
+declare -x BUILD_DIR=$TOP_DIR/$BIN_DIR
+declare -x SIM_DIR=$TOP_DIR/sim
+PROGRAM=$BUILD_DIR/build/bin/tsim
+PRG_DIR=$SIM_DIR/tsim
+ASAN_DIR=$SIM_DIR/asan
+SYSTEM_TEST_DIR=$TOP_DIR/tests/system-test
+
+chmod -R 777 $PRG_DIR
+echo "------------------------------------------------------------------------"
+echo "Start TDengine Testing Case ..."
+echo "BUILD_DIR: $BUILD_DIR"
+echo "SYSTEM_TEST_DIR : $SYSTEM_TEST_DIR"
+echo "SIM_DIR  : $SIM_DIR"
+echo "CODE_DIR : $CODE_DIR"
+echo "ASAN_DIR  : $ASAN_DIR"
+
+rm -rf $SIM_DIR/*
+
+mkdir -p $PRG_DIR
+mkdir -p $ASAN_DIR
+
+cd $SYSTEM_TEST_DIR
+ulimit -n 600000
+ulimit -c unlimited
+
+#sudo sysctl -w kernel.core_pattern=$TOP_DIR/core.%p.%e
+
+echo "ExcuteCmd:" $*
+echo "AsanDir:" $ASAN_DIR/psim.asan
+
+export LD_PRELOAD=libasan.so.5
+$* -a 2> $ASAN_DIR/psim.asan
+result=$?
+echo "Execute result:" $result
+
+if [ $result -eq 0 ]; then
+  $CODE_DIR/sh/checkAsan.sh
+else
+  exit 1
+fi
+