From 613bd5c5231409cf66f184bd24925ef275f687f6 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 24 Aug 2021 10:49:47 +0800 Subject: [PATCH] [TD-6169]: windows dll client can not quit. --- src/client/src/tscSystem.c | 3 -- src/connector/python/taos/cinterface.py | 7 ++- src/os/src/detail/osFile.c | 2 +- src/util/src/tcache.c | 4 +- ...o-Run-Test-And-How-To-Add-New-Test-Case.md | 6 +++ tests/pytest/cluster/clusterSetup.py | 4 +- tests/pytest/dockerCluster/basic.py | 4 +- tests/pytest/insert/binary.py | 5 +- tests/pytest/query/queryWithTaosdKilled.py | 3 +- tests/pytest/test.py | 2 +- tests/pytest/util/dnodes-default.py | 39 ++++++++------- tests/pytest/util/dnodes-no-random-fail.py | 39 ++++++++------- tests/pytest/util/dnodes-random-fail.py | 39 ++++++++------- tests/pytest/util/dnodes.py | 50 +++++++++++-------- tests/pytest/wal/addOldWalTest.py | 2 +- 15 files changed, 123 insertions(+), 86 deletions(-) diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 8c8afc8d88..c04765b065 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -199,10 +199,7 @@ void taos_init_imp(void) { // In the APIs of other program language, taos_cleanup is not available yet. // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. - // But in the dll, the child thread will be killed before atexit takes effect.So taos_cleanup is not necessary. -#if !defined(TD_WINDOWS) atexit(taos_cleanup); -#endif tscDebug("client is initialized successfully"); } diff --git a/src/connector/python/taos/cinterface.py b/src/connector/python/taos/cinterface.py index 51e9a8667d..9bb1494f4e 100644 --- a/src/connector/python/taos/cinterface.py +++ b/src/connector/python/taos/cinterface.py @@ -3,6 +3,7 @@ import ctypes import platform import sys +import os from ctypes import * try: from typing import Any @@ -37,7 +38,11 @@ def _load_taos_darwin(): def _load_taos_windows(): - return ctypes.windll.LoadLibrary("taos") + if os.path.exists("c:\\Windows\\System32\\taos.dll"): + return ctypes.windll.LoadLibrary("taos") + else: + print("Please copy the \"C:\\TDengine\\driver\\taos.dll\" file to the \"C:\\windows\\system32\" directory.") + return ctypes.windll.LoadLibrary("C:\\TDengine\\driver\\taos.dll") def _load_taos(): diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index cc12968c72..57e0765750 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -368,7 +368,7 @@ int32_t taosFsync(FileFd fd) { HANDLE h = (HANDLE)_get_osfhandle(fd); - return FlushFileBuffers(h); + return !FlushFileBuffers(h); } int32_t taosRename(char *oldName, char *newName) { diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 69b3741e13..dc1f961725 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -537,7 +537,9 @@ void taosCacheCleanup(SCacheObj *pCacheObj) { pCacheObj->deleting = 1; // wait for the refresh thread quit before destroying the cache object. - while(atomic_load_8(&pCacheObj->deleting) != 0) { + // But in the dll, the child thread will be killed before atexit takes effect.So here we only wait for one second. + while(atomic_load_8(&pCacheObj->deleting)) { + // for (int i = 0; i < 60&&atomic_load_8(&pCacheObj->deleting) != 0; i++) { taosMsleep(50); } diff --git a/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md b/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md index 6845d091b5..c5229aa0e5 100644 --- a/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md +++ b/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md @@ -49,6 +49,12 @@ > before the script line. Then you can look for the core file in > \/tests/pytest after the program crash. +> Note3: if you are on the windows platform, you can install the git client, +> and then add %GitPath%\usr\bin;%GitPath%\mingw64\bin;%GitPath%\bin to the system +> environment variable Path. Note that %GitPath% is the installation path of git, +> such as C:\Program Files\Git. Then you can run the test script using the +> "sh smoketest.sh" command. + ### How to add a new test case diff --git a/tests/pytest/cluster/clusterSetup.py b/tests/pytest/cluster/clusterSetup.py index 87414303f8..809e0e9d25 100644 --- a/tests/pytest/cluster/clusterSetup.py +++ b/tests/pytest/cluster/clusterSetup.py @@ -92,13 +92,13 @@ class Node: self.conn.run("yes|./install.sh") def configTaosd(self, taosConfigKey, taosConfigValue): - self.conn.run("sudo echo '%s %s' >> %s" % (taosConfigKey, taosConfigValue, "/etc/taos/taos.cfg")) + self.conn.run("sudo echo %s %s >> %s" % (taosConfigKey, taosConfigValue, "/etc/taos/taos.cfg")) def removeTaosConfig(self, taosConfigKey, taosConfigValue): self.conn.run("sudo sed -in-place -e '/%s %s/d' %s" % (taosConfigKey, taosConfigValue, "/etc/taos/taos.cfg")) def configHosts(self, ip, name): - self.conn.run("echo '%s %s' >> %s" % (ip, name, '/etc/hosts')) + self.conn.run("echo %s %s >> %s" % (ip, name, '/etc/hosts')) def removeData(self): try: diff --git a/tests/pytest/dockerCluster/basic.py b/tests/pytest/dockerCluster/basic.py index 871d69790d..5188aa4a80 100644 --- a/tests/pytest/dockerCluster/basic.py +++ b/tests/pytest/dockerCluster/basic.py @@ -113,7 +113,7 @@ class BuildDockerCluser: def cfg(self, option, value, nodeIndex): cfgPath = "%s/node%d/cfg/taos.cfg" % (self.dockerDir, nodeIndex) - cmd = "echo '%s %s' >> %s" % (option, value, cfgPath) + cmd = "echo %s %s >> %s" % (option, value, cfgPath) self.execCmd(cmd) def updateLocalhosts(self): @@ -122,7 +122,7 @@ class BuildDockerCluser: print(result) if result is None or result.isspace(): print("==========") - cmd = "echo '172.27.0.7 tdnode1' >> /etc/hosts" + cmd = "echo 172.27.0.7 tdnode1 >> /etc/hosts" display = "echo %s" % cmd self.execCmd(display) self.execCmd(cmd) diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py index 0cbb7876c6..44e42bec03 100644 --- a/tests/pytest/insert/binary.py +++ b/tests/pytest/insert/binary.py @@ -53,8 +53,9 @@ class TDTestCase: tdLog.info("tdSql.checkData(0, 0, '34567')") tdSql.checkData(0, 0, '34567') tdLog.info("insert into tb values (now+4a, \"'';\")") - config_dir = subprocess.check_output(str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), stderr=subprocess.STDOUT, shell=True).decode('utf-8').replace('\n', '') - result = ''.join(os.popen(r"""taos -s "insert into db.tb values (now+4a, \"'';\")" -c %s"""%(config_dir)).readlines()) + # config_dir = subprocess.check_output(str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), stderr=subprocess.STDOUT, shell=True).decode('utf-8').replace('\n', '') + # result = ''.join(os.popen(r"""taos -s "insert into db.tb values (now+4a, \"'';\")" -c %s"""%(config_dir)).readlines()) + result = ''.join(os.popen(r"""taos -s "insert into db.tb values (now+4a, \"'';\")" -c %s"""%(tdSql.cursor._connection._config)).readlines()) if "Query OK" not in result: tdLog.exit("err:insert '';") tdLog.info('drop database db') tdSql.execute('drop database db') diff --git a/tests/pytest/query/queryWithTaosdKilled.py b/tests/pytest/query/queryWithTaosdKilled.py index 28f9b87636..fb1384093c 100644 --- a/tests/pytest/query/queryWithTaosdKilled.py +++ b/tests/pytest/query/queryWithTaosdKilled.py @@ -34,7 +34,8 @@ class TDTestCase: path = tdDnodes.dnodes[1].getDnodeRootDir(1) print(path) tdLog.info("sudo mkdir -p %s/data/vnode/vnode2/wal/old" % path) - os.system("sudo mkdir -p %s/data/vnode/vnode2/wal/old" % path) + # os.system("sudo mkdir -p %s/data/vnode/vnode2/wal/old" % path) + os.makedirs("%s/data/vnode/vnode2/wal/old" % path, exist_ok=True) # like "mkdir -p" def run(self): # os.system("rm -rf %s/ " % tdDnodes.getDnodesRootDir()) diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 97dca6be18..e0a9e339a9 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -55,7 +55,7 @@ if __name__ == "__main__": restart = True if key in ['-f', '--file']: - fileName = value + fileName = value.replace('\r', "") if key in ['-p', '--path']: deployPath = value diff --git a/tests/pytest/util/dnodes-default.py b/tests/pytest/util/dnodes-default.py index 085e083149..43a92431ba 100644 --- a/tests/pytest/util/dnodes-default.py +++ b/tests/pytest/util/dnodes-default.py @@ -60,7 +60,7 @@ class TDSimClient: self.cfgDict.update({option: value}) def cfg(self, option, value): - cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) @@ -73,17 +73,19 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.logDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) cmd = "rm -rf " + self.cfgDir if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.cfgDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -149,17 +151,20 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.dataDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.logDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.cfgDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -320,7 +325,7 @@ class TDDnode: tdLog.exit(cmd) def cfg(self, option, value): - cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) diff --git a/tests/pytest/util/dnodes-no-random-fail.py b/tests/pytest/util/dnodes-no-random-fail.py index 2627575e61..3bcea493dc 100644 --- a/tests/pytest/util/dnodes-no-random-fail.py +++ b/tests/pytest/util/dnodes-no-random-fail.py @@ -58,7 +58,7 @@ class TDSimClient: self.cfgDict.update({option: value}) def cfg(self, option, value): - cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) @@ -71,17 +71,19 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.logDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) cmd = "rm -rf " + self.cfgDir if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.cfgDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -147,17 +149,20 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.dataDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.logDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.cfgDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -318,7 +323,7 @@ class TDDnode: tdLog.exit(cmd) def cfg(self, option, value): - cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) diff --git a/tests/pytest/util/dnodes-random-fail.py b/tests/pytest/util/dnodes-random-fail.py index 4f4cdcc0d0..7d99980e67 100644 --- a/tests/pytest/util/dnodes-random-fail.py +++ b/tests/pytest/util/dnodes-random-fail.py @@ -58,7 +58,7 @@ class TDSimClient: self.cfgDict.update({option: value}) def cfg(self, option, value): - cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) @@ -71,17 +71,19 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.logDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) cmd = "rm -rf " + self.cfgDir if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.cfgDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -147,17 +149,20 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.dataDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.logDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.cfgDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -318,7 +323,7 @@ class TDDnode: tdLog.exit(cmd) def cfg(self, option, value): - cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 0f4919ba96..589bde7474 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -14,6 +14,7 @@ import sys import os import os.path +import platform import subprocess from time import sleep from util.log import * @@ -61,7 +62,7 @@ class TDSimClient: self.cfgDict.update({option: value}) def cfg(self, option, value): - cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) @@ -74,17 +75,19 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.logDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) cmd = "rm -rf " + self.cfgDir if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.cfgDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -185,17 +188,20 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.dataDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.logDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + # cmd = "mkdir -p " + self.cfgDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -246,7 +252,7 @@ class TDDnode: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if (("taosd.exe") in files) or (("taosd") in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root)-len("/build/bin")] @@ -267,7 +273,11 @@ class TDDnode: tdLog.exit("dnode:%d is not deployed" % (self.index)) if self.valgrind == 0: - cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( + if platform.system()=="Windows": + cmd = "mintty %s -c %s" % ( + binPath, self.cfgDir) + else: + cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( binPath, self.cfgDir) else: valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" @@ -292,7 +302,7 @@ class TDDnode: i += 1 if i>50: break - popen = subprocess.Popen('tail -f ' + logFile, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + popen = subprocess.Popen('tail -f -n +0 ' + logFile, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) pid = popen.pid # print('Popen.pid:' + str(pid)) timeout = time.time() + 60*2 @@ -404,7 +414,7 @@ class TDDnode: tdLog.exit(cmd) def cfg(self, option, value): - cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) diff --git a/tests/pytest/wal/addOldWalTest.py b/tests/pytest/wal/addOldWalTest.py index 2f4dcd5ce8..36056d1bc2 100644 --- a/tests/pytest/wal/addOldWalTest.py +++ b/tests/pytest/wal/addOldWalTest.py @@ -31,7 +31,7 @@ class TDTestCase: def createOldDirAndAddWal(self): oldDir = tdDnodes.getDnodesRootDir() + "dnode1/data/vnode/vnode2/wal/old" - os.system("sudo echo 'test' >> %s/wal" % oldDir) + os.system("sudo echo test >> %s/wal" % oldDir) def run(self): -- GitLab