From 3a0efe72550981de3e48a5a27af71fd01e217a9c Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 25 May 2022 16:30:26 +0800 Subject: [PATCH] fix(os): run case on win --- source/client/src/clientHb.c | 5 + source/libs/catalog/src/ctgCache.c | 5 +- source/os/src/osDir.c | 5 +- source/os/src/osFile.c | 1 - source/os/src/osSemaphore.c | 9 +- source/util/src/tlog.c | 2 +- tests/pytest/fulltest.bat | 2 + tests/pytest/test-all.bat | 25 +++++ tests/pytest/test.py | 156 ++++++++++++++++++----------- tests/pytest/util/cases.py | 10 +- tests/pytest/util/dnodes.py | 109 ++++++++++++++++---- 11 files changed, 246 insertions(+), 83 deletions(-) create mode 100644 tests/pytest/fulltest.bat create mode 100644 tests/pytest/test-all.bat diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 1ae6fa405f..eca9102bf0 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -582,8 +582,13 @@ void hbClearReqInfo(SAppHbMgr *pAppHbMgr) { } } +void hbThreadFuncUnexpectedStopped(void) { + atomic_store_8(&clientHbMgr.threadStop, 2); +} + static void *hbThreadFunc(void *param) { setThreadName("hb"); + atexit(hbThreadFuncUnexpectedStopped); while (1) { int8_t threadStop = atomic_val_compare_exchange_8(&clientHbMgr.threadStop, 1, 2); if (1 == threadStop) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 6335a056b9..425a0bf926 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1457,10 +1457,13 @@ _return: CTG_RET(code); } +void ctgUpdateThreadFuncUnexpectedStopped(void) { + CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); +} void* ctgUpdateThreadFunc(void* param) { setThreadName("catalog"); - + atexit(ctgUpdateThreadFuncUnexpectedStopped); qInfo("catalog update thread started"); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index c4b7c9386e..75797048ca 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -107,13 +107,14 @@ int32_t taosMkDir(const char *dirname) { int32_t taosMulMkDir(const char *dirname) { if (dirname == NULL) return -1; char temp[1024]; + char * pos = temp; + int32_t code = 0; #ifdef WINDOWS taosRealPath(dirname, temp, sizeof(temp)); + if (temp[1] == ':') pos += 3; #else strcpy(temp, dirname); #endif - char * pos = temp; - int32_t code = 0; if (taosDirExist(temp)) return code; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index e08b668163..c75cca79f6 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -69,7 +69,6 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha } strcpy(tmpPath + len, tdengineTmpFileNamePrefix); - strcat(tmpPath, tdengineTmpFileNamePrefix); if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { strcat(tmpPath, fileNamePrefix); strcat(tmpPath, "-%d-%s"); diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index d4cfe4fc39..3b68073c7e 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -50,10 +50,15 @@ int32_t taosGetAppName(char* name, int32_t* len) { if (sub != NULL) { *sub = '\0'; } - strcpy(name, filepath); + char* end = strrchr(filepath, TD_DIRSEP[0]); + if (end == NULL) { + end = filepath; + } + + strcpy(name, end); if (len != NULL) { - *len = (int32_t)strlen(filepath); + *len = (int32_t)strlen(end); } return 0; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index c1fc2c48c0..e8a1ceb18b 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -226,7 +226,7 @@ static void *taosThreadToOpenNewFile(void *param) { tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; - taosSsleep(10); + taosSsleep(20); taosCloseLogByFd(pOldFile); uInfo(" new log file:%d is opened", tsLogObj.flag); diff --git a/tests/pytest/fulltest.bat b/tests/pytest/fulltest.bat new file mode 100644 index 0000000000..5758691c88 --- /dev/null +++ b/tests/pytest/fulltest.bat @@ -0,0 +1,2 @@ + +python .\test.py -f insert\basic.py \ No newline at end of file diff --git a/tests/pytest/test-all.bat b/tests/pytest/test-all.bat new file mode 100644 index 0000000000..437472f7b8 --- /dev/null +++ b/tests/pytest/test-all.bat @@ -0,0 +1,25 @@ +@echo off +SETLOCAL EnableDelayedExpansion +for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a") +set /a a=0 +@REM echo Windows Taosd Test +@REM for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( +@REM echo Processing %%i +@REM set /a a+=1 +@REM call %%i ARG1 -w -m localhost > result_!a!.txt 2>error_!a!.txt +@REM if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) +@REM ) +echo Linux Taosd Test +for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( + echo Processing %%i + set /a a+=1 + call %%i ARG1 -w 1 -m %1 > result_!a!.txt 2>error_!a!.txt + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) +) +exit + +:colorEcho +echo off + "%~2" +findstr /v /a:%1 /R "^$" "%~2" nul +del "%~2" > nul 2>&1i \ No newline at end of file diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 97dca6be18..5c9dae688f 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -35,8 +35,9 @@ if __name__ == "__main__": logSql = True stop = 0 restart = False - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghr', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help']) + windows = 0 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrw', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'windows']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -61,7 +62,10 @@ if __name__ == "__main__": deployPath = value if key in ['-m', '--master']: - masterIp = value + masterIp = value + + if key in ['-w', '--windows']: + windows = 1 if key in ['-l', '--logSql']: if (value.upper() == "TRUE"): @@ -110,67 +114,107 @@ if __name__ == "__main__": time.sleep(2) tdLog.info('stop All dnodes') - - tdDnodes.init(deployPath) - tdDnodes.setTestCluster(testCluster) - tdDnodes.setValgrind(valgrind) - tdDnodes.stopAll() - is_test_framework = 0 - key_word = 'tdCases.addLinux' - try: - if key_word in open(fileName).read(): - is_test_framework = 1 - except: - pass - if is_test_framework: - moduleName = fileName.replace(".py", "").replace("/", ".") - uModule = importlib.import_module(moduleName) - try: - ucase = uModule.TDTestCase() - tdDnodes.deploy(1,ucase.updatecfgDict) - except : - tdDnodes.deploy(1,{}) - else: - tdDnodes.deploy(1,{}) - tdDnodes.start(1) if masterIp == "": host = '127.0.0.1' else: host = masterIp - tdLog.info("Procedures for tdengine deployed in %s" % (host)) - - tdCases.logSql(logSql) - - if testCluster: - tdLog.info("Procedures for testing cluster") - if fileName == "all": - tdCases.runAllCluster() - else: - tdCases.runOneCluster(fileName) - else: + if (windows): + tdCases.logSql(logSql) tdLog.info("Procedures for testing self-deployment") + if masterIp == "" or masterIp == "localhost": + tdDnodes.init(deployPath) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + tdDnodes.stopAll() + is_test_framework = 0 + key_word = 'tdCases.addWindows' + try: + if key_word in open(fileName).read(): + is_test_framework = 1 + except: + pass + if is_test_framework: + moduleName = fileName.replace(".py", "").replace(os.sep, ".") + uModule = importlib.import_module(moduleName) + try: + ucase = uModule.TDTestCase() + tdDnodes.deploy(1,ucase.updatecfgDict) + except : + tdDnodes.deploy(1,{}) + else: + pass + tdDnodes.deploy(1,{}) + tdDnodes.startWin(1) + else: + remote_conn = Connection("root@%s"%host) + with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'): + remote_conn.run("python3 ./test.py") + tdDnodes.init(deployPath) conn = taos.connect( - host, - config=tdDnodes.getSimCfgPath()) - if fileName == "all": - tdCases.runAllLinux(conn) + host="%s" % (host), + config=tdDnodes.sim.getCfgDir()) + print(host) + print(tdDnodes.sim.getCfgDir()) + tdCases.runOneWindows(conn, fileName) + tdCases.logSql(logSql) + else: + tdDnodes.init(deployPath) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + tdDnodes.stopAll() + is_test_framework = 0 + key_word = 'tdCases.addLinux' + try: + if key_word in open(fileName).read(): + is_test_framework = 1 + except: + pass + if is_test_framework: + moduleName = fileName.replace(".py", "").replace("/", ".") + uModule = importlib.import_module(moduleName) + try: + ucase = uModule.TDTestCase() + tdDnodes.deploy(1,ucase.updatecfgDict) + except : + tdDnodes.deploy(1,{}) + else: + tdDnodes.deploy(1,{}) + tdDnodes.start(1) + + tdLog.info("Procedures for tdengine deployed in %s" % (host)) + + tdCases.logSql(logSql) + + if testCluster: + tdLog.info("Procedures for testing cluster") + if fileName == "all": + tdCases.runAllCluster() + else: + tdCases.runOneCluster(fileName) else: - tdCases.runOneLinux(conn, fileName) - if restart: - if fileName == "all": - tdLog.info("not need to query ") - else: - sp = fileName.rsplit(".", 1) - if len(sp) == 2 and sp[1] == "py": - tdDnodes.stopAll() - tdDnodes.start(1) - time.sleep(1) - conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) - tdLog.info("Procedures for tdengine deployed in %s" % (host)) - tdLog.info("query test after taosd restart") - tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") + tdLog.info("Procedures for testing self-deployment") + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + if fileName == "all": + tdCases.runAllLinux(conn) else: - tdLog.info("not need to query") + tdCases.runOneLinux(conn, fileName) + if restart: + if fileName == "all": + tdLog.info("not need to query ") + else: + sp = fileName.rsplit(".", 1) + if len(sp) == 2 and sp[1] == "py": + tdDnodes.stopAll() + tdDnodes.start(1) + time.sleep(1) + conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) + tdLog.info("Procedures for tdengine deployed in %s" % (host)) + tdLog.info("query test after taosd restart") + tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") + else: + tdLog.info("not need to query") conn.close() diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 2fc1ac8515..2bfd8efdcd 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -34,7 +34,7 @@ class TDCases: self.clusterCases = [] def __dynamicLoadModule(self, fileName): - moduleName = fileName.replace(".py", "").replace("/", ".") + moduleName = fileName.replace(".py", "").replace(os.sep, ".") return importlib.import_module(moduleName, package='..') def logSql(self, logSql): @@ -101,8 +101,12 @@ class TDCases: for tmp in self.windowsCases: if tmp.name.find(fileName) != -1: case = testModule.TDTestCase() - case.init(conn) - case.run() + case.init(conn, self._logSql) + try: + case.run() + except Exception as e: + tdLog.notice(repr(e)) + tdLog.exit("%s failed" % (fileName)) case.stop() runNum += 1 continue diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 9190943dfd..cc02703c4f 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -67,17 +67,19 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + # cmd = "mkdir -p " + self.logDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + os.makedirs(self.logDir) cmd = "rm -rf " + self.cfgDir if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + # cmd = "mkdir -p " + self.cfgDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + os.makedirs(self.cfgDir) cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -179,17 +181,20 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + # cmd = "mkdir -p " + self.dataDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + os.makedirs(self.dataDir) - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + # cmd = "mkdir -p " + self.logDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + os.makedirs(self.logDir) - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + # cmd = "mkdir -p " + self.cfgDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + os.makedirs(self.cfgDir) cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -247,6 +252,8 @@ class TDDnode: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def start(self): @@ -309,6 +316,69 @@ class TDDnode: time.sleep(10) # time.sleep(5) + def startWin(self): + binPath = self.getPath("taosd.exe") + + if (binPath == ""): + tdLog.exit("taosd.exe not found!") + else: + tdLog.info("taosd.exe found: %s" % binPath) + + taosadapterBinPath = self.getPath("taosadapter.exe") + if (taosadapterBinPath == ""): + tdLog.info("taosAdapter.exe not found!") + else: + tdLog.info("taosAdapter.exe found in %s" % taosadapterBuildPath) + + if self.deployed == 0: + tdLog.exit("dnode:%d is not deployed" % (self.index)) + + cmd = "mintty -h never %s -c %s" % ( + binPath, self.cfgDir) + + if (taosadapterBinPath != ""): + taosadapterCmd = "mintty -h never -w hide %s --monitor.writeToTD=false " % ( + taosadapterBinPath) + if os.system(taosadapterCmd) != 0: + tdLog.exit(taosadapterCmd) + + if os.system(cmd) != 0: + tdLog.exit(cmd) + + self.running = 1 + tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) + if self.valgrind == 0: + time.sleep(0.1) + key = 'from offline to online' + bkey = bytes(key, encoding="utf8") + logFile = self.logDir + "/taosdlog.0" + i = 0 + while not os.path.exists(logFile): + sleep(0.1) + i += 1 + if i > 50: + break + popen = subprocess.Popen( + 'tail -n +0 -f ' + logFile, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True) + pid = popen.pid + # print('Popen.pid:' + str(pid)) + timeout = time.time() + 60 * 2 + while True: + line = popen.stdout.readline().strip() + if bkey in line: + popen.kill() + break + if time.time() > timeout: + tdLog.exit('wait too long for taosd start') + tdLog.debug("the dnode:%d has been started." % (self.index)) + else: + tdLog.debug( + "wait 10 seconds for the dnode:%d to start." % + (self.index)) + time.sleep(10) def startWithoutSleep(self): binPath = self.getPath() @@ -448,7 +518,8 @@ class TDDnodes: processID = subprocess.check_output( psCmd, shell=True).decode("utf-8") - binPath = self.dnodes[0].getPath() + "/../../../" + binPath = os.path.dirname(os.path.realpath(__file__)) + binPath = binPath + "/../../../debug/" tdLog.debug("binPath %s" % (binPath)) binPath = os.path.realpath(binPath) tdLog.debug("binPath real path %s" % (binPath)) @@ -475,7 +546,7 @@ class TDDnodes: for i in range(len(self.dnodes)): self.dnodes[i].init(self.path) - + print(self.path) self.sim = TDSimClient(self.path) def setTestCluster(self, value): @@ -504,6 +575,10 @@ class TDDnodes: self.check(index) self.dnodes[index - 1].start() + def startWin(self, index): + self.check(index) + self.dnodes[index - 1].startWin() + def startWithoutSleep(self, index): self.check(index) self.dnodes[index - 1].startWithoutSleep() -- GitLab