Merge pull request #12977 from taosdata/fix/ZhiqiangWang/TD-15849-run-case-on-win

fix(os): run case on win

source/client/src/clientHb.c

@@ -582,8 +582,15 @@ void hbClearReqInfo(SAppHbMgr *pAppHbMgr) {
   }
 }
 
+void hbThreadFuncUnexpectedStopped(void) {
+  atomic_store_8(&clientHbMgr.threadStop, 2);
+}
+
 static void *hbThreadFunc(void *param) {
   setThreadName("hb");
+#ifdef WINDOWS
+  atexit(hbThreadFuncUnexpectedStopped);
+#endif
   while (1) {
     int8_t threadStop = atomic_val_compare_exchange_8(&clientHbMgr.threadStop, 1, 2);
     if (1 == threadStop) {

source/libs/catalog/src/ctgCache.c

@@ -1457,10 +1457,15 @@ _return:
   CTG_RET(code);
 }
 
+void ctgUpdateThreadFuncUnexpectedStopped(void) {
+  if (CTG_IS_LOCKED(&gCtgMgmt.lock) == TD_RWLATCH_WRITE_FLAG_COPY) CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock);
+}
 
 void* ctgUpdateThreadFunc(void* param) {
   setThreadName("catalog");
-
+#ifdef WINDOWS
+  atexit(ctgUpdateThreadFuncUnexpectedStopped);
+#endif
   qInfo("catalog update thread started");
 
   CTG_LOCK(CTG_READ, &gCtgMgmt.lock);
@@ -1494,7 +1499,7 @@ void* ctgUpdateThreadFunc(void* param) {
     ctgdShowClusterCache(pCtg);
   }
 
-  CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock);
+  if (CTG_IS_LOCKED(&gCtgMgmt.lock)) CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock);
 
   qInfo("catalog update thread stopped");
   

source/os/src/osDir.c

@@ -107,13 +107,14 @@ int32_t taosMkDir(const char *dirname) {
 int32_t taosMulMkDir(const char *dirname) {
   if (dirname == NULL) return -1;
   char temp[1024];
+  char *  pos = temp;
+  int32_t code = 0;
 #ifdef WINDOWS
   taosRealPath(dirname, temp, sizeof(temp));
+  if (temp[1] == ':') pos += 3;
 #else
   strcpy(temp, dirname);
 #endif
-  char *  pos = temp;
-  int32_t code = 0;
 
   if (taosDirExist(temp)) return code;
 

source/os/src/osFile.c

@@ -69,7 +69,6 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha
   }
 
   strcpy(tmpPath + len, tdengineTmpFileNamePrefix);
-  strcat(tmpPath, tdengineTmpFileNamePrefix);
   if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
     strcat(tmpPath, fileNamePrefix);
     strcat(tmpPath, "-%d-%s");

source/os/src/osSemaphore.c

@@ -50,10 +50,15 @@ int32_t taosGetAppName(char* name, int32_t* len) {
   if (sub != NULL) {
     *sub = '\0';
   }
-  strcpy(name, filepath);
+  char* end = strrchr(filepath, TD_DIRSEP[0]);
+  if (end == NULL) {
+    end = filepath;
+  }
+
+  strcpy(name, end);
 
   if (len != NULL) {
-    *len = (int32_t)strlen(filepath);
+    *len = (int32_t)strlen(end);
   }
 
   return 0;

source/util/src/tlog.c

@@ -226,7 +226,7 @@ static void *taosThreadToOpenNewFile(void *param) {
   tsLogObj.logHandle->pFile = pFile;
   tsLogObj.lines = 0;
   tsLogObj.openInProgress = 0;
-  taosSsleep(10);
+  taosSsleep(20);
   taosCloseLogByFd(pOldFile);
 
   uInfo("   new log file:%d is opened", tsLogObj.flag);

tests/pytest/fulltest.bat

+
+python .\test.py -f insert\basic.py 
\ No newline at end of file

tests/pytest/test-all.bat

+@echo off
+SETLOCAL EnableDelayedExpansion
+for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do     rem"') do (  set "DEL=%%a")
+set /a a=0
+@REM echo Windows Taosd Test
+@REM for /F "usebackq tokens=*" %%i in (fulltest.bat) do (
+@REM     echo Processing %%i
+@REM     set /a a+=1
+@REM     call %%i ARG1 -w -m localhost > result_!a!.txt 2>error_!a!.txt
+@REM     if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && exit 8 ) else ( call :colorEcho 0a "Success" &echo. )
+@REM )
+echo Linux Taosd Test
+for /F "usebackq tokens=*" %%i in (fulltest.bat) do (
+    echo Processing %%i
+    set /a a+=1
+    call %%i ARG1 -w 1 -m %1 > result_!a!.txt 2>error_!a!.txt
+    if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) 
+)
+exit
+
+:colorEcho
+echo off
+<nul set /p ".=%DEL%" > "%~2"
+findstr /v /a:%1 /R "^$" "%~2" nul
+del "%~2" > nul 2>&1i
\ No newline at end of file

tests/pytest/test.py

@@ -35,8 +35,9 @@ if __name__ == "__main__":
     logSql = True
     stop = 0
     restart = False
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghr', [
-        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help'])
+    windows = 0
+    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrw', [
+        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'windows'])
     for key, value in opts:
         if key in ['-h', '--help']:
             tdLog.printNoPrefix(
@@ -61,7 +62,10 @@ if __name__ == "__main__":
             deployPath = value
 
         if key in ['-m', '--master']:
-            masterIp = value
+            masterIp = value 
+
+        if key in ['-w', '--windows']:
+            windows = 1
 
         if key in ['-l', '--logSql']:
             if (value.upper() == "TRUE"):
@@ -110,67 +114,105 @@ if __name__ == "__main__":
             time.sleep(2)
 
         tdLog.info('stop All dnodes')
-    
-    tdDnodes.init(deployPath)
-    tdDnodes.setTestCluster(testCluster)
-    tdDnodes.setValgrind(valgrind)
-    tdDnodes.stopAll()
-    is_test_framework = 0
-    key_word = 'tdCases.addLinux'
-    try:
-        if key_word in open(fileName).read():
-            is_test_framework = 1
-    except:
-        pass
-    if is_test_framework:
-        moduleName = fileName.replace(".py", "").replace("/", ".")
-        uModule = importlib.import_module(moduleName)
-        try:
-            ucase = uModule.TDTestCase()
-            tdDnodes.deploy(1,ucase.updatecfgDict)
-        except :
-            tdDnodes.deploy(1,{})
-    else:
-        tdDnodes.deploy(1,{})
-    tdDnodes.start(1)
 
     if masterIp == "":
         host = '127.0.0.1'
     else:
         host = masterIp
 
-    tdLog.info("Procedures for tdengine deployed in %s" % (host))
-
-    tdCases.logSql(logSql)
-
-    if testCluster:
-        tdLog.info("Procedures for testing cluster")
-        if fileName == "all":
-            tdCases.runAllCluster()
-        else:
-            tdCases.runOneCluster(fileName)
-    else:
+    if (windows):
+        tdCases.logSql(logSql)
         tdLog.info("Procedures for testing self-deployment")
+        if masterIp == "" or masterIp == "localhost":
+            tdDnodes.init(deployPath)
+            tdDnodes.setTestCluster(testCluster)
+            tdDnodes.setValgrind(valgrind)
+            tdDnodes.stopAll()
+            is_test_framework = 0
+            key_word = 'tdCases.addWindows'
+            try:
+                if key_word in open(fileName).read():
+                    is_test_framework = 1
+            except:
+                pass
+            if is_test_framework:
+                moduleName = fileName.replace(".py", "").replace(os.sep, ".")
+                uModule = importlib.import_module(moduleName)
+                try:
+                    ucase = uModule.TDTestCase()
+                    tdDnodes.deploy(1,ucase.updatecfgDict)
+                except :
+                    tdDnodes.deploy(1,{})
+            else:
+                pass
+                tdDnodes.deploy(1,{})
+            tdDnodes.startWin(1)
+        else:
+            remote_conn = Connection("root@%s"%host)
+            with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
+                remote_conn.run("python3 ./test.py")
+        tdDnodes.init(deployPath)
         conn = taos.connect(
-            host,
-            config=tdDnodes.getSimCfgPath())
-        if fileName == "all":
-            tdCases.runAllLinux(conn)
+            host="%s" % (host),
+            config=tdDnodes.sim.getCfgDir())
+        tdCases.runOneWindows(conn, fileName)
+        tdCases.logSql(logSql)
+    else:
+        tdDnodes.init(deployPath)
+        tdDnodes.setTestCluster(testCluster)
+        tdDnodes.setValgrind(valgrind)
+        tdDnodes.stopAll()
+        is_test_framework = 0
+        key_word = 'tdCases.addLinux'
+        try:
+            if key_word in open(fileName).read():
+                is_test_framework = 1
+        except:
+            pass
+        if is_test_framework:
+            moduleName = fileName.replace(".py", "").replace("/", ".")
+            uModule = importlib.import_module(moduleName)
+            try:
+                ucase = uModule.TDTestCase()
+                tdDnodes.deploy(1,ucase.updatecfgDict)
+            except :
+                tdDnodes.deploy(1,{})
+        else:
+            tdDnodes.deploy(1,{})
+        tdDnodes.start(1)
+
+        tdLog.info("Procedures for tdengine deployed in %s" % (host))
+
+        tdCases.logSql(logSql)
+
+        if testCluster:
+            tdLog.info("Procedures for testing cluster")
+            if fileName == "all":
+                tdCases.runAllCluster()
+            else:
+                tdCases.runOneCluster(fileName)
         else:
-            tdCases.runOneLinux(conn, fileName)
-    if restart:
-        if fileName == "all":
-            tdLog.info("not need to query ")
-        else:    
-            sp = fileName.rsplit(".", 1)
-            if len(sp) == 2 and sp[1] == "py":
-                tdDnodes.stopAll()
-                tdDnodes.start(1)
-                time.sleep(1)            
-                conn = taos.connect( host, config=tdDnodes.getSimCfgPath())
-                tdLog.info("Procedures for tdengine deployed in %s" % (host))
-                tdLog.info("query test after taosd restart")
-                tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")
+            tdLog.info("Procedures for testing self-deployment")
+            conn = taos.connect(
+                host,
+                config=tdDnodes.getSimCfgPath())
+            if fileName == "all":
+                tdCases.runAllLinux(conn)
             else:
-                tdLog.info("not need to query")
+                tdCases.runOneLinux(conn, fileName)
+        if restart:
+            if fileName == "all":
+                tdLog.info("not need to query ")
+            else:    
+                sp = fileName.rsplit(".", 1)
+                if len(sp) == 2 and sp[1] == "py":
+                    tdDnodes.stopAll()
+                    tdDnodes.start(1)
+                    time.sleep(1)            
+                    conn = taos.connect( host, config=tdDnodes.getSimCfgPath())
+                    tdLog.info("Procedures for tdengine deployed in %s" % (host))
+                    tdLog.info("query test after taosd restart")
+                    tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")
+                else:
+                    tdLog.info("not need to query")
     conn.close()

tests/pytest/util/cases.py

@@ -34,7 +34,7 @@ class TDCases:
         self.clusterCases = []
 
     def __dynamicLoadModule(self, fileName):
-        moduleName = fileName.replace(".py", "").replace("/", ".")
+        moduleName = fileName.replace(".py", "").replace(os.sep, ".")
         return importlib.import_module(moduleName, package='..')
 
     def logSql(self, logSql):
@@ -101,8 +101,12 @@ class TDCases:
         for tmp in self.windowsCases:
             if tmp.name.find(fileName) != -1:
                 case = testModule.TDTestCase()
-                case.init(conn)
-                case.run()
+                case.init(conn, self._logSql)
+                try:
+                    case.run()
+                except Exception as e:
+                    tdLog.notice(repr(e))
+                    tdLog.exit("%s failed" % (fileName))
                 case.stop()
                 runNum += 1
                 continue

tests/pytest/util/dnodes.py

@@ -67,17 +67,19 @@ class TDSimClient:
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
 
-        cmd = "mkdir -p " + self.logDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        # cmd = "mkdir -p " + self.logDir
+        # if os.system(cmd) != 0:
+        #     tdLog.exit(cmd)
+        os.makedirs(self.logDir)
 
         cmd = "rm -rf " + self.cfgDir
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
 
-        cmd = "mkdir -p " + self.cfgDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        # cmd = "mkdir -p " + self.cfgDir
+        # if os.system(cmd) != 0:
+        #     tdLog.exit(cmd)
+        os.makedirs(self.cfgDir)
 
         cmd = "touch " + self.cfgPath
         if os.system(cmd) != 0:
@@ -179,17 +181,20 @@ class TDDnode:
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
 
-        cmd = "mkdir -p " + self.dataDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        # cmd = "mkdir -p " + self.dataDir
+        # if os.system(cmd) != 0:
+        #     tdLog.exit(cmd)
+        os.makedirs(self.dataDir)
 
-        cmd = "mkdir -p " + self.logDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        # cmd = "mkdir -p " + self.logDir
+        # if os.system(cmd) != 0:
+        #     tdLog.exit(cmd)
+        os.makedirs(self.logDir)
 
-        cmd = "mkdir -p " + self.cfgDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        # cmd = "mkdir -p " + self.cfgDir
+        # if os.system(cmd) != 0:
+        #     tdLog.exit(cmd)
+        os.makedirs(self.cfgDir)
 
         cmd = "touch " + self.cfgPath
         if os.system(cmd) != 0:
@@ -247,6 +252,8 @@ class TDDnode:
                 if ("packaging" not in rootRealPath):
                     paths.append(os.path.join(root, tool))
                     break
+        if (len(paths) == 0):
+                return ""
         return paths[0]
 
     def start(self):
@@ -309,6 +316,69 @@ class TDDnode:
             time.sleep(10)
 
         # time.sleep(5)
+    def startWin(self):
+        binPath = self.getPath("taosd.exe")
+
+        if (binPath == ""):
+            tdLog.exit("taosd.exe not found!")
+        else:
+            tdLog.info("taosd.exe found: %s" % binPath)
+
+        taosadapterBinPath = self.getPath("taosadapter.exe")
+        if (taosadapterBinPath == ""):
+            tdLog.info("taosAdapter.exe not found!")
+        else:
+            tdLog.info("taosAdapter.exe found in %s" % taosadapterBuildPath)
+
+        if self.deployed == 0:
+            tdLog.exit("dnode:%d is not deployed" % (self.index))
+
+        cmd = "mintty -h never %s -c %s" % (
+            binPath, self.cfgDir)
+        
+        if (taosadapterBinPath != ""):
+            taosadapterCmd = "mintty -h never -w hide %s --monitor.writeToTD=false " % (
+                taosadapterBinPath)
+            if os.system(taosadapterCmd) != 0:
+                tdLog.exit(taosadapterCmd)
+
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        self.running = 1
+        tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
+        if self.valgrind == 0:
+            time.sleep(0.1)
+            key = 'from offline to online'
+            bkey = bytes(key, encoding="utf8")
+            logFile = self.logDir + "/taosdlog.0"
+            i = 0
+            while not os.path.exists(logFile):
+                sleep(0.1)
+                i += 1
+                if i > 50:
+                    break
+            popen = subprocess.Popen(
+                'tail -n +0 -f ' + logFile,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                shell=True)
+            pid = popen.pid
+            # print('Popen.pid:' + str(pid))
+            timeout = time.time() + 60 * 2
+            while True:
+                line = popen.stdout.readline().strip()
+                if bkey in line:
+                    popen.kill()
+                    break
+                if time.time() > timeout:
+                    tdLog.exit('wait too long for taosd start')
+            tdLog.debug("the dnode:%d has been started." % (self.index))
+        else:
+            tdLog.debug(
+                "wait 10 seconds for the dnode:%d to start." %
+                (self.index))
+            time.sleep(10)
 
     def startWithoutSleep(self):
         binPath = self.getPath()
@@ -475,7 +545,6 @@ class TDDnodes:
 
         for i in range(len(self.dnodes)):
             self.dnodes[i].init(self.path)
-
         self.sim = TDSimClient(self.path)
 
     def setTestCluster(self, value):
@@ -504,6 +573,10 @@ class TDDnodes:
         self.check(index)
         self.dnodes[index - 1].start()
 
+    def startWin(self, index):
+        self.check(index)
+        self.dnodes[index - 1].startWin()
+
     def startWithoutSleep(self, index):
         self.check(index)
         self.dnodes[index - 1].startWithoutSleep()