Merge pull request #9552 from taosdata/fix/ZhiqiangWang/TD-345-win-run-python-test-case

[TD-345]<fix>: win run python test case

src/os/src/detail/osFile.c

@@ -370,8 +370,11 @@ int32_t taosFsync(FileFd fd) {
   }
 
   HANDLE h = (HANDLE)_get_osfhandle(fd);
-
-  return FlushFileBuffers(h);
+  
+  //If the function succeeds, the return value is nonzero.
+  //If the function fails, the return value is zero. To get extended error information, call GetLastError.
+  //The function fails if hFile is a handle to the console output. That is because the console output is not buffered. The function returns FALSE, and GetLastError returns ERROR_INVALID_HANDLE.
+  return FlushFileBuffers(h)-1;
 }
 
 int32_t taosRename(char *oldName, char *newName) {

src/util/src/tcompare.c

@@ -524,11 +524,11 @@ int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
 
   assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE);
 
-  wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t));
-  wchar_t *str = calloc(size + 1, sizeof(wchar_t));
+  char *pattern = calloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1);
+  char *str = calloc(varDataLen(pLeft) + TSDB_NCHAR_SIZE, 1);
 
   memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
-  memcpy(str, varDataVal(pLeft), size * sizeof(wchar_t));
+  memcpy(str, varDataVal(pLeft), varDataLen(pLeft));
 
   int32_t ret = WCSPatternMatch((uint32_t *)pattern, (uint32_t *)str, size, &pInfo);
 

tests/pytest/dockerCluster/basic.py

@@ -45,8 +45,7 @@ class BuildDockerCluser:
             "qdebugFlag":"135",
             "maxSQLLength":"1048576"
         }
-        cmd = "mkdir -p %s" % self.dockerDir
-        self.execCmd(cmd)
+        os.makedirs(self.dockerDir, exist_ok=True) # like "mkdir -p"
 
         cmd = "cp *.yml %s" % self.dockerDir
         self.execCmd(cmd)
@@ -100,8 +99,7 @@ class BuildDockerCluser:
             self.removeFile(self.dockerDir, i, self.dirs[2])
 
     def createDir(self, rootDir, index, dir):
-        cmd = "mkdir -p %s/node%d/%s" % (rootDir, index, dir)
-        self.execCmd(cmd)
+        os.makedirs("%s/node%d/%s" % (rootDir, index, dir), exist_ok=True) # like "mkdir -p"
 
     def createDirs(self):
         for i in range(1, self.numOfNodes + 1):

tests/pytest/query/queryWithTaosdKilled.py

@@ -34,7 +34,8 @@ class TDTestCase:
         path = tdDnodes.dnodes[1].getDnodeRootDir(1)
         print(path)
         tdLog.info("sudo mkdir -p %s/data/vnode/vnode2/wal/old" % path)
-        os.system("sudo mkdir -p %s/data/vnode/vnode2/wal/old" % path)       
+        os.makedirs("%s/data/vnode/vnode2/wal/old" % path, exist_ok=True) # like "mkdir -p"
+   
 
     def run(self):
         # os.system("rm -rf %s/ " % tdDnodes.getDnodesRootDir())

tests/pytest/test.py

@@ -58,7 +58,7 @@ if __name__ == "__main__":
             restart = True
 
         if key in ['-f', '--file']:
-            fileName = value
+            fileName = os.path.normpath(value)
 
         if key in ['-p', '--path']:
             deployPath = value
@@ -122,15 +122,40 @@ if __name__ == "__main__":
     else:
         host = masterIp
 
-    tdLog.info("Procedures for tdengine deployed in %s" % (host))
+    tdLog.info("Procedures for tdengine deployed in %s" % (host)) 
     if windows:
         tdCases.logSql(logSql)
         tdLog.info("Procedures for testing self-deployment")
         td_clinet = TDSimClient("C:\\TDengine")
         td_clinet.deploy()
-        remote_conn = Connection("root@%s"%host)
-        with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
-            remote_conn.run("python3 ./test.py")
+        if masterIp == "" or masterIp == "localhost":
+            tdDnodes.init(deployPath)
+            tdDnodes.setTestCluster(testCluster)
+            tdDnodes.setValgrind(valgrind)
+            tdDnodes.stopAll()
+            is_test_framework = 0
+            key_word = 'tdCases.addWindows'
+            try:
+                if key_word in open(fileName).read():
+                    is_test_framework = 1
+            except:
+                pass
+            if is_test_framework:
+                moduleName = fileName.replace(".py", "").replace(os.sep, ".")
+                uModule = importlib.import_module(moduleName)
+                try:
+                    ucase = uModule.TDTestCase()
+                    tdDnodes.deploy(1,ucase.updatecfgDict)
+                except :
+                    tdDnodes.deploy(1,{})
+            else:
+                pass
+                tdDnodes.deploy(1,{})
+            tdDnodes.startWin(1)
+        else:
+            remote_conn = Connection("root@%s"%host)
+            with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
+                remote_conn.run("python3 ./test.py")
         conn = taos.connect(
             host="%s"%(host),
             config=td_clinet.cfgDir)
@@ -148,7 +173,7 @@ if __name__ == "__main__":
         except:
             pass
         if is_test_framework:
-            moduleName = fileName.replace(".py", "").replace("/", ".")
+            moduleName = fileName.replace(".py", "").replace(os.sep, ".")
             uModule = importlib.import_module(moduleName)
             try:
                 ucase = uModule.TDTestCase()

tests/pytest/util/cases.py

@@ -53,7 +53,7 @@ class TDCases:
         # TODO: load all Linux cases here
         runNum = 0
         for tmp in self.linuxCases:
-            if tmp.name.find(fileName) != -1:
+            if tmp.name.find(os.path.normcase(fileName)) != -1:
                 case = testModule.TDTestCase()
                 case.init(conn)
                 case.run()
@@ -68,7 +68,7 @@ class TDCases:
 
         runNum = 0
         for tmp in self.linuxCases:
-            if tmp.name.find(fileName) != -1:
+            if tmp.name.find(os.path.normcase(fileName)) != -1:
                 case = testModule.TDTestCase()
                 case.init(conn, self._logSql)
                 try:
@@ -84,7 +84,7 @@ class TDCases:
         # TODO: load all Windows cases here
         runNum = 0
         for tmp in self.windowsCases:
-            if tmp.name.find(fileName) != -1:
+            if tmp.name.find(os.path.normcase(fileName)) != -1:
                 case = testModule.TDTestCase()
                 case.init(conn)
                 case.run()
@@ -118,7 +118,7 @@ class TDCases:
 
         runNum = 0
         for tmp in self.clusterCases:
-            if tmp.name.find(fileName) != -1:
+            if tmp.name.find(os.path.normcase(fileName)) != -1:
                 tdLog.notice("run cases like %s" % (fileName))
                 case = testModule.TDTestCase()
                 case.init()
@@ -134,7 +134,7 @@ class TDCases:
 
         runNum = 0
         for tmp in self.clusterCases:
-            if tmp.name.find(fileName) != -1:
+            if tmp.name.find(os.path.normcase(fileName)) != -1:
                 tdLog.notice("run cases like %s" % (fileName))
                 case = testModule.TDTestCase()
                 case.init()

tests/pytest/util/dnodes-default.py

@@ -73,17 +73,13 @@ class TDSimClient:
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
     
-        cmd = "mkdir -p " + self.logDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p"
 
         cmd = "rm -rf " + self.cfgDir
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
 
-        cmd = "mkdir -p " + self.cfgDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p"
 
         cmd = "touch " + self.cfgPath
         if os.system(cmd) != 0:
@@ -149,17 +145,11 @@ class TDDnode:
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
 
-        cmd = "mkdir -p " + self.dataDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p"
 
-        cmd = "mkdir -p " + self.logDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p"
 
-        cmd = "mkdir -p " + self.cfgDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p"
 
         cmd = "touch " + self.cfgPath
         if os.system(cmd) != 0:

tests/pytest/util/dnodes-no-random-fail.py

@@ -71,17 +71,13 @@ class TDSimClient:
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
     
-        cmd = "mkdir -p " + self.logDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p"
 
         cmd = "rm -rf " + self.cfgDir
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
 
-        cmd = "mkdir -p " + self.cfgDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p"
 
         cmd = "touch " + self.cfgPath
         if os.system(cmd) != 0:
@@ -147,17 +143,11 @@ class TDDnode:
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
 
-        cmd = "mkdir -p " + self.dataDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p"
 
-        cmd = "mkdir -p " + self.logDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p"
 
-        cmd = "mkdir -p " + self.cfgDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p"
 
         cmd = "touch " + self.cfgPath
         if os.system(cmd) != 0:

tests/pytest/util/dnodes-random-fail.py

@@ -71,17 +71,13 @@ class TDSimClient:
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
     
-        cmd = "mkdir -p " + self.logDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p"
 
         cmd = "rm -rf " + self.cfgDir
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
 
-        cmd = "mkdir -p " + self.cfgDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p"
 
         cmd = "touch " + self.cfgPath
         if os.system(cmd) != 0:
@@ -147,17 +143,11 @@ class TDDnode:
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
 
-        cmd = "mkdir -p " + self.dataDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p"
 
-        cmd = "mkdir -p " + self.logDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p"
 
-        cmd = "mkdir -p " + self.cfgDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p"
 
         cmd = "touch " + self.cfgPath
         if os.system(cmd) != 0:

tests/pytest/util/dnodes.py

@@ -200,17 +200,11 @@ class TDDnode:
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
 
-        cmd = "mkdir -p " + self.dataDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p"
 
-        cmd = "mkdir -p " + self.logDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p"
 
-        cmd = "mkdir -p " + self.cfgDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p"
 
         cmd = "touch " + self.cfgPath
         if os.system(cmd) != 0:
@@ -332,6 +326,59 @@ class TDDnode:
 
         
         # time.sleep(5)
+    def startWin(self):
+        buildPath = self.getBuildPath("taosd.exe")
+
+        if (buildPath == ""):
+            tdLog.exit("taosd.exe not found!")
+        else:
+            tdLog.info("taosd.exe found in %s" % buildPath)
+
+        binPath = buildPath + "/build/bin/taosd.exe"
+        taosadapterBinPath = buildPath + "/build/bin/taosadapter.exe"
+
+        if self.deployed == 0:
+            tdLog.exit("dnode:%d is not deployed" % (self.index))
+
+        cmd = "mintty -h never -w hide %s -c %s" % (
+                binPath, self.cfgDir)
+
+        taosadapterCmd = "mintty -h never -w hide %s " % (
+                taosadapterBinPath)
+        if os.system(taosadapterCmd) != 0:
+            tdLog.exit(taosadapterCmd)
+
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        self.running = 1
+        tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
+        if self.valgrind == 0:
+            time.sleep(0.1)
+            key = 'from offline to online'
+            bkey = bytes(key,encoding="utf8")
+            logFile = self.logDir + "/taosdlog.0"
+            i = 0
+            while not os.path.exists(logFile):
+                sleep(0.1)
+                i += 1
+                if i>50:
+                    break
+            popen = subprocess.Popen('tail -n +0 -f ' + logFile, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+            pid = popen.pid
+            # print('Popen.pid:' + str(pid))
+            timeout = time.time() + 60*2
+            while True:
+                line = popen.stdout.readline().strip()
+                if bkey in line:
+                    popen.kill()
+                    break
+                if time.time() > timeout:
+                    tdLog.exit('wait too long for taosd start')
+            tdLog.debug("the dnode:%d has been started." % (self.index))
+        else:
+            tdLog.debug("wait 10 seconds for the dnode:%d to start." % (self.index))
+            time.sleep(10)
     
     def startWithoutSleep(self):
         buildPath = self.getBuildPath()
@@ -547,6 +594,10 @@ class TDDnodes:
     def start(self, index):
         self.check(index)
         self.dnodes[index - 1].start()
+
+    def startWin(self, index):
+        self.check(index)
+        self.dnodes[index - 1].startWin()
     
     def startWithoutSleep(self, index):
         self.check(index)

tests/pytest/wal/addOldWalTest.py

@@ -27,7 +27,7 @@ class TDTestCase:
     
     def createOldDir(self):
         oldDir = tdDnodes.getDnodesRootDir() + "dnode1/data/vnode/vnode2/wal/old"
-        os.system("sudo mkdir -p %s" % oldDir)
+        os.makedirs(oldDir, exist_ok=True) # like "mkdir -p"
     
     def createOldDirAndAddWal(self):
         oldDir = tdDnodes.getDnodesRootDir() + "dnode1/data/vnode/vnode2/wal/old"