From 9e87c3c5270ddc9e6a969112c777e77218612a9f Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sun, 2 Jan 2022 18:01:04 +0800 Subject: [PATCH] [TD-345]: win run python test case --- src/os/src/detail/osFile.c | 7 ++- src/util/src/tcompare.c | 6 +- tests/pytest/dockerCluster/basic.py | 6 +- tests/pytest/query/queryWithTaosdKilled.py | 3 +- tests/pytest/test.py | 35 +++++++++-- tests/pytest/util/cases.py | 10 ++-- tests/pytest/util/dnodes-default.py | 20 ++----- tests/pytest/util/dnodes-no-random-fail.py | 20 ++----- tests/pytest/util/dnodes-random-fail.py | 20 ++----- tests/pytest/util/dnodes.py | 69 +++++++++++++++++++--- tests/pytest/wal/addOldWalTest.py | 2 +- 11 files changed, 123 insertions(+), 75 deletions(-) diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 039d688526..f18fb6a6a8 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -370,8 +370,11 @@ int32_t taosFsync(FileFd fd) { } HANDLE h = (HANDLE)_get_osfhandle(fd); - - return FlushFileBuffers(h); + + //If the function succeeds, the return value is nonzero. + //If the function fails, the return value is zero. To get extended error information, call GetLastError. + //The function fails if hFile is a handle to the console output. That is because the console output is not buffered. The function returns FALSE, and GetLastError returns ERROR_INVALID_HANDLE. + return FlushFileBuffers(h)-1; } int32_t taosRename(char *oldName, char *newName) { diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 02b0e83061..445e8ed49e 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -524,11 +524,11 @@ int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); - wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); - wchar_t *str = calloc(size + 1, sizeof(wchar_t)); + char *pattern = calloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1); + char *str = calloc(varDataLen(pLeft) + TSDB_NCHAR_SIZE, 1); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - memcpy(str, varDataVal(pLeft), size * sizeof(wchar_t)); + memcpy(str, varDataVal(pLeft), varDataLen(pLeft)); int32_t ret = WCSPatternMatch((uint32_t *)pattern, (uint32_t *)str, size, &pInfo); diff --git a/tests/pytest/dockerCluster/basic.py b/tests/pytest/dockerCluster/basic.py index 871d69790d..182ff069e8 100644 --- a/tests/pytest/dockerCluster/basic.py +++ b/tests/pytest/dockerCluster/basic.py @@ -45,8 +45,7 @@ class BuildDockerCluser: "qdebugFlag":"135", "maxSQLLength":"1048576" } - cmd = "mkdir -p %s" % self.dockerDir - self.execCmd(cmd) + os.makedirs(self.dockerDir, exist_ok=True) # like "mkdir -p" cmd = "cp *.yml %s" % self.dockerDir self.execCmd(cmd) @@ -100,8 +99,7 @@ class BuildDockerCluser: self.removeFile(self.dockerDir, i, self.dirs[2]) def createDir(self, rootDir, index, dir): - cmd = "mkdir -p %s/node%d/%s" % (rootDir, index, dir) - self.execCmd(cmd) + os.makedirs("%s/node%d/%s" % (rootDir, index, dir), exist_ok=True) # like "mkdir -p" def createDirs(self): for i in range(1, self.numOfNodes + 1): diff --git a/tests/pytest/query/queryWithTaosdKilled.py b/tests/pytest/query/queryWithTaosdKilled.py index 28f9b87636..a9b442ff2f 100644 --- a/tests/pytest/query/queryWithTaosdKilled.py +++ b/tests/pytest/query/queryWithTaosdKilled.py @@ -34,7 +34,8 @@ class TDTestCase: path = tdDnodes.dnodes[1].getDnodeRootDir(1) print(path) tdLog.info("sudo mkdir -p %s/data/vnode/vnode2/wal/old" % path) - os.system("sudo mkdir -p %s/data/vnode/vnode2/wal/old" % path) + os.makedirs("%s/data/vnode/vnode2/wal/old" % path, exist_ok=True) # like "mkdir -p" + def run(self): # os.system("rm -rf %s/ " % tdDnodes.getDnodesRootDir()) diff --git a/tests/pytest/test.py b/tests/pytest/test.py index a96ac21496..62555f78da 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -58,7 +58,7 @@ if __name__ == "__main__": restart = True if key in ['-f', '--file']: - fileName = value + fileName = os.path.normpath(value) if key in ['-p', '--path']: deployPath = value @@ -128,9 +128,34 @@ if __name__ == "__main__": tdLog.info("Procedures for testing self-deployment") td_clinet = TDSimClient("C:\\TDengine") td_clinet.deploy() - remote_conn = Connection("root@%s"%host) - with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'): - remote_conn.run("python3 ./test.py") + if masterIp == "" or masterIp == "localhost": + tdDnodes.init(deployPath) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + tdDnodes.stopAll() + is_test_framework = 0 + key_word = 'tdCases.addWindows' + try: + if key_word in open(fileName).read(): + is_test_framework = 1 + except: + pass + if is_test_framework: + moduleName = fileName.replace(".py", "").replace(os.sep, ".") + uModule = importlib.import_module(moduleName) + try: + ucase = uModule.TDTestCase() + tdDnodes.deploy(1,ucase.updatecfgDict) + except : + tdDnodes.deploy(1,{}) + else: + pass + tdDnodes.deploy(1,{}) + tdDnodes.startWin(1) + else: + remote_conn = Connection("root@%s"%host) + with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'): + remote_conn.run("python3 ./test.py") conn = taos.connect( host="%s"%(host), config=td_clinet.cfgDir) @@ -148,7 +173,7 @@ if __name__ == "__main__": except: pass if is_test_framework: - moduleName = fileName.replace(".py", "").replace("/", ".") + moduleName = fileName.replace(".py", "").replace(os.sep, ".") uModule = importlib.import_module(moduleName) try: ucase = uModule.TDTestCase() diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index fd3926a6f1..36a7a3cd3f 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -53,7 +53,7 @@ class TDCases: # TODO: load all Linux cases here runNum = 0 for tmp in self.linuxCases: - if tmp.name.find(fileName) != -1: + if tmp.name.find(os.path.normcase(fileName)) != -1: case = testModule.TDTestCase() case.init(conn) case.run() @@ -68,7 +68,7 @@ class TDCases: runNum = 0 for tmp in self.linuxCases: - if tmp.name.find(fileName) != -1: + if tmp.name.find(os.path.normcase(fileName)) != -1: case = testModule.TDTestCase() case.init(conn, self._logSql) try: @@ -84,7 +84,7 @@ class TDCases: # TODO: load all Windows cases here runNum = 0 for tmp in self.windowsCases: - if tmp.name.find(fileName) != -1: + if tmp.name.find(os.path.normcase(fileName)) != -1: case = testModule.TDTestCase() case.init(conn) case.run() @@ -118,7 +118,7 @@ class TDCases: runNum = 0 for tmp in self.clusterCases: - if tmp.name.find(fileName) != -1: + if tmp.name.find(os.path.normcase(fileName)) != -1: tdLog.notice("run cases like %s" % (fileName)) case = testModule.TDTestCase() case.init() @@ -134,7 +134,7 @@ class TDCases: runNum = 0 for tmp in self.clusterCases: - if tmp.name.find(fileName) != -1: + if tmp.name.find(os.path.normcase(fileName)) != -1: tdLog.notice("run cases like %s" % (fileName)) case = testModule.TDTestCase() case.init() diff --git a/tests/pytest/util/dnodes-default.py b/tests/pytest/util/dnodes-default.py index 8da36f3074..7d8fc3f630 100644 --- a/tests/pytest/util/dnodes-default.py +++ b/tests/pytest/util/dnodes-default.py @@ -73,17 +73,13 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" cmd = "rm -rf " + self.cfgDir if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -149,17 +145,11 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" cmd = "touch " + self.cfgPath if os.system(cmd) != 0: diff --git a/tests/pytest/util/dnodes-no-random-fail.py b/tests/pytest/util/dnodes-no-random-fail.py index a973f8da52..86ef9e178e 100644 --- a/tests/pytest/util/dnodes-no-random-fail.py +++ b/tests/pytest/util/dnodes-no-random-fail.py @@ -71,17 +71,13 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" cmd = "rm -rf " + self.cfgDir if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -147,17 +143,11 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" cmd = "touch " + self.cfgPath if os.system(cmd) != 0: diff --git a/tests/pytest/util/dnodes-random-fail.py b/tests/pytest/util/dnodes-random-fail.py index 7cadca64a3..6590f1e204 100644 --- a/tests/pytest/util/dnodes-random-fail.py +++ b/tests/pytest/util/dnodes-random-fail.py @@ -71,17 +71,13 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" cmd = "rm -rf " + self.cfgDir if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -147,17 +143,11 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" cmd = "touch " + self.cfgPath if os.system(cmd) != 0: diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 235e4d25e0..f80c7be6cd 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -200,17 +200,11 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -332,6 +326,59 @@ class TDDnode: # time.sleep(5) + def startWin(self): + buildPath = self.getBuildPath("taosd.exe") + + if (buildPath == ""): + tdLog.exit("taosd.exe not found!") + else: + tdLog.info("taosd.exe found in %s" % buildPath) + + binPath = buildPath + "/build/bin/taosd.exe" + taosadapterBinPath = buildPath + "/build/bin/taosadapter.exe" + + if self.deployed == 0: + tdLog.exit("dnode:%d is not deployed" % (self.index)) + + cmd = "mintty -h never -w hide %s -c %s" % ( + binPath, self.cfgDir) + + taosadapterCmd = "mintty -h never -w hide %s " % ( + taosadapterBinPath) + if os.system(taosadapterCmd) != 0: + tdLog.exit(taosadapterCmd) + + if os.system(cmd) != 0: + tdLog.exit(cmd) + + self.running = 1 + tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) + if self.valgrind == 0: + time.sleep(0.1) + key = 'from offline to online' + bkey = bytes(key,encoding="utf8") + logFile = self.logDir + "/taosdlog.0" + i = 0 + while not os.path.exists(logFile): + sleep(0.1) + i += 1 + if i>50: + break + popen = subprocess.Popen('tail -n +0 -f ' + logFile, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + pid = popen.pid + # print('Popen.pid:' + str(pid)) + timeout = time.time() + 60*2 + while True: + line = popen.stdout.readline().strip() + if bkey in line: + popen.kill() + break + if time.time() > timeout: + tdLog.exit('wait too long for taosd start') + tdLog.debug("the dnode:%d has been started." % (self.index)) + else: + tdLog.debug("wait 10 seconds for the dnode:%d to start." % (self.index)) + time.sleep(10) def startWithoutSleep(self): buildPath = self.getBuildPath() @@ -547,6 +594,10 @@ class TDDnodes: def start(self, index): self.check(index) self.dnodes[index - 1].start() + + def startWin(self, index): + self.check(index) + self.dnodes[index - 1].startWin() def startWithoutSleep(self, index): self.check(index) diff --git a/tests/pytest/wal/addOldWalTest.py b/tests/pytest/wal/addOldWalTest.py index 2f4dcd5ce8..104329ede6 100644 --- a/tests/pytest/wal/addOldWalTest.py +++ b/tests/pytest/wal/addOldWalTest.py @@ -27,7 +27,7 @@ class TDTestCase: def createOldDir(self): oldDir = tdDnodes.getDnodesRootDir() + "dnode1/data/vnode/vnode2/wal/old" - os.system("sudo mkdir -p %s" % oldDir) + os.makedirs(oldDir, exist_ok=True) # like "mkdir -p" def createOldDirAndAddWal(self): oldDir = tdDnodes.getDnodesRootDir() + "dnode1/data/vnode/vnode2/wal/old" -- GitLab