Merge pull request #13101 from taosdata/fix/ZhiqiangWang/TD-15849-windows-timezone-error

fix(os): windows timezone error

source/libs/catalog/src/catalog.c

@@ -1194,7 +1194,7 @@ void catalogDestroy(void) {
   taosHashCleanup(gCtgMgmt.pCluster);
   gCtgMgmt.pCluster = NULL;
 
-  CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.lock);
+  if (CTG_IS_LOCKED(&gCtgMgmt.lock) == TD_RWLATCH_WRITE_FLAG_COPY) CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.lock);
 
   qInfo("catalog destroyed");
 }

source/os/src/osDir.c

@@ -204,7 +204,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
 int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) {
   wordexp_t full_path;
   if (0 != wordexp(dirname, &full_path, 0)) {
-    // printf("failed to expand path:%s since %s", dirname, strerror(errno));
+    printf("failed to expand path:%s since %s", dirname, strerror(errno));
     wordfree(&full_path);
     return -1;
   }

tests/pytest/cluster/clusterSetup.py

@@ -92,13 +92,13 @@ class Node:
             self.conn.run("yes|./install.sh")
 
     def configTaosd(self, taosConfigKey, taosConfigValue):
-        self.conn.run("sudo echo '%s %s' >> %s" % (taosConfigKey, taosConfigValue, "/etc/taos/taos.cfg"))
+        self.conn.run("sudo echo %s %s >> %s" % (taosConfigKey, taosConfigValue, "/etc/taos/taos.cfg"))
 
     def removeTaosConfig(self, taosConfigKey, taosConfigValue): 
         self.conn.run("sudo sed -in-place -e '/%s %s/d' %s" % (taosConfigKey, taosConfigValue, "/etc/taos/taos.cfg"))
     
     def configHosts(self, ip, name):
-        self.conn.run("echo '%s %s' >> %s" % (ip, name, '/etc/hosts'))
+        self.conn.run("echo %s %s >> %s" % (ip, name, '/etc/hosts'))
 
     def removeData(self):
         try:

tests/pytest/dockerCluster/basic.py

@@ -113,7 +113,7 @@ class BuildDockerCluser:
 
     def cfg(self, option, value, nodeIndex):
         cfgPath = "%s/node%d/cfg/taos.cfg" % (self.dockerDir, nodeIndex)
-        cmd = "echo '%s %s' >> %s" % (option, value, cfgPath)
+        cmd = "echo %s %s >> %s" % (option, value, cfgPath)
         self.execCmd(cmd)
     
     def updateLocalhosts(self):
@@ -122,7 +122,7 @@ class BuildDockerCluser:
         print(result)
         if result is None or result.isspace():
             print("==========")
-            cmd = "echo '172.27.0.7 tdnode1' >> /etc/hosts"
+            cmd = "echo 172.27.0.7 tdnode1 >> /etc/hosts"
             display = "echo %s" % cmd
             self.execCmd(display)
             self.execCmd(cmd)

tests/pytest/fulltest.bat

 
-python .\test.py -f insert\basic.py 
+python .\test.py -f insert\basic.py 
\ No newline at end of file
+python .\test.py -f insert\int.py 
+python .\test.py -f insert\float.py 
+python .\test.py -f insert\bigint.py 
+python .\test.py -f insert\bool.py 
+python .\test.py -f insert\double.py 
+python .\test.py -f insert\smallint.py 
+python .\test.py -f insert\tinyint.py 
+python .\test.py -f insert\date.py 
+python .\test.py -f insert\binary.py 
+python .\test.py -f insert\nchar.py 
+
+python .\test.py -f query\filter.py 
+python .\test.py -f query\filterCombo.py 
+python .\test.py -f query\queryNormal.py 
+python .\test.py -f query\queryError.py 
+python .\test.py -f query\filterAllIntTypes.py 
+python .\test.py -f query\filterFloatAndDouble.py 
+python .\test.py -f query\filterOtherTypes.py 
+python .\test.py -f query\querySort.py 
+python .\test.py -f query\queryJoin.py 
\ No newline at end of file

tests/pytest/manualTest/TD-5114/rollingUpgrade.py

@@ -38,7 +38,7 @@ class Node:
     def buildTaosd(self):
         try:
             print(self.conn)
-            # self.conn.run('echo "1234" > /home/chr/installtest/test.log')
+            # self.conn.run('echo 1234 > /home/chr/installtest/test.log')
             self.conn.run("cd /home/chr/installtest/ && tar -xvf  %s " %self.verName)
             self.conn.run("cd /home/chr/installtest/%s && ./install.sh " % self.installPath)
         except Exception as e:
@@ -49,7 +49,7 @@ class Node:
     def rebuildTaosd(self):
         try:
             print(self.conn)
-            # self.conn.run('echo "1234" > /home/chr/installtest/test.log')
+            # self.conn.run('echo 1234 > /home/chr/installtest/test.log')
             self.conn.run("cd /home/chr/installtest/%s && ./install.sh " % self.installPath)
         except Exception as e:
             print("Build Taosd error for node %d " % self.index)
@@ -108,7 +108,7 @@ class oneNode:
         # install TDengine at 192.168.103/104/141
         try:
             node = Node(id, username, IP, passwd, version)
-            node.conn.run('echo "start taosd"')
+            node.conn.run('echo start taosd')
             node.buildTaosd()
             # clear DataPath  , if need clear data 
             node.clearData()
@@ -128,7 +128,7 @@ class oneNode:
         # start TDengine 
         try:
             node = Node(id, username, IP, passwd, version)
-            node.conn.run('echo "restart taosd"')
+            node.conn.run('echo restart taosd')
             # clear DataPath  , if need clear data 
             node.clearData()
             node.restartTaosd()
@@ -149,14 +149,14 @@ class oneNode:
         verName = "TDengine-enterprise-server-%s-Linux-x64.tar.gz" % version
         # installPath = "TDengine-enterprise-server-%s" % self.version
         node131 = Node(131, 'ubuntu', '192.168.1.131', 'tbase125!', '2.0.20.0')
-        node131.conn.run('echo "upgrade cluster"')
+        node131.conn.run('echo upgrade cluster')
         node131.conn.run('sshpass  -p  tbase125! scp /nas/TDengine/v%s/enterprise/%s root@192.168.1.%d:/home/chr/installtest/' % (version,verName,id))
         node131.conn.close() 
         # upgrade TDengine at 192.168.103/104/141
         try:
             node = Node(id, username, IP, passwd, version)
-            node.conn.run('echo "start taosd"')
-            node.conn.run('echo "1234" > /home/chr/test.log')
+            node.conn.run('echo start taosd')
+            node.conn.run('echo 1234 > /home/chr/test.log')
             node.buildTaosd()
             time.sleep(5)
             node.startTaosd()
@@ -176,7 +176,7 @@ class oneNode:
         # backCluster TDengine at 192.168.103/104/141
         try:
             node = Node(id, username, IP, passwd, version)
-            node.conn.run('echo "rollback taos"')
+            node.conn.run('echo rollback taos')
             node.rebuildTaosd()
             time.sleep(5)
             node.startTaosd()

tests/pytest/test-all.bat

@@ -14,12 +14,14 @@ for /F "usebackq tokens=*" %%i in (fulltest.bat) do (
     echo Processing %%i
     set /a a+=1
     call %%i ARG1 -w 1 -m %1 > result_!a!.txt 2>error_!a!.txt
-    if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) 
+    if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && goto :end ) else ( call :colorEcho 0a "Success" &echo. ) 
 )
-exit
+goto :end
 
 :colorEcho
 echo off
 <nul set /p ".=%DEL%" > "%~2"
 findstr /v /a:%1 /R "^$" "%~2" nul
-del "%~2" > nul 2>&1i
+del "%~2" > nul 2>&1i
\ No newline at end of file
+
+:end
\ No newline at end of file

tests/pytest/util/dnodes.py

@@ -247,7 +247,7 @@ class TDDnode:
 
         paths = []
         for root, dirs, files in os.walk(projPath):
-            if ((tool) in files):
+            if ((tool) in files or ("%s.exe"%tool) in files):
                 rootRealPath = os.path.dirname(os.path.realpath(root))
                 if ("packaging" not in rootRealPath):
                     paths.append(os.path.join(root, tool))

tests/pytest/wal/addOldWalTest.py

@@ -31,7 +31,7 @@ class TDTestCase:
     
     def createOldDirAndAddWal(self):
         oldDir = tdDnodes.getDnodesRootDir() + "dnode1/data/vnode/vnode2/wal/old"        
-        os.system("sudo echo 'test' >> %s/wal" % oldDir)
+        os.system("sudo echo test >> %s/wal" % oldDir)
 
 
     def run(self):

tests/system-test/0-others/taosShell.py

@@ -3,8 +3,12 @@ import taos
 import sys
 import time
 import socket
-import pexpect
 import os
+import platform
+if platform.system().lower() == 'windows':
+    import wexpect as taosExpect
+else:
+    import pexpect as taosExpect
 
 from util.log import *
 from util.sql import *
@@ -15,7 +19,11 @@ def taos_command (buildPath, key, value, expectString, cfgDir, sqlString='', key
     if len(key) == 0:
         tdLog.exit("taos test key is null!")
     
-    taosCmd = buildPath + '/build/bin/taos '
+    if platform.system().lower() == 'windows':
+        taosCmd = buildPath + '\\build\\bin\\taos.exe '
+        taosCmd = taosCmd.replace('\\','\\\\')
+    else:
+        taosCmd = buildPath + '/build/bin/taos '
     if len(cfgDir) != 0:
         taosCmd = taosCmd + '-c ' + cfgDir
 
@@ -36,25 +44,30 @@ def taos_command (buildPath, key, value, expectString, cfgDir, sqlString='', key
 
     tdLog.info ("taos cmd: %s" % taosCmd)
 
-    child = pexpect.spawn(taosCmd, timeout=3)
+    child = taosExpect.spawn(taosCmd, timeout=3)
     #output = child.readline()
     #print (output.decode())
     if len(expectString) != 0:
-        i = child.expect([expectString, pexpect.TIMEOUT, pexpect.EOF], timeout=6)
+        i = child.expect([expectString, taosExpect.TIMEOUT, taosExpect.EOF], timeout=6)
     else:
-        i = child.expect([pexpect.TIMEOUT, pexpect.EOF], timeout=6)
+        i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=6)
 
-    retResult = child.before.decode()
+    if platform.system().lower() == 'windows':
+        retResult = child.before
+    else:
+        retResult = child.before.decode()
     print(retResult)
     #print(child.after.decode())
     if i == 0:
         print ('taos login success! Here can run sql, taos> ')
         if len(sqlString) != 0:
             child.sendline (sqlString)
-            w = child.expect(["Query OK", pexpect.TIMEOUT, pexpect.EOF], timeout=1)
+            w = child.expect(["Query OK", taosExpect.TIMEOUT, taosExpect.EOF], timeout=1)
             if w == 0:
                 return "TAOS_OK"
             else:
+                print(1)
+                print(retResult)
                 return "TAOS_FAIL"
         else:
             if key == 'A' or key1 == 'A' or key == 'C' or key1 == 'C' or key == 'V' or key1 == 'V':
@@ -102,7 +115,7 @@ class TDTestCase:
             projPath = selfPath[:selfPath.find("tests")]
 
         for root, dirs, files in os.walk(projPath):
-            if ("taosd" in files):
+            if ("taosd" in files or "taosd.exe" in files):
                 rootRealPath = os.path.dirname(os.path.realpath(root))
                 if ("packaging" not in rootRealPath):
                     buildPath = root[:len(root) - len("/build/bin")]
@@ -275,11 +288,15 @@ class TDTestCase:
         pwd=os.getcwd()
         newDbName="dbf"
         sqlFile = pwd + "/0-others/sql.txt"
-        sql1 = "echo 'create database " + newDbName + "' > " + sqlFile
-        sql2 = "echo 'use " + newDbName + "' >> " + sqlFile
-        sql3 = "echo 'create table ntbf (ts timestamp, c binary(40))' >> " + sqlFile
-        sql4 = "echo 'insert into ntbf values (\"2021-04-01 08:00:00.000\", \"test taos -f1\")(\"2021-04-01 08:00:01.000\", \"test taos -f2\")' >> " + sqlFile 
-        sql5 = "echo 'show databases' >> " + sqlFile       
+        sql1 = "echo create database " + newDbName + " > " + sqlFile
+        sql2 = "echo use " + newDbName + " >> " + sqlFile
+        if platform.system().lower() == 'windows':
+            sql3 = "echo create table ntbf (ts timestamp, c binary(40)) >> " + sqlFile
+            sql4 = "echo insert into ntbf values (\"2021-04-01 08:00:00.000\", \"test taos -f1\")(\"2021-04-01 08:00:01.000\", \"test taos -f2\") >> " + sqlFile 
+        else:
+            sql3 = "echo 'create table ntbf (ts timestamp, c binary(40))' >> " + sqlFile
+            sql4 = "echo 'insert into ntbf values (\"2021-04-01 08:00:00.000\", \"test taos -f1\")(\"2021-04-01 08:00:01.000\", \"test taos -f2\")' >> " + sqlFile 
+        sql5 = "echo show databases >> " + sqlFile       
         os.system(sql1)       
         os.system(sql2)       
         os.system(sql3)       

tests/system-test/0-others/taosShellError.py

@@ -216,11 +216,15 @@ class TDTestCase:
         pwd=os.getcwd()
         newDbName="dbf"
         sqlFile = pwd + "/0-others/sql.txt"
-        sql1 = "echo 'create database " + newDbName + "' > " + sqlFile
-        sql2 = "echo 'use " + newDbName + "' >> " + sqlFile
-        sql3 = "echo 'create table ntbf (ts timestamp, c binary(40)) no this item' >> " + sqlFile
-        sql4 = "echo 'insert into ntbf values (\"2021-04-01 08:00:00.000\", \"test taos -f1\")(\"2021-04-01 08:00:01.000\", \"test taos -f2\")' >> " + sqlFile 
-        sql5 = "echo 'show databases' >> " + sqlFile       
+        sql1 = "echo create database " + newDbName + " > " + sqlFile
+        sql2 = "echo use " + newDbName + " >> " + sqlFile
+        if platform.system().lower() == 'windows':
+            sql3 = "echo create table ntbf (ts timestamp, c binary(40)) no this item >> " + sqlFile
+            sql4 = "echo insert into ntbf values (\"2021-04-01 08:00:00.000\", \"test taos -f1\")(\"2021-04-01 08:00:01.000\", \"test taos -f2\") >> " + sqlFile 
+        else:
+            sql3 = "echo 'create table ntbf (ts timestamp, c binary(40)) no this item' >> " + sqlFile
+            sql4 = "echo 'insert into ntbf values (\"2021-04-01 08:00:00.000\", \"test taos -f1\")(\"2021-04-01 08:00:01.000\", \"test taos -f2\")' >> " + sqlFile 
+        sql5 = "echo show databases >> " + sqlFile       
         os.system(sql1)       
         os.system(sql2)       
         os.system(sql3)       

tests/system-test/test.py

@@ -17,6 +17,8 @@ import sys
 import getopt
 import subprocess
 import time
+import base64
+import json
 from distutils.log import warn as printf
 from fabric2 import Connection
 sys.path.append("../pytest")
@@ -38,8 +40,9 @@ if __name__ == "__main__":
     stop = 0
     restart = False
     windows = 0
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrw', [
-        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'windows'])
+    updateCfgDict = {}
+    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrwd:', [
+        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'windows', 'updateCfgDict'])
     for key, value in opts:
         if key in ['-h', '--help']:
             tdLog.printNoPrefix(
@@ -53,6 +56,7 @@ if __name__ == "__main__":
             tdLog.printNoPrefix('-g valgrind Test Flag')
             tdLog.printNoPrefix('-r taosd restart test')
             tdLog.printNoPrefix('-w taos on windows')
+            tdLog.printNoPrefix('-d update cfg dict, base64 json str')
             sys.exit(0)
 
         if key in ['-r', '--restart']: 
@@ -88,6 +92,12 @@ if __name__ == "__main__":
         if key in ['-w', '--windows']:
             windows = 1
 
+        if key in ['-d', '--updateCfgDict']:
+            try:
+                updateCfgDict = eval(base64.b64decode(value.encode()).decode())
+            except:
+                print('updateCfgDict convert fail.')
+                sys.exit(0)
     if (stop != 0):
         if (valgrind == 0):
             toBeKilled = "taosd"
@@ -127,15 +137,47 @@ if __name__ == "__main__":
     if windows:
         tdCases.logSql(logSql)
         tdLog.info("Procedures for testing self-deployment")
-        td_clinet = TDSimClient("C:\\TDengine")
-        td_clinet.deploy()
-        remote_conn = Connection("root@%s"%host)
-        with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
-            remote_conn.run("python3 ./test.py")
+        tdDnodes.init(deployPath)
+        tdDnodes.setTestCluster(testCluster)
+        tdDnodes.setValgrind(valgrind)
+        tdDnodes.stopAll()
+        key_word = 'tdCases.addWindows'
+        is_test_framework = 0
+        try:
+            if key_word in open(fileName).read():
+                is_test_framework = 1
+        except:
+            pass
+        updateCfgDictStr = ''
+        if is_test_framework:
+            moduleName = fileName.replace(".py", "").replace(os.sep, ".")
+            uModule = importlib.import_module(moduleName)
+            try:
+                ucase = uModule.TDTestCase()
+                if ((json.dumps(updateCfgDict) == '{}') and (ucase.updatecfgDict is not None)):
+                    updateCfgDict = ucase.updatecfgDict
+                    updateCfgDictStr = "-d %s"%base64.b64encode(json.dumps(updateCfgDict).encode()).decode()
+            except :
+                pass
+        else:
+            pass
+        tdDnodes.deploy(1,updateCfgDict)
+        if masterIp == "" or masterIp == "localhost":
+            tdDnodes.startWin(1)
+        else:
+            remote_conn = Connection("root@%s"%host)
+            with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
+                remote_conn.run("python3 ./test.py %s"%updateCfgDictStr)
+            # print("docker exec -d cross_platform bash -c \"cd ~/test/community/tests/system-test && python3 ./test.py %s\""%updateCfgDictStr)
+            # os.system("docker exec -d cross_platform bash -c \"cd ~/test/community/tests/system-test && python3 ./test.py %s\""%updateCfgDictStr)
+            # time.sleep(2)
         conn = taos.connect(
             host="%s"%(host),
-            config=td_clinet.cfgDir)
-        tdCases.runOneWindows(conn, fileName)
+            config=tdDnodes.sim.getCfgDir())
+        if is_test_framework:
+            tdCases.runOneWindows(conn, fileName)
+        else:
+            tdCases.runAllWindows(conn)
     else:
         tdDnodes.init(deployPath)
         tdDnodes.setTestCluster(testCluster)
@@ -153,16 +195,13 @@ if __name__ == "__main__":
             uModule = importlib.import_module(moduleName)
             try:
                 ucase = uModule.TDTestCase()
-                tdDnodes.deploy(1,ucase.updatecfgDict)
-            except :
-                tdDnodes.deploy(1,{})
-        else:
-            pass
-            tdDnodes.deploy(1,{})
+                if (json.dumps(updateCfgDict) == '{}'):
+                    updateCfgDict = ucase.updatecfgDict
+            except:
+                pass
+        tdDnodes.deploy(1,updateCfgDict)
         tdDnodes.start(1)
 
-
-
         tdCases.logSql(logSql)
 
         if testCluster:

tools/shell/CMakeLists.txt

 aux_source_directory(src SHELL_SRC)
 
 add_executable(shell ${SHELL_SRC})
+if(TD_WINDOWS)
+    target_link_libraries(shell PUBLIC taos_static)
+else()
+    target_link_libraries(shell PUBLIC taos)
+endif ()
 target_link_libraries(
     shell
-    PUBLIC taos
     PRIVATE os common transport util
 )
 target_include_directories(

tools/shell/src/shellArguments.c

@@ -36,6 +36,8 @@
 #define SHELL_VERSION  "Print program version."
 #define SHELL_EMAIL    "<support@taosdata.com>"
 
+static int32_t shellParseSingleOpt(int32_t key, char *arg);
+
 void shellPrintHelp() {
   char indent[] = "  ";
   printf("Usage: taos [OPTION...] \n\n");
@@ -90,6 +92,21 @@ static struct argp_option shellOptions[] = {
     {0},
 };
 
+static error_t shellParseOpt(int32_t key, char *arg, struct argp_state *state) { return shellParseSingleOpt(key, arg); }
+
+static struct argp shellArgp = {shellOptions, shellParseOpt, "", ""};
+
+static void shellParseArgsUseArgp(int argc, char *argv[]) {
+  argp_program_version = shell.info.programVersion;
+  argp_parse(&shellArgp, argc, argv, 0, 0, &shell.args);
+}
+
+#endif
+
+#ifndef ARGP_ERR_UNKNOWN
+  #define ARGP_ERR_UNKNOWN E2BIG
+#endif
+
 static int32_t shellParseSingleOpt(int32_t key, char *arg) {
   SShellArgs *pArgs = &shell.args;
 
@@ -196,8 +213,8 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) {
       }
       shellParseSingleOpt(key[1], val);
       i++;
-    } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'c' || key[1] == 'r' || key[1] == 'k' || key[1] == 't' ||
-               key[1] == 'V') {
+    } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C' || key[1] == 'r' || key[1] == 'k' || 
+               key[1] == 't' || key[1] == 'V' || key[1] == '?' || key[1] == 1) {
       shellParseSingleOpt(key[1], NULL);
     } else {
       fprintf(stderr, "invalid option %s\n", key);
@@ -208,21 +225,10 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) {
   return 0;
 }
 
-static error_t shellParseOpt(int32_t key, char *arg, struct argp_state *state) { return shellParseSingleOpt(key, arg); }
-
-static struct argp shellArgp = {shellOptions, shellParseOpt, "", ""};
-
-static void shellParseArgsUseArgp(int argc, char *argv[]) {
-  argp_program_version = shell.info.programVersion;
-  argp_parse(&shellArgp, argc, argv, 0, 0, &shell.args);
-}
-
-#endif
-
 static void shellInitArgs(int argc, char *argv[]) {
   for (int i = 1; i < argc; i++) {
     if (strncmp(argv[i], "-p", 2) == 0) {
-      printf(shell.info.clientVersion, tsOsName, taos_get_client_info());
+      // printf(shell.info.clientVersion, tsOsName, taos_get_client_info());
       if (strlen(argv[i]) == 2) {
         printf("Enter password: ");
         taosSetConsoleEcho(false);
@@ -341,7 +347,7 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) {
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
   shell.info.osname = "Windows";
   snprintf(shell.history.file, TSDB_FILENAME_LEN, "C:/TDengine/%s", SHELL_HISTORY_FILE);
-  // if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1;
+  if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1;
 #elif defined(_TD_DARWIN_64)
   shell.info.osname = "Darwin";
   snprintf(shell.history.file, TSDB_FILENAME_LEN, "%s/%s", getpwuid(getuid())->pw_dir, SHELL_HISTORY_FILE);