merge from 2.0 branch to master

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java

@@ -121,7 +121,7 @@ public class Utils {
     }
 
     private static void findValuesClauseRangeSet(String preparedSql, RangeSet<Integer> clauseRangeSet) {
-        Matcher matcher = Pattern.compile("(values|,)\\s*(\\([^)]*\\))").matcher(preparedSql);
+        Matcher matcher = Pattern.compile("(values||,)\\s*(\\([^)]*\\))").matcher(preparedSql);
         while (matcher.find()) {
             int start = matcher.start(2);
             int end = matcher.end(2);

src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java

@@ -73,6 +73,48 @@ public class UtilsTest {
         Assert.assertEquals(expected, actual);
     }
 
+    @Test
+    public void multiValuesAndWhitespace() {
+        // given
+        String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?)  VALUES (?,?,?,?)  (?,?,?,?) (?,?,?,?)";
+        Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5, 300, 3.141592, "uvw", 6).toArray();
+
+        // when
+        String actual = Utils.getNativeSql(nativeSql, parameters);
+
+        // then
+        String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1)  VALUES (100,3.14,'abc',4)  (200,3.1415,'xyz',5) (300,3.141592,'uvw',6)";
+        Assert.assertEquals(expected, actual);
+    }
+
+    @Test
+    public void multiValuesNoSeparator() {
+        // given
+        String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?)  VALUES (?,?,?,?)(?,?,?,?)(?,?,?,?)";
+        Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5, 300, 3.141592, "uvw", 6).toArray();
+
+        // when
+        String actual = Utils.getNativeSql(nativeSql, parameters);
+
+        // then
+        String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1)  VALUES (100,3.14,'abc',4)(200,3.1415,'xyz',5)(300,3.141592,'uvw',6)";
+        Assert.assertEquals(expected, actual);
+    }
+
+    @Test
+    public void multiValuesMultiSeparator() {
+        // given
+        String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?)  VALUES (?,?,?,?) (?,?,?,?), (?,?,?,?)";
+        Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5, 300, 3.141592, "uvw", 6).toArray();
+
+        // when
+        String actual = Utils.getNativeSql(nativeSql, parameters);
+
+        // then
+        String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1)  VALUES (100,3.14,'abc',4) (200,3.1415,'xyz',5), (300,3.141592,'uvw',6)";
+        Assert.assertEquals(expected, actual);
+    }
+
     @Test
     public void lineTerminator() {
         // given
@@ -100,6 +142,32 @@ public class UtilsTest {
         Assert.assertEquals(expected, actual);
     }
 
+    @Test
+    public void lineTerminatorAndMultiValuesAndNoneOrMoreWhitespace() {
+        String nativeSql = "INSERT Into ? TAGS(?) VALUES(?,?,\r\n?,?),(?,? ,\r\n?,?) t? tags (?) Values (?,?,?\r\n,?) (?,?,?,?) t? Tags(?) values  (?,?,?,?) , (?,?,?,?)";
+        Object[] parameters = Stream.of("t1", "abc", 100, 1.1, "xxx", "xxx", 200, 2.2, "xxx", "xxx", 2, "bcd", 300, 3.3, "xxx", "xxx", 400, 4.4, "xxx", "xxx", 3, "cde", 500, 5.5, "xxx", "xxx", 600, 6.6, "xxx", "xxx").toArray();
+
+        // when
+        String actual = Utils.getNativeSql(nativeSql, parameters);
+
+        // then
+        String expected = "INSERT Into t1 TAGS('abc') VALUES(100,1.1,\r\n'xxx','xxx'),(200,2.2 ,\r\n'xxx','xxx') t2 tags ('bcd') Values (300,3.3,'xxx'\r\n,'xxx') (400,4.4,'xxx','xxx') t3 Tags('cde') values  (500,5.5,'xxx','xxx') , (600,6.6,'xxx','xxx')";
+        Assert.assertEquals(expected, actual);
+    }
+
+    @Test
+    public void multiValuesAndNoneOrMoreWhitespace() {
+        String nativeSql = "INSERT INTO ? USING traces TAGS (?, ?) VALUES (?, ?, ?, ?, ?, ?, ?)  (?, ?, ?, ?, ?, ?, ?)";
+        Object[] parameters = Stream.of("t1", "t1", "t2", 1632968284000L, 111.111, 119.001, 0.4, 90, 99.1, "WGS84", 1632968285000L, 111.21109999999999, 120.001, 0.5, 91, 99.19999999999999, "WGS84").toArray();
+
+        // when
+        String actual = Utils.getNativeSql(nativeSql, parameters);
+
+        // then
+        String expected = "INSERT INTO t1 USING traces TAGS ('t1', 't2') VALUES (1632968284000, 111.111, 119.001, 0.4, 90, 99.1, 'WGS84')  (1632968285000, 111.21109999999999, 120.001, 0.5, 91, 99.19999999999999, 'WGS84')";
+        Assert.assertEquals(expected, actual);
+    }
+
     @Test
     public void replaceNothing() {
         // given

src/kit/shell/CMakeLists.txt

@@ -26,6 +26,8 @@ ENDIF ()
 
   SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos)
 ELSEIF (TD_WINDOWS)
+  ADD_DEFINITIONS(-DUNICODE)
+  ADD_DEFINITIONS(-D_UNICODE)
   LIST(APPEND SRC ./src/shellEngine.c)
   LIST(APPEND SRC ./src/shellMain.c)
   LIST(APPEND SRC ./src/shellWindows.c)

src/kit/shell/src/shellMain.c

@@ -95,6 +95,9 @@ SShellArguments args = {
  */
 int main(int argc, char* argv[]) {
   /*setlocale(LC_ALL, "en_US.UTF-8"); */
+#ifdef WINDOWS
+  SetConsoleOutputCP(CP_UTF8);
+#endif
 
   if (!checkVersion()) {
     exit(EXIT_FAILURE);

src/kit/shell/src/shellWindows.c

@@ -256,13 +256,16 @@ int32_t shellReadCommand(TAOS *con, char command[]) {
   cmd.command = (char *)calloc(1, MAX_COMMAND_SIZE);
 
   // Read input.
-  char c;
+  void *console = GetStdHandle(STD_INPUT_HANDLE);
+  unsigned long read;
+  wchar_t c;
+  char mbStr[16];
   while (1) {
-    c = getchar();
-
+    int ret = ReadConsole(console, &c, 1, &read, NULL);
+    int size = WideCharToMultiByte(CP_UTF8, 0, &c, read, mbStr, sizeof(mbStr), NULL, NULL);
+    mbStr[size] = 0;
     switch (c) {
       case '\n':
-      case '\r':
         if (isReadyGo(&cmd)) {
           sprintf(command, "%s%s", cmd.buffer, cmd.command);
           free(cmd.buffer);
@@ -275,8 +278,12 @@ int32_t shellReadCommand(TAOS *con, char command[]) {
           updateBuffer(&cmd);
         }
         break;
+      case '\r':
+        break;
       default:
-        insertChar(&cmd, c);
+        for (int i = 0; i < size; ++i) {
+          insertChar(&cmd, mbStr[i]);
+        }
     }
   }
 

src/util/src/tcompression.c

@@ -346,7 +346,7 @@ int tsCompressBoolImp(const char *const input, const int nelements, char *const
       /* t = (~((( uint8_t)1) << (7-i%BITS_PER_BYTE))); */
       output[pos] |= t;
     } else {
-      uError("Invalid compress bool value:%d", output[pos]);
+      uError("Invalid compress bool value:%d", input[i]);
       return -1;
     }
   }

tests/pytest/compress/compressChangeVersion.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+sys.path.insert(0, os.getcwd())
+from util.log import *
+from util.sql import *
+from util.dnodes import *
+import taos
+import threading
+import subprocess
+from random import choice
+
+ 
+class TwoClients:
+    def initConnection(self):
+        self.host = "chenhaoran01"
+        self.user = "root"
+        self.password = "taosdata"
+        self.config = "/home/chr/cfg/single/"     
+        self.port =6030 
+        self.rowNum = 10
+        self.ts = 1537146000000  
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
+
+    def run(self):
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        binPath = buildPath+ "/build/bin/"
+        walFilePath = "/var/lib/taos/mnode_bak/wal/"
+        
+        # new taos client
+        conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
+        print(conn1)
+        cur1 = conn1.cursor()
+        tdSql.init(cur1, True)
+
+        # create backgroud db and tb
+        tdSql.execute("drop database if exists db1")
+        os.system("%staosdemo -f compress/insertDataDb1.json -y " % binPath)
+        # create foreground db and tb 
+        tdSql.execute("drop database if exists foredb")
+        tdSql.execute("create database foredb")
+        tdSql.execute("use foredb")
+        print("123test")
+        tdSql.execute("create stable if not exists stb (ts timestamp, dataInt int, dataDouble  double,dataStr nchar(200)) tags(loc nchar(50),t1 int)")
+        tdSql.execute("create table tb1 using stb tags('beijing1', 10)")
+        tdSql.execute("insert into tb1 values(1614218412000,8635,98.861,'qazwsxedcrfvtgbyhnujmikolp1')(1614218422000,8636,98.862,'qazwsxedcrfvtgbyhnujmikolp2')")
+        tdSql.execute("create table tb2 using stb tags('beijing2', 11)")
+        tdSql.execute("insert into tb2 values(1614218432000,8647,98.863,'qazwsxedcrfvtgbyhnujmikolp3')")
+        tdSql.execute("insert into tb2 values(1614218442000,8648,98.864,'qazwsxedcrfvtgbyhnujmikolp4')")
+
+
+        # check data correct 
+        tdSql.execute("use db1")
+        tdSql.query("select count(tbname) from stb0")
+        tdSql.checkData(0, 0, 50000)
+        tdSql.query("select count(*) from stb0")
+        tdSql.checkData(0, 0, 5000000)        
+        tdSql.execute("use foredb")
+        tdSql.query("select count (tbname) from stb")
+        tdSql.checkData(0, 0, 2)
+        tdSql.query("select count (*) from stb")
+        tdSql.checkData(0, 0, 4)
+        tdSql.query("select * from tb1 order by ts")
+        tdSql.checkData(0, 3, "qazwsxedcrfvtgbyhnujmikolp1")
+        tdSql.query("select * from tb2 order by ts")
+        tdSql.checkData(1, 3, "qazwsxedcrfvtgbyhnujmikolp4")
+
+
+
+        # delete useless file       
+        testcaseFilename = os.path.split(__file__)[-1]
+        os.system("rm -rf ./insert_res.txt")
+        # os.system("rm -rf compress/%s.sql" % testcaseFilename )       
+
+clients = TwoClients()
+clients.initConnection()
+# clients.getBuildPath()
+clients.run()
\ No newline at end of file

tests/pytest/compress/insertDataDb1.json

+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,
+    "thread_count_create_tbl": 4,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "interlace_rows": 10,
+    "num_of_records_per_req": 1000,
+    "max_sql_len": 1024000,
+    "databases": [{
+        "dbinfo": {
+            "name": "db1",
+            "drop": "yes",
+            "replica": 1,
+            "days": 10,
+            "cache": 50,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 3650,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb0",
+            "child_table_exists":"no",
+            "childtable_count": 50000,
+            "childtable_prefix": "stb00_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 100,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 100,
+            "childtable_limit": 0,
+            "childtable_offset":0,
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"},{"type": "TINYINT"},{"type": "smallint"},{"type": "bool"},{"type": "bigint"},{"type": "float"},{"type": "double"}, {"type": "BINARY","len": 32}, {"type": "nchar","len": 32}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
+        }]
+    }]
+}
+

tests/pytest/tools/taosdemoTestTblAlt.py

@@ -54,7 +54,9 @@ class TDTestCase:
         binPath = buildPath + "/build/bin/"
 
         if(threadID == 0):
-            os.system("%staosdemo -y -t %d -n %d -b INT,INT,INT,INT -m t" %
+            print("%staosdemo -y -t %d -n %d -b INT,INT,INT,INT -m t" %
+                      (binPath, self.numberOfTables, self.numberOfRecords))
+            os.system("%staosdemo -y -t %d -n %d -b INT,INT,INT,INT" %
                       (binPath, self.numberOfTables, self.numberOfRecords))
         if(threadID == 1):
             time.sleep(2)