diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java index 9cf61903f001e84f237e25c3c10fdbb8aac28fd7..e1c4bddb2812f658336c895249886f603681e632 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java @@ -121,7 +121,7 @@ public class Utils { } private static void findValuesClauseRangeSet(String preparedSql, RangeSet clauseRangeSet) { - Matcher matcher = Pattern.compile("(values|,)\\s*(\\([^)]*\\))").matcher(preparedSql); + Matcher matcher = Pattern.compile("(values||,)\\s*(\\([^)]*\\))").matcher(preparedSql); while (matcher.find()) { int start = matcher.start(2); int end = matcher.end(2); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java index 1cbd95b2492284b9c85f31bd6b6848d9c223df18..66ecb9d63beb7c57ffb992a9ba5999b8fb70e739 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java @@ -73,6 +73,48 @@ public class UtilsTest { Assert.assertEquals(expected, actual); } + @Test + public void multiValuesAndWhitespace() { + // given + String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,?,?) (?,?,?,?) (?,?,?,?)"; + Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5, 300, 3.141592, "uvw", 6).toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1) VALUES (100,3.14,'abc',4) (200,3.1415,'xyz',5) (300,3.141592,'uvw',6)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void multiValuesNoSeparator() { + // given + String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,?,?)(?,?,?,?)(?,?,?,?)"; + Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5, 300, 3.141592, "uvw", 6).toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1) VALUES (100,3.14,'abc',4)(200,3.1415,'xyz',5)(300,3.141592,'uvw',6)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void multiValuesMultiSeparator() { + // given + String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,?,?) (?,?,?,?), (?,?,?,?)"; + Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5, 300, 3.141592, "uvw", 6).toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1) VALUES (100,3.14,'abc',4) (200,3.1415,'xyz',5), (300,3.141592,'uvw',6)"; + Assert.assertEquals(expected, actual); + } + @Test public void lineTerminator() { // given @@ -100,6 +142,32 @@ public class UtilsTest { Assert.assertEquals(expected, actual); } + @Test + public void lineTerminatorAndMultiValuesAndNoneOrMoreWhitespace() { + String nativeSql = "INSERT Into ? TAGS(?) VALUES(?,?,\r\n?,?),(?,? ,\r\n?,?) t? tags (?) Values (?,?,?\r\n,?) (?,?,?,?) t? Tags(?) values (?,?,?,?) , (?,?,?,?)"; + Object[] parameters = Stream.of("t1", "abc", 100, 1.1, "xxx", "xxx", 200, 2.2, "xxx", "xxx", 2, "bcd", 300, 3.3, "xxx", "xxx", 400, 4.4, "xxx", "xxx", 3, "cde", 500, 5.5, "xxx", "xxx", 600, 6.6, "xxx", "xxx").toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT Into t1 TAGS('abc') VALUES(100,1.1,\r\n'xxx','xxx'),(200,2.2 ,\r\n'xxx','xxx') t2 tags ('bcd') Values (300,3.3,'xxx'\r\n,'xxx') (400,4.4,'xxx','xxx') t3 Tags('cde') values (500,5.5,'xxx','xxx') , (600,6.6,'xxx','xxx')"; + Assert.assertEquals(expected, actual); + } + + @Test + public void multiValuesAndNoneOrMoreWhitespace() { + String nativeSql = "INSERT INTO ? USING traces TAGS (?, ?) VALUES (?, ?, ?, ?, ?, ?, ?) (?, ?, ?, ?, ?, ?, ?)"; + Object[] parameters = Stream.of("t1", "t1", "t2", 1632968284000L, 111.111, 119.001, 0.4, 90, 99.1, "WGS84", 1632968285000L, 111.21109999999999, 120.001, 0.5, 91, 99.19999999999999, "WGS84").toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT INTO t1 USING traces TAGS ('t1', 't2') VALUES (1632968284000, 111.111, 119.001, 0.4, 90, 99.1, 'WGS84') (1632968285000, 111.21109999999999, 120.001, 0.5, 91, 99.19999999999999, 'WGS84')"; + Assert.assertEquals(expected, actual); + } + @Test public void replaceNothing() { // given diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index 747b1bf28ddf24869dba4db261921f869bedd5e1..d0edde83898321c8eac76c4ffc527be0c6b1b6f8 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -26,6 +26,8 @@ ENDIF () SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ELSEIF (TD_WINDOWS) + ADD_DEFINITIONS(-DUNICODE) + ADD_DEFINITIONS(-D_UNICODE) LIST(APPEND SRC ./src/shellEngine.c) LIST(APPEND SRC ./src/shellMain.c) LIST(APPEND SRC ./src/shellWindows.c) diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 5c9dc0995dacecebd10b7f2b77e216ca97157db0..afed5d2d2ffa680852c1155334499975cd58cfea 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -95,6 +95,9 @@ SShellArguments args = { */ int main(int argc, char* argv[]) { /*setlocale(LC_ALL, "en_US.UTF-8"); */ +#ifdef WINDOWS + SetConsoleOutputCP(CP_UTF8); +#endif if (!checkVersion()) { exit(EXIT_FAILURE); diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index abec34b84c5ff65d6cb13492028cd36321d2d0ca..de8f6ad4aa31634f9ba3e4098f250a7fe7cb30d8 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -256,13 +256,16 @@ int32_t shellReadCommand(TAOS *con, char command[]) { cmd.command = (char *)calloc(1, MAX_COMMAND_SIZE); // Read input. - char c; + void *console = GetStdHandle(STD_INPUT_HANDLE); + unsigned long read; + wchar_t c; + char mbStr[16]; while (1) { - c = getchar(); - + int ret = ReadConsole(console, &c, 1, &read, NULL); + int size = WideCharToMultiByte(CP_UTF8, 0, &c, read, mbStr, sizeof(mbStr), NULL, NULL); + mbStr[size] = 0; switch (c) { case '\n': - case '\r': if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); free(cmd.buffer); @@ -275,8 +278,12 @@ int32_t shellReadCommand(TAOS *con, char command[]) { updateBuffer(&cmd); } break; + case '\r': + break; default: - insertChar(&cmd, c); + for (int i = 0; i < size; ++i) { + insertChar(&cmd, mbStr[i]); + } } } diff --git a/src/util/src/tcompression.c b/src/util/src/tcompression.c index 48bba75926415752cfd777242a55ef71c5c96c2c..66150c46fb22a7225444b4e13b2d35fc636ea15b 100644 --- a/src/util/src/tcompression.c +++ b/src/util/src/tcompression.c @@ -346,7 +346,7 @@ int tsCompressBoolImp(const char *const input, const int nelements, char *const /* t = (~((( uint8_t)1) << (7-i%BITS_PER_BYTE))); */ output[pos] |= t; } else { - uError("Invalid compress bool value:%d", output[pos]); + uError("Invalid compress bool value:%d", input[i]); return -1; } } diff --git a/tests/pytest/compress/compressChangeVersion.py b/tests/pytest/compress/compressChangeVersion.py new file mode 100644 index 0000000000000000000000000000000000000000..b7b9ebe6b35dc4729e0dcae705ac7d93c73010e7 --- /dev/null +++ b/tests/pytest/compress/compressChangeVersion.py @@ -0,0 +1,109 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import sys +sys.path.insert(0, os.getcwd()) +from util.log import * +from util.sql import * +from util.dnodes import * +import taos +import threading +import subprocess +from random import choice + + +class TwoClients: + def initConnection(self): + self.host = "chenhaoran01" + self.user = "root" + self.password = "taosdata" + self.config = "/home/chr/cfg/single/" + self.port =6030 + self.rowNum = 10 + self.ts = 1537146000000 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + walFilePath = "/var/lib/taos/mnode_bak/wal/" + + # new taos client + conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) + print(conn1) + cur1 = conn1.cursor() + tdSql.init(cur1, True) + + # create backgroud db and tb + tdSql.execute("drop database if exists db1") + os.system("%staosdemo -f compress/insertDataDb1.json -y " % binPath) + # create foreground db and tb + tdSql.execute("drop database if exists foredb") + tdSql.execute("create database foredb") + tdSql.execute("use foredb") + print("123test") + tdSql.execute("create stable if not exists stb (ts timestamp, dataInt int, dataDouble double,dataStr nchar(200)) tags(loc nchar(50),t1 int)") + tdSql.execute("create table tb1 using stb tags('beijing1', 10)") + tdSql.execute("insert into tb1 values(1614218412000,8635,98.861,'qazwsxedcrfvtgbyhnujmikolp1')(1614218422000,8636,98.862,'qazwsxedcrfvtgbyhnujmikolp2')") + tdSql.execute("create table tb2 using stb tags('beijing2', 11)") + tdSql.execute("insert into tb2 values(1614218432000,8647,98.863,'qazwsxedcrfvtgbyhnujmikolp3')") + tdSql.execute("insert into tb2 values(1614218442000,8648,98.864,'qazwsxedcrfvtgbyhnujmikolp4')") + + + # check data correct + tdSql.execute("use db1") + tdSql.query("select count(tbname) from stb0") + tdSql.checkData(0, 0, 50000) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 5000000) + tdSql.execute("use foredb") + tdSql.query("select count (tbname) from stb") + tdSql.checkData(0, 0, 2) + tdSql.query("select count (*) from stb") + tdSql.checkData(0, 0, 4) + tdSql.query("select * from tb1 order by ts") + tdSql.checkData(0, 3, "qazwsxedcrfvtgbyhnujmikolp1") + tdSql.query("select * from tb2 order by ts") + tdSql.checkData(1, 3, "qazwsxedcrfvtgbyhnujmikolp4") + + + + # delete useless file + testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf ./insert_res.txt") + # os.system("rm -rf compress/%s.sql" % testcaseFilename ) + +clients = TwoClients() +clients.initConnection() +# clients.getBuildPath() +clients.run() \ No newline at end of file diff --git a/tests/pytest/compress/insertDataDb1.json b/tests/pytest/compress/insertDataDb1.json new file mode 100644 index 0000000000000000000000000000000000000000..65cec71a65ff4ef3814bee4949def151c32945ee --- /dev/null +++ b/tests/pytest/compress/insertDataDb1.json @@ -0,0 +1,62 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db1", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 50000, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 100, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"},{"type": "TINYINT"},{"type": "smallint"},{"type": "bool"},{"type": "bigint"},{"type": "float"},{"type": "double"}, {"type": "BINARY","len": 32}, {"type": "nchar","len": 32}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} + diff --git a/tests/pytest/tools/taosdemoTestTblAlt.py b/tests/pytest/tools/taosdemoTestTblAlt.py index 70df535f59cbb97469b7a73e4e230d9a8671bfc7..bdc6769bdb8ab437fc1a4d2a4478768e35402048 100644 --- a/tests/pytest/tools/taosdemoTestTblAlt.py +++ b/tests/pytest/tools/taosdemoTestTblAlt.py @@ -54,7 +54,9 @@ class TDTestCase: binPath = buildPath + "/build/bin/" if(threadID == 0): - os.system("%staosdemo -y -t %d -n %d -b INT,INT,INT,INT -m t" % + print("%staosdemo -y -t %d -n %d -b INT,INT,INT,INT -m t" % + (binPath, self.numberOfTables, self.numberOfRecords)) + os.system("%staosdemo -y -t %d -n %d -b INT,INT,INT,INT" % (binPath, self.numberOfTables, self.numberOfRecords)) if(threadID == 1): time.sleep(2)