diff --git a/dbms/src/Common/ConfigProcessor.cpp b/dbms/src/Common/ConfigProcessor.cpp index e0202b559b9e2d529100d7a582ae51fb1f58dadc..4c08df0602c9c4fe431b99664360a098a50c056e 100644 --- a/dbms/src/Common/ConfigProcessor.cpp +++ b/dbms/src/Common/ConfigProcessor.cpp @@ -11,15 +11,11 @@ #include #include +#include using namespace Poco::XML; -static bool endsWith(const std::string & s, const std::string & suffix) -{ - return s.size() >= suffix.size() && s.substr(s.size() - suffix.size()) == suffix; -} - /// Extracts from a string the first encountered number consisting of at least two digits. static std::string numberFromHost(const std::string & s) { @@ -27,7 +23,7 @@ static std::string numberFromHost(const std::string & s) { std::string res; size_t j = i; - while (j < s.size() && isdigit(s[j])) + while (j < s.size() && isNumericASCII(s[j])) res += s[j++]; if (res.size() >= 2) { diff --git a/dbms/src/Functions/FunctionsStringSearch.cpp b/dbms/src/Functions/FunctionsStringSearch.cpp index daf595f7415089d18b1a7b62e9b11906a06161fc..dbc64a2cc2fe27bc1f4ca0582dfa8c9d877b8b11 100644 --- a/dbms/src/Functions/FunctionsStringSearch.cpp +++ b/dbms/src/Functions/FunctionsStringSearch.cpp @@ -587,7 +587,7 @@ struct ReplaceRegexpImpl { if (s[i] == '\\' && i + 1 < s.size()) { - if (isdigit(s[i + 1])) /// Substitution + if (isNumericASCII(s[i + 1])) /// Substitution { if (!now.empty()) { diff --git a/dbms/src/IO/ReadHelpers.cpp b/dbms/src/IO/ReadHelpers.cpp index d6fc6d34e5985a6a12d010193580fa1429e2c329..acc3706fa831ce668426bf77ff34c36597cfca14 100644 --- a/dbms/src/IO/ReadHelpers.cpp +++ b/dbms/src/IO/ReadHelpers.cpp @@ -683,7 +683,7 @@ void skipJSONFieldPlain(ReadBuffer & buf, const StringRef & name_of_filed) NullSink sink; readJSONStringInto(sink, buf); } - else if (isNumericASCII(*buf.position())) /// skip number + else if (isNumericASCII(*buf.position()) || *buf.position() == '-' || *buf.position() == '+') /// skip number { double v; if (!tryReadFloatText(v, buf)) @@ -737,7 +737,7 @@ void skipJSONFieldPlain(ReadBuffer & buf, const StringRef & name_of_filed) } else { - throw Exception("Unexpected symbol for key '" + name_of_filed.toString() + "'", ErrorCodes::INCORRECT_DATA); + throw Exception("Unexpected symbol '" + std::string(*buf.position(), 1) + "' for key '" + name_of_filed.toString() + "'", ErrorCodes::INCORRECT_DATA); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 91bf9b6407a0eb46d5793b064ae1a63ff27161fe..cd4807fb3ab4a700eb367661b20d977d8a5e4ebe 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1706,7 +1706,7 @@ static std::pair getMonthNameAndDayNum(const Field & partition ? toString(partition.get()) : partition.safeGet(); - if (month_name.size() != 6 || !std::all_of(month_name.begin(), month_name.end(), isdigit)) + if (month_name.size() != 6 || !std::all_of(month_name.begin(), month_name.end(), isNumericASCII)) throw Exception("Invalid partition format: " + month_name + ". Partition should consist of 6 digits: YYYYMM", ErrorCodes::INVALID_PARTITION_NAME); diff --git a/dbms/src/TableFunctions/TableFunctionRemote.cpp b/dbms/src/TableFunctions/TableFunctionRemote.cpp index 20bcb35ecdcea7d63c481e5083b7aa54b24a84c9..128f6d5003649f9b76c290ff2719f64e3d3efdec 100644 --- a/dbms/src/TableFunctions/TableFunctionRemote.cpp +++ b/dbms/src/TableFunctions/TableFunctionRemote.cpp @@ -52,7 +52,7 @@ static bool parseNumber(const String & description, size_t l, size_t r, size_t & res = 0; for (size_t pos = l; pos < r; pos ++) { - if (!isdigit(description[pos])) + if (!isNumericASCII(description[pos])) return false; res = res * 10 + description[pos] - '0'; if (res > 1e15) diff --git a/dbms/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.reference b/dbms/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.reference index 4150b9e80b1421458e383adc07a91fc3277e4fec..e9d98aa456e234c3f133b47fd56a3611d340ae0d 100644 --- a/dbms/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.reference +++ b/dbms/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.reference @@ -3,3 +3,4 @@ 0 1 ok 1 ok +1 ok diff --git a/dbms/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.sh b/dbms/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.sh index 0b4b33143ae48883217ed93e8cb0635338e11b87..7b34db0b9404edc7ae321e9706dc9abf7d1c7fbc 100755 --- a/dbms/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.sh +++ b/dbms/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.sh @@ -1,11 +1,16 @@ #!/usr/bin/env bash -clickhouse-client -n --query "DROP TABLE IF EXISTS test.json_noisy; CREATE TABLE test.json_noisy (d1 UInt8, d2 String) ENGINE = Memory" +set -e + +clickhouse-client -q "DROP TABLE IF EXISTS test.json_noisy" +clickhouse-client -q "CREATE TABLE test.json_noisy (d1 UInt8, d2 String) ENGINE = Memory" echo '{"d1" : 1, "d2" : "ok"} { } {"t1" : 0, "t2":true,"t3":false, "t4":null,"t5":[],"t6":"trash" } {"d2":"ok","t1":[[[]],true, null, false, "1","2",9.03,101], "t2":[["1","2"]], "d1":"1"} -{"d2":"ok","t1":[[[]],true, null, false, "1","2", 0.03, 1], "d1":"1", "t2":["1","2"]}' \ -| clickhouse-client -n --query "SET input_format_skip_unknown_fields = 1; INSERT INTO test.json_noisy FORMAT JSONEachRow" +{"d2":"ok","t1":[[[]],true, null, false, "1","2", 0.03, 1], "d1":"1", "t2":["1","2"]} +{"t0" : -0.1, "t1" : +1, "t2" : 0, "t3" : [0.0, -0.1], "d2" : "ok", "d1" : 1}' \ +| clickhouse-client --input_format_skip_unknown_fields=1 -q "INSERT INTO test.json_noisy FORMAT JSONEachRow" -clickhouse-client -n --query "SELECT * FROM test.json_noisy; DROP TABLE IF EXISTS test.json_noisy;" +clickhouse-client --max_threads=1 -q "SELECT * FROM test.json_noisy" +clickhouse-client -q "DROP TABLE IF EXISTS test.json_noisy"