Resolves #964. Fixed input_format_skip_unknown_fields for negative numbers. [#CLICKHOUSE-3]

dbms/src/Common/ConfigProcessor.cpp

@@ -11,15 +11,11 @@
 #include <Poco/Util/XMLConfiguration.h>
 
 #include <Common/ZooKeeper/ZooKeeperNodeCache.h>
+#include <Common/StringUtils.h>
 
 using namespace Poco::XML;
 
 
-static bool endsWith(const std::string & s, const std::string & suffix)
-{
-    return s.size() >= suffix.size() && s.substr(s.size() - suffix.size()) == suffix;
-}
-
 /// Extracts from a string the first encountered number consisting of at least two digits.
 static std::string numberFromHost(const std::string & s)
 {
@@ -27,7 +23,7 @@ static std::string numberFromHost(const std::string & s)
     {
         std::string res;
         size_t j = i;
-        while (j < s.size() && isdigit(s[j]))
+        while (j < s.size() && isNumericASCII(s[j]))
             res += s[j++];
         if (res.size() >= 2)
         {

dbms/src/Functions/FunctionsStringSearch.cpp

@@ -587,7 +587,7 @@ struct ReplaceRegexpImpl
         {
             if (s[i] == '\\' && i + 1 < s.size())
             {
-                if (isdigit(s[i + 1])) /// Substitution
+                if (isNumericASCII(s[i + 1])) /// Substitution
                 {
                     if (!now.empty())
                     {

dbms/src/IO/ReadHelpers.cpp

@@ -683,7 +683,7 @@ void skipJSONFieldPlain(ReadBuffer & buf, const StringRef & name_of_filed)
         NullSink sink;
         readJSONStringInto(sink, buf);
     }
-    else if (isNumericASCII(*buf.position())) /// skip number
+    else if (isNumericASCII(*buf.position()) || *buf.position() == '-' || *buf.position() == '+') /// skip number
     {
         double v;
         if (!tryReadFloatText(v, buf))
@@ -737,7 +737,7 @@ void skipJSONFieldPlain(ReadBuffer & buf, const StringRef & name_of_filed)
     }
     else
     {
-        throw Exception("Unexpected symbol for key '" + name_of_filed.toString() + "'", ErrorCodes::INCORRECT_DATA);
+        throw Exception("Unexpected symbol '" + std::string(*buf.position(), 1) + "' for key '" + name_of_filed.toString() + "'", ErrorCodes::INCORRECT_DATA);
     }
 }
 

dbms/src/Storages/MergeTree/MergeTreeData.cpp

@@ -1706,7 +1706,7 @@ static std::pair<String, DayNum_t> getMonthNameAndDayNum(const Field & partition
         ? toString(partition.get<UInt64>())
         : partition.safeGet<String>();
 
-    if (month_name.size() != 6 || !std::all_of(month_name.begin(), month_name.end(), isdigit))
+    if (month_name.size() != 6 || !std::all_of(month_name.begin(), month_name.end(), isNumericASCII))
         throw Exception("Invalid partition format: " + month_name + ". Partition should consist of 6 digits: YYYYMM",
             ErrorCodes::INVALID_PARTITION_NAME);
 

dbms/src/TableFunctions/TableFunctionRemote.cpp

@@ -52,7 +52,7 @@ static bool parseNumber(const String & description, size_t l, size_t r, size_t &
     res = 0;
     for (size_t pos = l; pos < r; pos ++)
     {
-        if (!isdigit(description[pos]))
+        if (!isNumericASCII(description[pos]))
             return false;
         res = res * 10 + description[pos] - '0';
         if (res > 1e15)

dbms/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.reference

@@ -3,3 +3,4 @@
 0	
 1	ok
 1	ok
+1	ok

dbms/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.sh

 #!/usr/bin/env bash
-clickhouse-client -n --query "DROP TABLE IF EXISTS test.json_noisy; CREATE TABLE test.json_noisy (d1 UInt8, d2 String) ENGINE = Memory"
+set -e
+
+clickhouse-client -q "DROP TABLE IF EXISTS test.json_noisy"
+clickhouse-client -q "CREATE TABLE test.json_noisy (d1 UInt8, d2 String) ENGINE = Memory"
 
 echo '{"d1" : 1, "d2" : "ok"}
 { }
 {"t1" : 0, "t2":true,"t3":false, "t4":null,"t5":[],"t6":"trash" }
 {"d2":"ok","t1":[[[]],true, null, false, "1","2",9.03,101], "t2":[["1","2"]], "d1":"1"}
-{"d2":"ok","t1":[[[]],true, null, false, "1","2", 0.03, 1], "d1":"1", "t2":["1","2"]}' \
-| clickhouse-client -n --query "SET input_format_skip_unknown_fields = 1; INSERT INTO test.json_noisy FORMAT JSONEachRow"
+{"d2":"ok","t1":[[[]],true, null, false, "1","2", 0.03, 1], "d1":"1", "t2":["1","2"]}
+{"t0" : -0.1, "t1" : +1, "t2" : 0, "t3" : [0.0, -0.1], "d2" : "ok", "d1" : 1}' \
+| clickhouse-client --input_format_skip_unknown_fields=1 -q "INSERT INTO test.json_noisy FORMAT JSONEachRow"
 
-clickhouse-client -n --query "SELECT * FROM test.json_noisy; DROP TABLE IF EXISTS test.json_noisy;"
+clickhouse-client --max_threads=1 -q "SELECT * FROM test.json_noisy"
+clickhouse-client -q "DROP TABLE IF EXISTS test.json_noisy"