Merge pull request #16008 from zhang2014/fix/ISSUES-15883

ISSUES-15883 try fix collate name

src/DataTypes/DataTypeString.cpp

@@ -384,7 +384,7 @@ static DataTypePtr create(const ASTPtr & arguments)
             throw Exception("String data type family mustn't have more than one argument - size in characters", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
 
         const auto * argument = arguments->children[0]->as<ASTLiteral>();
-        if (!argument || argument->value.getType() != Field::Types::UInt64 || argument->value.get<UInt64>() == 0)
+        if (!argument || argument->value.getType() != Field::Types::UInt64)
             throw Exception("String data type family may have only a number (positive integer) as its argument", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
     }
 

src/Parsers/MySQL/ASTAlterCommand.cpp

@@ -303,9 +303,9 @@ static inline bool parseOtherCommand(IParser::Pos & pos, ASTPtr & node, Expected
                 OptionDescribe("ENABLE KEYS", "enable_keys", std::make_shared<ParserAlwaysTrue>()),
                 OptionDescribe("DISABLE KEYS", "enable_keys", std::make_shared<ParserAlwaysFalse>()),
                 /// TODO: with collate
-                OptionDescribe("CONVERT TO CHARACTER SET", "charset", std::make_shared<ParserCharsetName>()),
-                OptionDescribe("CHARACTER SET", "charset", std::make_shared<ParserCharsetName>()),
-                OptionDescribe("DEFAULT CHARACTER SET", "charset", std::make_shared<ParserCharsetName>()),
+                OptionDescribe("CONVERT TO CHARACTER SET", "charset", std::make_shared<ParserCharsetOrCollateName>()),
+                OptionDescribe("CHARACTER SET", "charset", std::make_shared<ParserCharsetOrCollateName>()),
+                OptionDescribe("DEFAULT CHARACTER SET", "charset", std::make_shared<ParserCharsetOrCollateName>()),
                 OptionDescribe("LOCK", "lock", std::make_shared<ParserIdentifier>())
             }
         };

src/Parsers/MySQL/ASTDeclareColumn.cpp

@@ -51,8 +51,8 @@ static inline bool parseColumnDeclareOptions(IParser::Pos & pos, ASTPtr & node,
             OptionDescribe("UNIQUE", "unique_key", std::make_unique<ParserAlwaysTrue>()),
             OptionDescribe("KEY", "primary_key", std::make_unique<ParserAlwaysTrue>()),
             OptionDescribe("COMMENT", "comment", std::make_unique<ParserStringLiteral>()),
-            OptionDescribe("CHARACTER SET", "charset_name", std::make_unique<ParserCharsetName>()),
-            OptionDescribe("COLLATE", "collate", std::make_unique<ParserCharsetName>()),
+            OptionDescribe("CHARACTER SET", "charset_name", std::make_unique<ParserCharsetOrCollateName>()),
+            OptionDescribe("COLLATE", "collate", std::make_unique<ParserCharsetOrCollateName>()),
             OptionDescribe("COLUMN_FORMAT", "column_format", std::make_unique<ParserIdentifier>()),
             OptionDescribe("STORAGE", "storage", std::make_unique<ParserIdentifier>()),
             OptionDescribe("AS", "generated", std::make_unique<ParserExpression>()),

src/Parsers/MySQL/ASTDeclareOption.cpp

@@ -4,6 +4,7 @@
 #include <IO/ReadHelpers.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTLiteral.h>
+#include <Parsers/ExpressionElementParsers.h>
 
 namespace DB
 {
@@ -94,41 +95,21 @@ bool ParserAlwaysFalse::parseImpl(IParser::Pos & /*pos*/, ASTPtr & node, Expecte
     return true;
 }
 
-bool ParserCharsetName::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected &)
+bool ParserCharsetOrCollateName::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected)
 {
-    /// Identifier in backquotes or in double quotes
-    if (pos->type == TokenType::QuotedIdentifier)
-    {
-        ReadBufferFromMemory buf(pos->begin, pos->size());
-        String s;
-
-        if (*pos->begin == '`')
-            readBackQuotedStringWithSQLStyle(s, buf);
-        else
-            readDoubleQuotedStringWithSQLStyle(s, buf);
+    ParserIdentifier p_identifier;
+    ParserStringLiteral p_string_literal;
 
-        if (s.empty()) /// Identifiers "empty string" are not allowed.
-            return false;
-
-        node = std::make_shared<ASTIdentifier>(s);
-        ++pos;
+    if (p_identifier.parse(pos, node, expected))
         return true;
-    }
-    else if (pos->type == TokenType::BareWord)
+    else
     {
-        const char * begin = pos->begin;
-
-        while (true)
+        if (p_string_literal.parse(pos, node, expected))
         {
-            if (!isWhitespaceASCII(*pos->end) && pos->type != TokenType::EndOfStream)
-                ++pos;
-            else
-                break;
+            const auto & string_value = node->as<ASTLiteral>()->value.safeGet<String>();
+            node = std::make_shared<ASTIdentifier>(string_value);
+            return true;
         }
-
-        node = std::make_shared<ASTIdentifier>(String(begin, pos->end));
-        ++pos;
-        return true;
     }
 
     return false;

src/Parsers/MySQL/ASTDeclareOption.h

@@ -59,12 +59,11 @@ public:
     bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
 };
 
-/// Copy and paste from ParserIdentifier,
-/// the difference is that multiple tokens are glued if there is no whitespace ASCII between them
-struct ParserCharsetName : public IParserBase
+/// identifier, string literal, binary keyword
+struct ParserCharsetOrCollateName : public IParserBase
 {
 protected:
-    const char * getName() const override { return "charset name"; }
+    const char * getName() const override { return "charset or collate name"; }
 
     bool parseImpl(Pos & pos, ASTPtr & node, Expected &) override;
 };

src/Parsers/MySQL/ASTDeclareTableOptions.cpp

@@ -68,12 +68,12 @@ bool ParserDeclareTableOptions::parseImpl(IParser::Pos & pos, ASTPtr & node, Exp
         {
             OptionDescribe("AUTO_INCREMENT", "auto_increment", std::make_shared<ParserLiteral>()),
             OptionDescribe("AVG_ROW_LENGTH", "avg_row_length", std::make_shared<ParserLiteral>()),
-            OptionDescribe("CHARSET", "character_set", std::make_shared<ParserCharsetName>()),
-            OptionDescribe("DEFAULT CHARSET", "character_set", std::make_shared<ParserCharsetName>()),
-            OptionDescribe("CHARACTER SET", "character_set", std::make_shared<ParserCharsetName>()),
+            OptionDescribe("CHARSET", "character_set", std::make_shared<ParserCharsetOrCollateName>()),
+            OptionDescribe("DEFAULT CHARSET", "character_set", std::make_shared<ParserCharsetOrCollateName>()),
+            OptionDescribe("CHARACTER SET", "character_set", std::make_shared<ParserCharsetOrCollateName>()),
             OptionDescribe("DEFAULT CHARACTER SET", "character_set", std::make_shared<ParserIdentifier>()),
             OptionDescribe("CHECKSUM", "checksum", std::make_shared<ParserBoolOption<false>>()),
-            OptionDescribe("COLLATE", "collate", std::make_shared<ParserCharsetName>()),
+            OptionDescribe("COLLATE", "collate", std::make_shared<ParserCharsetOrCollateName>()),
             OptionDescribe("DEFAULT COLLATE", "collate", std::make_shared<ParserIdentifier>()),
             OptionDescribe("COMMENT", "comment", std::make_shared<ParserStringLiteral>()),
             OptionDescribe("COMPRESSION", "compression", std::make_shared<ParserStringLiteral>()),

src/Parsers/MySQL/tests/gtest_column_parser.cpp

@@ -15,7 +15,7 @@ TEST(ParserColumn, AllNonGeneratedColumnOption)
 {
     ParserDeclareColumn p_column;
 
-    String input = "col_01 VARCHAR(100) NOT NULL DEFAULT NULL AUTO_INCREMENT UNIQUE KEY PRIMARY KEY COMMENT 'column comment' COLLATE utf-8 "
+    String input = "col_01 VARCHAR(100) NOT NULL DEFAULT NULL AUTO_INCREMENT UNIQUE KEY PRIMARY KEY COMMENT 'column comment' COLLATE utf8 "
                    "COLUMN_FORMAT FIXED STORAGE MEMORY REFERENCES tbl_name (col_01) CHECK 1";
     ASTPtr ast = parseQuery(p_column, input.data(), input.data() + input.size(), "", 0, 0);
     EXPECT_EQ(ast->as<ASTDeclareColumn>()->name, "col_01");
@@ -29,7 +29,7 @@ TEST(ParserColumn, AllNonGeneratedColumnOption)
     EXPECT_EQ(declare_options->changes["unique_key"]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);
     EXPECT_EQ(declare_options->changes["primary_key"]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);
     EXPECT_EQ(declare_options->changes["comment"]->as<ASTLiteral>()->value.safeGet<DB::String>(), "column comment");
-    EXPECT_EQ(declare_options->changes["collate"]->as<ASTIdentifier>()->name(), "utf-8");
+    EXPECT_EQ(declare_options->changes["collate"]->as<ASTIdentifier>()->name(), "utf8");
     EXPECT_EQ(declare_options->changes["column_format"]->as<ASTIdentifier>()->name(), "FIXED");
     EXPECT_EQ(declare_options->changes["storage"]->as<ASTIdentifier>()->name(), "MEMORY");
     EXPECT_TRUE(declare_options->changes["reference"]->as<ASTDeclareReference>());
@@ -40,7 +40,7 @@ TEST(ParserColumn, AllGeneratedColumnOption)
 {
     ParserDeclareColumn p_column;
 
-    String input = "col_01 VARCHAR(100) NULL UNIQUE KEY PRIMARY KEY COMMENT 'column comment' COLLATE utf-8 "
+    String input = "col_01 VARCHAR(100) NULL UNIQUE KEY PRIMARY KEY COMMENT 'column comment' COLLATE utf8 "
                    "REFERENCES tbl_name (col_01) CHECK 1 GENERATED ALWAYS AS (1) STORED";
     ASTPtr ast = parseQuery(p_column, input.data(), input.data() + input.size(), "", 0, 0);
     EXPECT_EQ(ast->as<ASTDeclareColumn>()->name, "col_01");
@@ -52,7 +52,7 @@ TEST(ParserColumn, AllGeneratedColumnOption)
     EXPECT_EQ(declare_options->changes["unique_key"]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);
     EXPECT_EQ(declare_options->changes["primary_key"]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);
     EXPECT_EQ(declare_options->changes["comment"]->as<ASTLiteral>()->value.safeGet<DB::String>(), "column comment");
-    EXPECT_EQ(declare_options->changes["collate"]->as<ASTIdentifier>()->name(), "utf-8");
+    EXPECT_EQ(declare_options->changes["collate"]->as<ASTIdentifier>()->name(), "utf8");
     EXPECT_EQ(declare_options->changes["generated"]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);
     EXPECT_EQ(declare_options->changes["is_stored"]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);
     EXPECT_TRUE(declare_options->changes["reference"]->as<ASTDeclareReference>());

src/Parsers/MySQL/tests/gtest_table_options_parser.cpp

@@ -11,7 +11,7 @@ using namespace DB::MySQLParser;
 
 TEST(ParserTableOptions, AllSubpatitionOptions)
 {
-    String input = "AUTO_INCREMENt = 1 AVG_ROW_LENGTh 3 CHARACTER SET utf-8 CHECKSUM 1 COLLATE utf8_bin"
+    String input = "AUTO_INCREMENt = 1 AVG_ROW_LENGTh 3 CHARACTER SET utf8 CHECKSUM 1 COLLATE utf8_bin"
                    " COMMENT 'table option comment' COMPRESSION 'LZ4' CONNECTION 'connect_string' DATA DIRECTORY 'data_directory'"
                    " INDEX DIRECTORY 'index_directory' DELAY_KEY_WRITE 0 ENCRYPTION 'Y' ENGINE INNODB INSERT_METHOD NO KEY_BLOCK_SIZE 3"
                    " MAX_ROWS 1000 MIN_ROWS 0 PACK_KEYS DEFAULT PASSWORD 'password' ROW_FORMAT DYNAMIC STATS_AUTO_RECALC DEFAULT "
@@ -23,7 +23,7 @@ TEST(ParserTableOptions, AllSubpatitionOptions)
     ASTDeclareOptions * declare_options = ast->as<ASTDeclareOptions>();
     EXPECT_EQ(declare_options->changes["auto_increment"]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);
     EXPECT_EQ(declare_options->changes["avg_row_length"]->as<ASTLiteral>()->value.safeGet<UInt64>(), 3);
-    EXPECT_EQ(declare_options->changes["character_set"]->as<ASTIdentifier>()->name(), "utf-8");
+    EXPECT_EQ(declare_options->changes["character_set"]->as<ASTIdentifier>()->name(), "utf8");
     EXPECT_EQ(declare_options->changes["checksum"]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);
     EXPECT_EQ(declare_options->changes["collate"]->as<ASTIdentifier>()->name(), "utf8_bin");
     EXPECT_EQ(declare_options->changes["comment"]->as<ASTLiteral>()->value.safeGet<String>(), "table option comment");