Merge pull request #12480 from azat/fix-SETTINGS-parse-after-FORMAT

[RFC] Fix SETTINGS parse after FORMAT

src/Parsers/ParserQueryWithOutput.cpp

@@ -14,12 +14,15 @@
 #include <Parsers/ParserWatchQuery.h>
 #include <Parsers/ParserSetQuery.h>
 #include <Parsers/ASTExplainQuery.h>
+#include <Parsers/ASTSelectWithUnionQuery.h>
+#include <Parsers/ASTSetQuery.h>
 #include <Parsers/ParserShowAccessEntitiesQuery.h>
 #include <Parsers/ParserShowAccessQuery.h>
 #include <Parsers/ParserShowCreateAccessEntityQuery.h>
 #include <Parsers/ParserShowGrantsQuery.h>
 #include <Parsers/ParserShowPrivilegesQuery.h>
 #include <Parsers/ParserExplainQuery.h>
+#include <Parsers/QueryWithOutputSettingsPushDownVisitor.h>
 
 
 namespace DB
@@ -107,6 +110,14 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
         if (!parser_settings.parse(pos, query_with_output.settings_ast, expected))
             return false;
         query_with_output.children.push_back(query_with_output.settings_ast);
+
+        // SETTINGS after FORMAT is not parsed by the SELECT parser (ParserSelectQuery)
+        // Pass them manually, to apply in InterpreterSelectQuery::initSettings()
+        if (query->as<ASTSelectWithUnionQuery>())
+        {
+            QueryWithOutputSettingsPushDownVisitor::Data data{query_with_output.settings_ast};
+            QueryWithOutputSettingsPushDownVisitor(data).visit(query);
+        }
     }
 
     node = std::move(query);

src/Parsers/QueryWithOutputSettingsPushDownVisitor.cpp

+#include <Common/SettingsChanges.h>
+#include <Parsers/QueryWithOutputSettingsPushDownVisitor.h>
+#include <Parsers/ASTSelectWithUnionQuery.h>
+#include <Parsers/ASTSelectQuery.h>
+#include <Parsers/ASTSetQuery.h>
+#include <Parsers/ASTSubquery.h>
+
+#include <iterator>
+#include <algorithm>
+
+namespace DB
+{
+
+bool QueryWithOutputSettingsPushDownMatcher::needChildVisit(ASTPtr & node, const ASTPtr & child)
+{
+    if (node->as<ASTSelectWithUnionQuery>())
+        return true;
+    if (node->as<ASTSubquery>())
+        return true;
+    if (child->as<ASTSelectQuery>())
+        return true;
+    return false;
+}
+
+void QueryWithOutputSettingsPushDownMatcher::visit(ASTPtr & ast, Data & data)
+{
+    if (auto * select_query = ast->as<ASTSelectQuery>())
+        visit(*select_query, ast, data);
+}
+
+void QueryWithOutputSettingsPushDownMatcher::visit(ASTSelectQuery & select_query, ASTPtr &, Data & data)
+{
+    ASTPtr select_settings_ast = select_query.settings();
+    if (!select_settings_ast)
+    {
+        select_query.setExpression(ASTSelectQuery::Expression::SETTINGS, data.settings_ast->clone());
+        return;
+    }
+
+    SettingsChanges & select_settings = select_settings_ast->as<ASTSetQuery &>().changes;
+    SettingsChanges & settings = data.settings_ast->as<ASTSetQuery &>().changes;
+
+    for (auto & setting : settings)
+    {
+        auto it = std::find_if(select_settings.begin(), select_settings.end(), [&](auto & select_setting)
+        {
+            return select_setting.name == setting.name;
+        });
+        if (it == select_settings.end())
+            select_settings.push_back(setting);
+        else
+            it->value = setting.value;
+    }
+}
+
+}

src/Parsers/QueryWithOutputSettingsPushDownVisitor.h

+#pragma once
+
+#include <Parsers/IAST.h>
+#include <Interpreters/InDepthNodeVisitor.h>
+
+namespace DB
+{
+
+class ASTSelectQuery;
+struct SettingChange;
+using SettingsChanges = std::vector<SettingChange>;
+
+/// Pushdown SETTINGS clause that goes after FORMAT to the SELECT query:
+/// (since settings after FORMAT parsed separatelly not in the ParserSelectQuery but in ParserQueryWithOutput)
+///
+///     SELECT 1                             FORMAT Null SETTINGS max_block_size = 1 ->
+///     SELECT 1 SETTINGS max_block_size = 1 FORMAT Null SETTINGS max_block_size = 1
+///
+/// Otherwise settings after FORMAT will not be applied.
+class QueryWithOutputSettingsPushDownMatcher
+{
+public:
+    using Visitor = InDepthNodeVisitor<QueryWithOutputSettingsPushDownMatcher, true>;
+
+    struct Data
+    {
+        const ASTPtr & settings_ast;
+    };
+
+    static bool needChildVisit(ASTPtr & node, const ASTPtr & child);
+    static void visit(ASTPtr & ast, Data & data);
+
+private:
+    static void visit(ASTSelectQuery &, ASTPtr &, Data &);
+};
+
+using QueryWithOutputSettingsPushDownVisitor = QueryWithOutputSettingsPushDownMatcher::Visitor;
+
+}

src/Parsers/ya.make

@@ -116,6 +116,7 @@ SRCS(
     parseUserName.cpp
     queryToString.cpp
     TokenIterator.cpp
+    QueryWithOutputSettingsPushDownVisitor.cpp
 )
 
 END()

tests/queries/0_stateless/01401_FORMAT_SETTINGS.reference

+1
+1
+1
+1
+1
+2
+2

tests/queries/0_stateless/01401_FORMAT_SETTINGS.sh

+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CURDIR/../shell_config.sh
+
+set -o pipefail
+
+# test via http, since client parses settings by itself additionally
+
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'SELECT DISTINCT blockSize() FROM numbers(2) SETTINGS max_block_size = 1 FORMAT CSV'
+# push down
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'SELECT DISTINCT blockSize() FROM numbers(2) FORMAT CSV SETTINGS max_block_size = 1'
+# push down append
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'SELECT DISTINCT blockSize() FROM numbers(2) SETTINGS max_compress_block_size = 1 FORMAT CSV SETTINGS max_block_size = 1'
+# overwrite on push down (since these settings goes latest)
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'SELECT DISTINCT blockSize() FROM numbers(2) SETTINGS max_block_size = 2 FORMAT CSV SETTINGS max_block_size = 1'
+# on push-down
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'SELECT DISTINCT blockSize() FROM numbers(2) SETTINGS max_block_size = 1 FORMAT CSV'
+# no settings
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'SELECT DISTINCT blockSize() FROM numbers(2) FORMAT CSV'
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'SELECT DISTINCT blockSize() FROM numbers(2)'

tests/queries/0_stateless/arcadia_skip_list.txt

@@ -132,3 +132,4 @@
 01319_optimize_skip_unused_shards_nesting
 01376_GROUP_BY_injective_elimination_dictGet
 01391_join_on_dict_crash
+01401_FORMAT_SETTINGS