未验证 提交 3877aad6 编写于 作者: A alexey-milovidov 提交者: GitHub

Merge pull request #3773 from 4ertus2/joins

InDepthNodeVisitor for AST traversal 
......@@ -9,42 +9,55 @@
#include <Parsers/ASTIdentifier.h>
#include <DataTypes/NestedUtils.h>
#include <Interpreters/InDepthNodeVisitor.h>
namespace DB
{
/// Visitors consist of functions with unified interface 'void visit(Casted & x, ASTPtr & y)', there x is y, successfully casted to Casted.
/// Both types and fuction could have const specifiers. The second argument is used by visitor to replaces AST node (y) if needed.
/// Fills the array_join_result_to_source: on which columns-arrays to replicate, and how to call them after that.
class ArrayJoinedColumnsVisitor
class ArrayJoinedColumnsMatcher
{
public:
ArrayJoinedColumnsVisitor(NameToNameMap & array_join_name_to_alias_,
NameToNameMap & array_join_alias_to_name_,
NameToNameMap & array_join_result_to_source_)
: array_join_name_to_alias(array_join_name_to_alias_),
array_join_alias_to_name(array_join_alias_to_name_),
array_join_result_to_source(array_join_result_to_source_)
{}
void visit(ASTPtr & ast) const
struct Data
{
if (!tryVisit<ASTTablesInSelectQuery>(ast) &&
!tryVisit<ASTIdentifier>(ast))
visitChildren(ast);
}
NameToNameMap & array_join_name_to_alias;
NameToNameMap & array_join_alias_to_name;
NameToNameMap & array_join_result_to_source;
};
private:
NameToNameMap & array_join_name_to_alias;
NameToNameMap & array_join_alias_to_name;
NameToNameMap & array_join_result_to_source;
static constexpr const char * label = "ArrayJoinedColumns";
static bool needChildVisit(ASTPtr & node, const ASTPtr & child)
{
/// Processed
if (typeid_cast<ASTIdentifier *>(node.get()))
return false;
if (typeid_cast<ASTTablesInSelectQuery *>(node.get()))
return false;
void visit(const ASTTablesInSelectQuery &, ASTPtr &) const
{}
if (typeid_cast<ASTSubquery *>(child.get()) ||
typeid_cast<ASTSelectQuery *>(child.get()))
return false;
void visit(const ASTIdentifier & node, ASTPtr &) const
return true;
}
static std::vector<ASTPtr *> visit(ASTPtr & ast, Data & data)
{
if (auto * t = typeid_cast<ASTIdentifier *>(ast.get()))
visit(*t, ast, data);
return {};
}
private:
static void visit(const ASTIdentifier & node, ASTPtr &, Data & data)
{
NameToNameMap & array_join_name_to_alias = data.array_join_name_to_alias;
NameToNameMap & array_join_alias_to_name = data.array_join_alias_to_name;
NameToNameMap & array_join_result_to_source = data.array_join_result_to_source;
if (!node.general())
return;
......@@ -74,34 +87,11 @@ private:
/** Example: SELECT ParsedParams.Key1 FROM ... ARRAY JOIN ParsedParams AS PP.
*/
array_join_result_to_source[ /// PP.Key1 -> ParsedParams.Key1
Nested::concatenateName(array_join_name_to_alias[splitted.first], splitted.second)] = node.name;
}
}
void visit(const ASTSubquery &, ASTPtr &) const
{}
void visit(const ASTSelectQuery &, ASTPtr &) const
{}
void visitChildren(ASTPtr & ast) const
{
for (auto & child : ast->children)
if (!tryVisit<ASTSubquery>(child) &&
!tryVisit<ASTSelectQuery>(child))
visit(child);
}
template <typename T>
bool tryVisit(ASTPtr & ast) const
{
if (const T * t = typeid_cast<const T *>(ast.get()))
{
visit(*t, ast);
return true;
Nested::concatenateName(array_join_name_to_alias[splitted.first], splitted.second)] = node.name;
}
return false;
}
};
using ArrayJoinedColumnsVisitor = InDepthNodeVisitor<ArrayJoinedColumnsMatcher, true>;
}
......@@ -35,17 +35,40 @@ static ASTPtr addTypeConversion(std::unique_ptr<ASTLiteral> && ast, const String
return res;
}
void ExecuteScalarSubqueriesVisitor::visit(const ASTSubquery & subquery, ASTPtr & ast) const
bool ExecuteScalarSubqueriesMatcher::needChildVisit(ASTPtr & node, const ASTPtr &)
{
Context subquery_context = context;
Settings subquery_settings = context.getSettings();
/// Processed
if (typeid_cast<ASTSubquery *>(node.get()) ||
typeid_cast<ASTFunction *>(node.get()))
return false;
/// Don't descend into subqueries in FROM section
if (typeid_cast<ASTTableExpression *>(node.get()))
return false;
return true;
}
std::vector<ASTPtr *> ExecuteScalarSubqueriesMatcher::visit(ASTPtr & ast, Data & data)
{
if (auto * t = typeid_cast<ASTSubquery *>(ast.get()))
visit(*t, ast, data);
if (auto * t = typeid_cast<ASTFunction *>(ast.get()))
return visit(*t, ast, data);
return {};
}
void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr & ast, Data & data)
{
Context subquery_context = data.context;
Settings subquery_settings = data.context.getSettings();
subquery_settings.max_result_rows = 1;
subquery_settings.extremes = 0;
subquery_context.setSettings(subquery_settings);
ASTPtr subquery_select = subquery.children.at(0);
BlockIO res = InterpreterSelectWithUnionQuery(
subquery_select, subquery_context, {}, QueryProcessingStage::Complete, subquery_depth + 1).execute();
subquery_select, subquery_context, {}, QueryProcessingStage::Complete, data.subquery_depth + 1).execute();
Block block;
try
......@@ -100,31 +123,29 @@ void ExecuteScalarSubqueriesVisitor::visit(const ASTSubquery & subquery, ASTPtr
}
}
void ExecuteScalarSubqueriesVisitor::visit(const ASTTableExpression &, ASTPtr &) const
{
/// Don't descend into subqueries in FROM section.
}
void ExecuteScalarSubqueriesVisitor::visit(const ASTFunction & func, ASTPtr & ast) const
std::vector<ASTPtr *> ExecuteScalarSubqueriesMatcher::visit(const ASTFunction & func, ASTPtr & ast, Data &)
{
/// Don't descend into subqueries in arguments of IN operator.
/// But if an argument is not subquery, than deeper may be scalar subqueries and we need to descend in them.
std::vector<ASTPtr *> out;
if (functionIsInOrGlobalInOperator(func.name))
{
for (auto & child : ast->children)
{
if (child != func.arguments)
visit(child);
out.push_back(&child);
else
for (size_t i = 0, size = func.arguments->children.size(); i < size; ++i)
if (i != 1 || !typeid_cast<ASTSubquery *>(func.arguments->children[i].get()))
visit(func.arguments->children[i]);
out.push_back(&func.arguments->children[i]);
}
}
else
visitChildren(ast);
for (auto & child : ast->children)
out.push_back(&child);
return out;
}
}
#pragma once
#include <Common/typeid_cast.h>
#include <Interpreters/Context.h>
#include <Parsers/DumpASTNode.h>
#include <Parsers/ASTSubquery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Parsers/ASTFunction.h>
#include <Interpreters/InDepthNodeVisitor.h>
namespace DB
{
/// Visitors consist of functions with unified interface 'void visit(Casted & x, ASTPtr & y)', there x is y, successfully casted to Casted.
/// Both types and fuction could have const specifiers. The second argument is used by visitor to replaces AST node (y) if needed.
class Context;
class ASTSubquery;
class ASTFunction;
struct ASTTableExpression;
/** Replace subqueries that return exactly one row
* ("scalar" subqueries) to the corresponding constants.
......@@ -29,51 +27,25 @@ namespace DB
* Scalar subqueries are executed on the request-initializer server.
* The request is sent to remote servers with already substituted constants.
*/
class ExecuteScalarSubqueriesVisitor
class ExecuteScalarSubqueriesMatcher
{
public:
ExecuteScalarSubqueriesVisitor(const Context & context_, size_t subquery_depth_, std::ostream * ostr_ = nullptr)
: context(context_),
subquery_depth(subquery_depth_),
visit_depth(0),
ostr(ostr_)
{}
void visit(ASTPtr & ast) const
struct Data
{
if (!tryVisit<ASTSubquery>(ast) &&
!tryVisit<ASTTableExpression>(ast) &&
!tryVisit<ASTFunction>(ast))
visitChildren(ast);
}
private:
const Context & context;
size_t subquery_depth;
mutable size_t visit_depth;
std::ostream * ostr;
const Context & context;
size_t subquery_depth;
};
void visit(const ASTSubquery & subquery, ASTPtr & ast) const;
void visit(const ASTFunction & func, ASTPtr & ast) const;
void visit(const ASTTableExpression &, ASTPtr &) const;
static constexpr const char * label = "ExecuteScalarSubqueries";
void visitChildren(ASTPtr & ast) const
{
for (auto & child : ast->children)
visit(child);
}
static bool needChildVisit(ASTPtr & node, const ASTPtr &);
static std::vector<ASTPtr *> visit(ASTPtr & ast, Data & data);
template <typename T>
bool tryVisit(ASTPtr & ast) const
{
if (const T * t = typeid_cast<const T *>(ast.get()))
{
DumpASTNode dump(*ast, ostr, visit_depth, "executeScalarSubqueries");
visit(*t, ast);
return true;
}
return false;
}
private:
static void visit(const ASTSubquery & subquery, ASTPtr & ast, Data & data);
static std::vector<ASTPtr *> visit(const ASTFunction & func, ASTPtr & ast, Data & data);
};
using ExecuteScalarSubqueriesVisitor = InDepthNodeVisitor<ExecuteScalarSubqueriesMatcher, true>;
}
......@@ -31,8 +31,6 @@
#include <Interpreters/ExternalDictionaries.h>
#include <Interpreters/Set.h>
#include <Interpreters/Join.h>
#include <Interpreters/TranslateQualifiedNamesVisitor.h>
#include <Interpreters/ExecuteScalarSubqueriesVisitor.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/parseAggregateFunctionParameters.h>
......@@ -62,11 +60,9 @@
#include <Interpreters/DatabaseAndTableWithAlias.h>
#include <Interpreters/QueryNormalizer.h>
#include <Interpreters/QueryAliasesVisitor.h>
#include <Interpreters/ActionsVisitor.h>
#include <Interpreters/ExternalTablesVisitor.h>
#include <Interpreters/GlobalSubqueriesVisitor.h>
#include <Interpreters/ArrayJoinedColumnsVisitor.h>
#include <Interpreters/RequiredSourceColumnsVisitor.h>
namespace DB
......@@ -246,14 +242,14 @@ void ExpressionAnalyzer::analyzeAggregation()
void ExpressionAnalyzer::initGlobalSubqueriesAndExternalTables()
{
/// Adds existing external tables (not subqueries) to the external_tables dictionary.
ExternalTablesVisitor tables_visitor(context, external_tables);
tables_visitor.visit(query);
ExternalTablesVisitor::Data tables_data{context, external_tables};
ExternalTablesVisitor(tables_data).visit(query);
if (do_global)
{
GlobalSubqueriesVisitor subqueries_visitor(context, subquery_depth, isRemoteStorage(),
GlobalSubqueriesVisitor::Data subqueries_data(context, subquery_depth, isRemoteStorage(),
external_tables, subqueries_for_sets, has_global_subqueries);
subqueries_visitor.visit(query);
GlobalSubqueriesVisitor(subqueries_data).visit(query);
}
}
......@@ -1031,8 +1027,8 @@ void ExpressionAnalyzer::collectUsedColumns()
{
/// Nothing needs to be ignored for expressions in ARRAY JOIN.
NameSet empty;
RequiredSourceColumnsVisitor visitor(available_columns, required, empty, empty, empty);
visitor.visit(expressions[i]);
RequiredSourceColumnsVisitor::Data visitor_data{available_columns, required, empty, empty, empty};
RequiredSourceColumnsVisitor(visitor_data).visit(expressions[i]);
}
ignored.insert(expressions[i]->getAliasOrColumnName());
......@@ -1048,15 +1044,17 @@ void ExpressionAnalyzer::collectUsedColumns()
NameSet required_joined_columns;
for (const auto & left_key_ast : analyzedJoin().key_asts_left)
for (const auto & left_key_ast : syntax->analyzed_join.key_asts_left)
{
NameSet empty;
RequiredSourceColumnsVisitor columns_visitor(available_columns, required, ignored, empty, required_joined_columns);
columns_visitor.visit(left_key_ast);
RequiredSourceColumnsVisitor::Data columns_data{available_columns, required, ignored, empty, required_joined_columns};
ASTPtr tmp = left_key_ast;
RequiredSourceColumnsVisitor(columns_data).visit(tmp);
}
RequiredSourceColumnsVisitor columns_visitor(available_columns, required, ignored, available_joined_columns, required_joined_columns);
columns_visitor.visit(query);
RequiredSourceColumnsVisitor::Data columns_visitor_data{available_columns, required, ignored,
available_joined_columns, required_joined_columns};
RequiredSourceColumnsVisitor(columns_visitor_data).visit(query);
columns_added_by_join = analyzedJoin().available_joined_columns;
for (auto it = columns_added_by_join.begin(); it != columns_added_by_join.end();)
......
......@@ -4,49 +4,43 @@
#include <Parsers/IAST.h>
#include <Parsers/ASTIdentifier.h>
#include <Common/typeid_cast.h>
#include <Interpreters/InDepthNodeVisitor.h>
namespace DB
{
/// Finds in the query the usage of external tables (as table identifiers). Fills in external_tables.
class ExternalTablesVisitor
/// If node is ASTIdentifier try to extract external_storage.
class ExternalTablesMatcher
{
public:
ExternalTablesVisitor(const Context & context_, Tables & tables)
: context(context_),
external_tables(tables)
{}
void visit(ASTPtr & ast) const
struct Data
{
/// Traverse from the bottom. Intentionally go into subqueries.
for (auto & child : ast->children)
visit(child);
const Context & context;
Tables & external_tables;
};
tryVisit<ASTIdentifier>(ast);
}
static constexpr const char * label = "ExternalTables";
private:
const Context & context;
Tables & external_tables;
void visit(const ASTIdentifier & node, ASTPtr &) const
static std::vector<ASTPtr *> visit(ASTPtr & ast, Data & data)
{
if (node.special())
if (StoragePtr external_storage = context.tryGetExternalTable(node.name))
external_tables[node.name] = external_storage;
if (auto * t = typeid_cast<ASTIdentifier *>(ast.get()))
return visit(*t, ast, data);
return {};
}
template <typename T>
bool tryVisit(ASTPtr & ast) const
static bool needChildVisit(ASTPtr &, const ASTPtr &) { return true; }
private:
static std::vector<ASTPtr *> visit(const ASTIdentifier & node, ASTPtr &, Data & data)
{
if (const T * t = typeid_cast<const T *>(ast.get()))
{
visit(*t, ast);
return true;
}
return false;
if (node.special())
if (StoragePtr external_storage = data.context.tryGetExternalTable(node.name))
data.external_tables[node.name] = external_storage;
return {};
}
};
/// Finds in the query the usage of external tables. Fills in external_tables.
using ExternalTablesVisitor = InDepthNodeVisitor<ExternalTablesMatcher, false>;
}
......@@ -15,175 +15,171 @@
#include <Databases/IDatabase.h>
#include <Storages/StorageMemory.h>
#include <IO/WriteHelpers.h>
#include <Interpreters/InDepthNodeVisitor.h>
namespace DB
{
/// Visitors consist of functions with unified interface 'void visit(Casted & x, ASTPtr & y)', there x is y, successfully casted to Casted.
/// Both types and fuction could have const specifiers. The second argument is used by visitor to replaces AST node (y) if needed.
/// Converts GLOBAL subqueries to external tables; Puts them into the external_tables dictionary: name -> StoragePtr.
class GlobalSubqueriesVisitor
class GlobalSubqueriesMatcher
{
public:
GlobalSubqueriesVisitor(const Context & context_, size_t subquery_depth_, bool is_remote_,
Tables & tables, SubqueriesForSets & subqueries_for_sets_, bool & has_global_subqueries_)
: context(context_),
subquery_depth(subquery_depth_),
is_remote(is_remote_),
external_table_id(1),
external_tables(tables),
subqueries_for_sets(subqueries_for_sets_),
has_global_subqueries(has_global_subqueries_)
{}
void visit(ASTPtr & ast) const
struct Data
{
/// Recursive calls. We do not go into subqueries.
for (auto & child : ast->children)
if (!typeid_cast<ASTSelectQuery *>(child.get()))
visit(child);
/// Bottom-up actions.
if (tryVisit<ASTFunction>(ast) ||
tryVisit<ASTTablesInSelectQueryElement>(ast))
const Context & context;
size_t subquery_depth;
bool is_remote;
size_t external_table_id;
Tables & external_tables;
SubqueriesForSets & subqueries_for_sets;
bool & has_global_subqueries;
Data(const Context & context_, size_t subquery_depth_, bool is_remote_,
Tables & tables, SubqueriesForSets & subqueries_for_sets_, bool & has_global_subqueries_)
: context(context_),
subquery_depth(subquery_depth_),
is_remote(is_remote_),
external_table_id(1),
external_tables(tables),
subqueries_for_sets(subqueries_for_sets_),
has_global_subqueries(has_global_subqueries_)
{}
}
private:
const Context & context;
size_t subquery_depth;
bool is_remote;
mutable size_t external_table_id = 1;
Tables & external_tables;
SubqueriesForSets & subqueries_for_sets;
bool & has_global_subqueries;
/// GLOBAL IN
void visit(ASTFunction & func, ASTPtr &) const
{
if (func.name == "globalIn" || func.name == "globalNotIn")
{
addExternalStorage(func.arguments->children.at(1));
has_global_subqueries = true;
}
}
/// GLOBAL JOIN
void visit(ASTTablesInSelectQueryElement & table_elem, ASTPtr &) const
{
if (table_elem.table_join
&& static_cast<const ASTTableJoin &>(*table_elem.table_join).locality == ASTTableJoin::Locality::Global)
void addExternalStorage(ASTPtr & subquery_or_table_name_or_table_expression)
{
addExternalStorage(table_elem.table_expression);
has_global_subqueries = true;
}
}
template <typename T>
bool tryVisit(ASTPtr & ast) const
{
if (T * t = typeid_cast<T *>(ast.get()))
{
visit(*t, ast);
return true;
}
return false;
}
/** Initialize InterpreterSelectQuery for a subquery in the GLOBAL IN/JOIN section,
* create a temporary table of type Memory and store it in the external_tables dictionary.
*/
void addExternalStorage(ASTPtr & subquery_or_table_name_or_table_expression) const
{
/// With nondistributed queries, creating temporary tables does not make sense.
if (!is_remote)
return;
/// With nondistributed queries, creating temporary tables does not make sense.
if (!is_remote)
return;
ASTPtr subquery;
ASTPtr table_name;
ASTPtr subquery_or_table_name;
ASTPtr subquery;
ASTPtr table_name;
ASTPtr subquery_or_table_name;
if (typeid_cast<const ASTIdentifier *>(subquery_or_table_name_or_table_expression.get()))
{
table_name = subquery_or_table_name_or_table_expression;
subquery_or_table_name = table_name;
}
else if (auto ast_table_expr = typeid_cast<const ASTTableExpression *>(subquery_or_table_name_or_table_expression.get()))
{
if (ast_table_expr->database_and_table_name)
if (typeid_cast<const ASTIdentifier *>(subquery_or_table_name_or_table_expression.get()))
{
table_name = ast_table_expr->database_and_table_name;
table_name = subquery_or_table_name_or_table_expression;
subquery_or_table_name = table_name;
}
else if (ast_table_expr->subquery)
else if (auto ast_table_expr = typeid_cast<const ASTTableExpression *>(subquery_or_table_name_or_table_expression.get()))
{
if (ast_table_expr->database_and_table_name)
{
table_name = ast_table_expr->database_and_table_name;
subquery_or_table_name = table_name;
}
else if (ast_table_expr->subquery)
{
subquery = ast_table_expr->subquery;
subquery_or_table_name = subquery;
}
}
else if (typeid_cast<const ASTSubquery *>(subquery_or_table_name_or_table_expression.get()))
{
subquery = ast_table_expr->subquery;
subquery = subquery_or_table_name_or_table_expression;
subquery_or_table_name = subquery;
}
}
else if (typeid_cast<const ASTSubquery *>(subquery_or_table_name_or_table_expression.get()))
{
subquery = subquery_or_table_name_or_table_expression;
subquery_or_table_name = subquery;
}
if (!subquery_or_table_name)
throw Exception("Logical error: unknown AST element passed to ExpressionAnalyzer::addExternalStorage method",
ErrorCodes::LOGICAL_ERROR);
if (!subquery_or_table_name)
throw Exception("Logical error: unknown AST element passed to ExpressionAnalyzer::addExternalStorage method",
ErrorCodes::LOGICAL_ERROR);
if (table_name)
{
/// If this is already an external table, you do not need to add anything. Just remember its presence.
if (external_tables.end() != external_tables.find(static_cast<const ASTIdentifier &>(*table_name).name))
return;
}
if (table_name)
{
/// If this is already an external table, you do not need to add anything. Just remember its presence.
if (external_tables.end() != external_tables.find(static_cast<const ASTIdentifier &>(*table_name).name))
return;
}
/// Generate the name for the external table.
String external_table_name = "_data" + toString(external_table_id);
while (external_tables.count(external_table_name))
{
++external_table_id;
external_table_name = "_data" + toString(external_table_id);
}
/// Generate the name for the external table.
String external_table_name = "_data" + toString(external_table_id);
while (external_tables.count(external_table_name))
{
++external_table_id;
external_table_name = "_data" + toString(external_table_id);
}
auto interpreter = interpretSubquery(subquery_or_table_name, context, subquery_depth, {});
Block sample = interpreter->getSampleBlock();
NamesAndTypesList columns = sample.getNamesAndTypesList();
auto interpreter = interpretSubquery(subquery_or_table_name, context, subquery_depth, {});
StoragePtr external_storage = StorageMemory::create(external_table_name, ColumnsDescription{columns});
external_storage->startup();
Block sample = interpreter->getSampleBlock();
NamesAndTypesList columns = sample.getNamesAndTypesList();
/** We replace the subquery with the name of the temporary table.
* It is in this form, the request will go to the remote server.
* This temporary table will go to the remote server, and on its side,
* instead of doing a subquery, you just need to read it.
*/
StoragePtr external_storage = StorageMemory::create(external_table_name, ColumnsDescription{columns});
external_storage->startup();
auto database_and_table_name = createDatabaseAndTableNode("", external_table_name);
/** We replace the subquery with the name of the temporary table.
* It is in this form, the request will go to the remote server.
* This temporary table will go to the remote server, and on its side,
* instead of doing a subquery, you just need to read it.
if (auto ast_table_expr = typeid_cast<ASTTableExpression *>(subquery_or_table_name_or_table_expression.get()))
{
ast_table_expr->subquery.reset();
ast_table_expr->database_and_table_name = database_and_table_name;
ast_table_expr->children.clear();
ast_table_expr->children.emplace_back(database_and_table_name);
}
else
subquery_or_table_name_or_table_expression = database_and_table_name;
external_tables[external_table_name] = external_storage;
subqueries_for_sets[external_table_name].source = interpreter->execute().in;
subqueries_for_sets[external_table_name].table = external_storage;
/** NOTE If it was written IN tmp_table - the existing temporary (but not external) table,
* then a new temporary table will be created (for example, _data1),
* and the data will then be copied to it.
* Maybe this can be avoided.
*/
}
};
static constexpr const char * label = "GlobalSubqueries";
static std::vector<ASTPtr *> visit(ASTPtr & ast, Data & data)
{
if (auto * t = typeid_cast<ASTFunction *>(ast.get()))
visit(*t, ast, data);
if (auto * t = typeid_cast<ASTTablesInSelectQueryElement *>(ast.get()))
visit(*t, ast, data);
return {};
}
auto database_and_table_name = createDatabaseAndTableNode("", external_table_name);
static bool needChildVisit(ASTPtr &, const ASTPtr & child)
{
/// We do not go into subqueries.
if (typeid_cast<ASTSelectQuery *>(child.get()))
return false;
return true;
}
if (auto ast_table_expr = typeid_cast<ASTTableExpression *>(subquery_or_table_name_or_table_expression.get()))
private:
/// GLOBAL IN
static void visit(ASTFunction & func, ASTPtr &, Data & data)
{
if (func.name == "globalIn" || func.name == "globalNotIn")
{
ast_table_expr->subquery.reset();
ast_table_expr->database_and_table_name = database_and_table_name;
data.addExternalStorage(func.arguments->children[1]);
data.has_global_subqueries = true;
}
}
ast_table_expr->children.clear();
ast_table_expr->children.emplace_back(database_and_table_name);
/// GLOBAL JOIN
static void visit(ASTTablesInSelectQueryElement & table_elem, ASTPtr &, Data & data)
{
if (table_elem.table_join
&& static_cast<const ASTTableJoin &>(*table_elem.table_join).locality == ASTTableJoin::Locality::Global)
{
data.addExternalStorage(table_elem.table_expression);
data.has_global_subqueries = true;
}
else
subquery_or_table_name_or_table_expression = database_and_table_name;
external_tables[external_table_name] = external_storage;
subqueries_for_sets[external_table_name].source = interpreter->execute().in;
subqueries_for_sets[external_table_name].table = external_storage;
/** NOTE If it was written IN tmp_table - the existing temporary (but not external) table,
* then a new temporary table will be created (for example, _data1),
* and the data will then be copied to it.
* Maybe this can be avoided.
*/
}
};
/// Converts GLOBAL subqueries to external tables; Puts them into the external_tables dictionary: name -> StoragePtr.
using GlobalSubqueriesVisitor = InDepthNodeVisitor<GlobalSubqueriesMatcher, false>;
}
#pragma once
#include <vector>
#include <Parsers/DumpASTNode.h>
namespace DB
{
/// Visits AST tree in depth, call functions for nodes according to Matcher type data.
/// You need to define Data, label, visit() and needChildVisit() in Matcher class.
template <typename Matcher, bool _topToBottom>
class InDepthNodeVisitor
{
public:
using Data = typename Matcher::Data;
InDepthNodeVisitor(Data & data_, std::ostream * ostr_ = nullptr)
: data(data_),
visit_depth(0),
ostr(ostr_)
{}
void visit(ASTPtr & ast)
{
DumpASTNode dump(*ast, ostr, visit_depth, Matcher::label);
if constexpr (!_topToBottom)
visitChildren(ast);
/// It operates with ASTPtr * cause we may want to rewrite ASTPtr in visit().
std::vector<ASTPtr *> additional_nodes = Matcher::visit(ast, data);
/// visit additional nodes (ex. only part of children)
for (ASTPtr * node : additional_nodes)
visit(*node);
if constexpr (_topToBottom)
visitChildren(ast);
}
private:
Data & data;
size_t visit_depth;
std::ostream * ostr;
void visitChildren(ASTPtr & ast)
{
for (auto & child : ast->children)
if (Matcher::needChildVisit(ast, child))
visit(child);
}
};
}
......@@ -311,9 +311,10 @@ ASTs PredicateExpressionsOptimizer::getSelectQueryProjectionColumns(ASTPtr & ast
std::unordered_map<String, ASTPtr> aliases;
std::vector<DatabaseAndTableWithAlias> tables = getDatabaseAndTables(*select_query, context.getCurrentDatabase());
TranslateQualifiedNamesVisitor({}, tables).visit(ast);
QueryAliasesVisitor query_aliases_visitor(aliases);
query_aliases_visitor.visit(ast);
TranslateQualifiedNamesVisitor::Data qn_visitor_data{{}, tables};
TranslateQualifiedNamesVisitor(qn_visitor_data).visit(ast);
QueryAliasesVisitor::Data query_aliases_data{aliases};
QueryAliasesVisitor(query_aliases_data).visit(ast);
QueryNormalizer(ast, aliases, settings, {}, {}).perform();
for (const auto & projection_column : select_query->select_expression_list->children)
......
#include <ostream>
#include <sstream>
#include <Common/typeid_cast.h>
#include <Interpreters/QueryAliasesVisitor.h>
#include <Parsers/ASTTablesInSelectQuery.h>
......@@ -16,33 +17,62 @@ namespace ErrorCodes
extern const int MULTIPLE_EXPRESSIONS_FOR_ALIAS;
}
void QueryAliasesVisitor::visit(const ASTPtr & ast) const
static String wrongAliasMessage(const ASTPtr & ast, const ASTPtr & prev_ast, const String & alias)
{
/// Bottom-up traversal. We do not go into subqueries.
visitChildren(ast);
std::stringstream message;
message << "Different expressions with the same alias " << backQuoteIfNeed(alias) << ":" << std::endl;
formatAST(*ast, message, false, true);
message << std::endl << "and" << std::endl;
formatAST(*prev_ast, message, false, true);
message << std::endl;
return message.str();
}
if (!tryVisit<ASTSubquery>(ast))
{
DumpASTNode dump(*ast, ostr, visit_depth, "getQueryAliases");
visitOther(ast);
}
bool QueryAliasesMatcher::needChildVisit(ASTPtr & node, const ASTPtr &)
{
/// Don't descent into table functions and subqueries and special case for ArrayJoin.
if (typeid_cast<ASTTableExpression *>(node.get()) ||
typeid_cast<ASTSelectWithUnionQuery *>(node.get()) ||
typeid_cast<ASTArrayJoin *>(node.get()))
return false;
return true;
}
std::vector<ASTPtr *> QueryAliasesMatcher::visit(ASTPtr & ast, Data & data)
{
if (auto * t = typeid_cast<ASTSubquery *>(ast.get()))
return visit(*t, ast, data);
if (auto * t = typeid_cast<ASTArrayJoin *>(ast.get()))
return visit(*t, ast, data);
visitOther(ast, data);
return {};
}
/// The top-level aliases in the ARRAY JOIN section have a special meaning, we will not add them
/// (skip the expression list itself and its children).
void QueryAliasesVisitor::visit(const ASTArrayJoin &, const ASTPtr & ast) const
std::vector<ASTPtr *> QueryAliasesMatcher::visit(const ASTArrayJoin &, const ASTPtr & ast, Data & data)
{
visitOther(ast, data);
/// @warning It breaks botom-to-top order (childs processed after node here), could lead to some effects.
/// It's possible to add ast back to result vec to save order. It will need two phase ASTArrayJoin visit (setting phase in data).
std::vector<ASTPtr *> out;
for (auto & child1 : ast->children)
for (auto & child2 : child1->children)
for (auto & child3 : child2->children)
visit(child3);
out.push_back(&child3);
return out;
}
/// set unique aliases for all subqueries. this is needed, because:
/// 1) content of subqueries could change after recursive analysis, and auto-generated column names could become incorrect
/// 2) result of different scalar subqueries can be cached inside expressions compilation cache and must have different names
void QueryAliasesVisitor::visit(ASTSubquery & subquery, const ASTPtr & ast) const
std::vector<ASTPtr *> QueryAliasesMatcher::visit(ASTSubquery & subquery, const ASTPtr & ast, Data & data)
{
Aliases & aliases = data.aliases;
static std::atomic_uint64_t subquery_index = 0;
if (subquery.alias.empty())
......@@ -59,42 +89,22 @@ void QueryAliasesVisitor::visit(ASTSubquery & subquery, const ASTPtr & ast) cons
aliases[alias] = ast;
}
else
visitOther(ast);
visitOther(ast, data);
return {};
}
void QueryAliasesVisitor::visitOther(const ASTPtr & ast) const
void QueryAliasesMatcher::visitOther(const ASTPtr & ast, Data & data)
{
Aliases & aliases = data.aliases;
String alias = ast->tryGetAlias();
if (!alias.empty())
{
if (aliases.count(alias) && ast->getTreeHash() != aliases[alias]->getTreeHash())
throw Exception(wrongAliasMessage(ast, alias), ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS);
throw Exception(wrongAliasMessage(ast, aliases[alias], alias), ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS);
aliases[alias] = ast;
}
}
void QueryAliasesVisitor::visitChildren(const ASTPtr & ast) const
{
for (auto & child : ast->children)
{
/// Don't descent into table functions and subqueries and special case for ArrayJoin.
if (!tryVisit<ASTTableExpression>(ast) &&
!tryVisit<ASTSelectWithUnionQuery>(ast) &&
!tryVisit<ASTArrayJoin>(ast))
visit(child);
}
}
String QueryAliasesVisitor::wrongAliasMessage(const ASTPtr & ast, const String & alias) const
{
std::stringstream message;
message << "Different expressions with the same alias " << backQuoteIfNeed(alias) << ":" << std::endl;
formatAST(*ast, message, false, true);
message << std::endl << "and" << std::endl;
formatAST(*aliases[alias], message, false, true);
message << std::endl;
return message.str();
}
}
#pragma once
#include <Common/typeid_cast.h>
#include <Parsers/DumpASTNode.h>
#include <unordered_map>
#include <Common/typeid_cast.h>
#include <Interpreters/InDepthNodeVisitor.h>
namespace DB
{
......@@ -15,47 +13,27 @@ struct ASTArrayJoin;
using Aliases = std::unordered_map<String, ASTPtr>;
/// Visitors consist of functions with unified interface 'void visit(Casted & x, ASTPtr & y)', there x is y, successfully casted to Casted.
/// Both types and fuction could have const specifiers. The second argument is used by visitor to replaces AST node (y) if needed.
/// Visits AST nodes and collect their aliases in one map (with links to source nodes).
class QueryAliasesVisitor
/// Visits AST node to collect aliases.
class QueryAliasesMatcher
{
public:
QueryAliasesVisitor(Aliases & aliases_, std::ostream * ostr_ = nullptr)
: aliases(aliases_),
visit_depth(0),
ostr(ostr_)
{}
void visit(const ASTPtr & ast) const;
private:
Aliases & aliases;
mutable size_t visit_depth;
std::ostream * ostr;
struct Data
{
Aliases & aliases;
};
void visit(const ASTTableExpression &, const ASTPtr &) const {}
void visit(const ASTSelectWithUnionQuery &, const ASTPtr &) const {}
static constexpr const char * label = "QueryAliases";
void visit(ASTSubquery & subquery, const ASTPtr & ast) const;
void visit(const ASTArrayJoin &, const ASTPtr & ast) const;
void visitOther(const ASTPtr & ast) const;
void visitChildren(const ASTPtr & ast) const;
static std::vector<ASTPtr *> visit(ASTPtr & ast, Data & data);
static bool needChildVisit(ASTPtr & node, const ASTPtr & child);
template <typename T>
bool tryVisit(const ASTPtr & ast) const
{
if (T * t = typeid_cast<T *>(ast.get()))
{
DumpASTNode dump(*ast, ostr, visit_depth, "getQueryAliases");
visit(*t, ast);
return true;
}
return false;
}
String wrongAliasMessage(const ASTPtr & ast, const String & alias) const;
private:
static std::vector<ASTPtr *> visit(ASTSubquery & subquery, const ASTPtr & ast, Data & data);
static std::vector<ASTPtr *> visit(const ASTArrayJoin &, const ASTPtr & ast, Data & data);
static void visitOther(const ASTPtr & ast, Data & data);
};
/// Visits AST nodes and collect their aliases in one map (with links to source nodes).
using QueryAliasesVisitor = InDepthNodeVisitor<QueryAliasesMatcher, false>;
}
......@@ -18,25 +18,49 @@ namespace ErrorCodes
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
/// Visitors consist of functions with unified interface 'void visit(Casted & x, ASTPtr & y)', there x is y, successfully casted to Casted.
/// Both types and fuction could have const specifiers. The second argument is used by visitor to replaces AST node (y) if needed.
/** Get a set of necessary columns to read from the table.
* In this case, the columns specified in ignored_names are considered unnecessary. And the ignored_names parameter can be modified.
* The set of columns available_joined_columns are the columns available from JOIN, they are not needed for reading from the main table.
* Put in required_joined_columns the set of columns available from JOIN and needed.
*/
class RequiredSourceColumnsVisitor
class RequiredSourceColumnsMatcher
{
public:
RequiredSourceColumnsVisitor(const NameSet & available_columns_, NameSet & required_source_columns_, NameSet & ignored_names_,
const NameSet & available_joined_columns_, NameSet & required_joined_columns_)
: available_columns(available_columns_),
required_source_columns(required_source_columns_),
ignored_names(ignored_names_),
available_joined_columns(available_joined_columns_),
required_joined_columns(required_joined_columns_)
{}
struct Data
{
const NameSet & available_columns;
NameSet & required_source_columns;
NameSet & ignored_names;
const NameSet & available_joined_columns;
NameSet & required_joined_columns;
};
static constexpr const char * label = "RequiredSourceColumns";
static bool needChildVisit(ASTPtr & node, const ASTPtr & child)
{
/// We will not go to the ARRAY JOIN section, because we need to look at the names of non-ARRAY-JOIN columns.
/// There, `collectUsedColumns` will send us separately.
if (typeid_cast<ASTSelectQuery *>(child.get()) ||
typeid_cast<ASTArrayJoin *>(child.get()) ||
typeid_cast<ASTTableExpression *>(child.get()) ||
typeid_cast<ASTTableJoin *>(child.get()))
return false;
/// Processed. Do not need children.
if (typeid_cast<ASTIdentifier *>(node.get()))
return false;
if (auto * f = typeid_cast<ASTFunction *>(node.get()))
{
/// "indexHint" is a special function for index analysis. Everything that is inside it is not calculated. @sa KeyCondition
/// "lambda" visit children itself.
if (f->name == "indexHint" || f->name == "lambda")
return false;
}
return true;
}
/** Find all the identifiers in the query.
* We will use depth first search in AST.
......@@ -46,36 +70,34 @@ public:
* - there is some exception for the ARRAY JOIN clause (it has a slightly different identifiers);
* - we put identifiers available from JOIN in required_joined_columns.
*/
void visit(const ASTPtr & ast) const
static std::vector<ASTPtr *> visit(ASTPtr & ast, Data & data)
{
if (!tryVisit<ASTIdentifier>(ast) &&
!tryVisit<ASTFunction>(ast))
visitChildren(ast);
if (auto * t = typeid_cast<ASTIdentifier *>(ast.get()))
visit(*t, ast, data);
if (auto * t = typeid_cast<ASTFunction *>(ast.get()))
visit(*t, ast, data);
return {};
}
private:
const NameSet & available_columns;
NameSet & required_source_columns;
NameSet & ignored_names;
const NameSet & available_joined_columns;
NameSet & required_joined_columns;
void visit(const ASTIdentifier & node, const ASTPtr &) const
static void visit(const ASTIdentifier & node, const ASTPtr &, Data & data)
{
if (node.general()
&& !ignored_names.count(node.name)
&& !ignored_names.count(Nested::extractTableName(node.name)))
&& !data.ignored_names.count(node.name)
&& !data.ignored_names.count(Nested::extractTableName(node.name)))
{
if (!available_joined_columns.count(node.name)
|| available_columns.count(node.name)) /// Read column from left table if has.
required_source_columns.insert(node.name);
/// Read column from left table if has.
if (!data.available_joined_columns.count(node.name) || data.available_columns.count(node.name))
data.required_source_columns.insert(node.name);
else
required_joined_columns.insert(node.name);
data.required_joined_columns.insert(node.name);
}
}
void visit(const ASTFunction & node, const ASTPtr & ast) const
static void visit(const ASTFunction & node, const ASTPtr &, Data & data)
{
NameSet & ignored_names = data.ignored_names;
if (node.name == "lambda")
{
if (node.arguments->children.size() != 2)
......@@ -102,47 +124,16 @@ private:
}
}
visit(node.arguments->children.at(1));
/// @note It's a special case where we visit children inside the matcher, not in visitor.
visit(node.arguments->children[1], data);
for (size_t i = 0; i < added_ignored.size(); ++i)
ignored_names.erase(added_ignored[i]);
return;
}
/// A special function `indexHint`. Everything that is inside it is not calculated
/// (and is used only for index analysis, see KeyCondition).
if (node.name == "indexHint")
return;
visitChildren(ast);
}
void visitChildren(const ASTPtr & ast) const
{
for (auto & child : ast->children)
{
/** We will not go to the ARRAY JOIN section, because we need to look at the names of non-ARRAY-JOIN columns.
* There, `collectUsedColumns` will send us separately.
*/
if (!typeid_cast<const ASTSelectQuery *>(child.get())
&& !typeid_cast<const ASTArrayJoin *>(child.get())
&& !typeid_cast<const ASTTableExpression *>(child.get())
&& !typeid_cast<const ASTTableJoin *>(child.get()))
visit(child);
}
}
template <typename T>
bool tryVisit(const ASTPtr & ast) const
{
if (const T * t = typeid_cast<const T *>(ast.get()))
{
visit(*t, ast);
return true;
}
return false;
}
};
/// Get a set of necessary columns to read from the table.
using RequiredSourceColumnsVisitor = InDepthNodeVisitor<RequiredSourceColumnsMatcher, true>;
}
......@@ -46,7 +46,7 @@ namespace
{
using LogAST = DebugASTLog<false>; /// set to true to enable logs
using Aliases = std::unordered_map<String, ASTPtr>;
using Aliases = SyntaxAnalyzerResult::Aliases;
/// Add columns from storage to source_columns list.
void collectSourceColumns(ASTSelectQuery * select_query, const Context & context,
......@@ -134,8 +134,8 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze(
/// Creates a dictionary `aliases`: alias -> ASTPtr
{
LogAST log;
QueryAliasesVisitor query_aliases_visitor(result.aliases, log.stream());
query_aliases_visitor.visit(query);
QueryAliasesVisitor::Data query_aliases_data{result.aliases};
QueryAliasesVisitor(query_aliases_data, log.stream()).visit(query);
}
/// Common subexpression elimination. Rewrite rules.
......@@ -228,7 +228,8 @@ void translateQualifiedNames(ASTPtr & query, ASTSelectQuery * select_query,
std::vector<DatabaseAndTableWithAlias> tables = getDatabaseAndTables(*select_query, context.getCurrentDatabase());
LogAST log;
TranslateQualifiedNamesVisitor visitor(source_columns, tables, log.stream());
TranslateQualifiedNamesVisitor::Data visitor_data{source_columns, tables};
TranslateQualifiedNamesVisitor visitor(visitor_data, log.stream());
visitor.visit(query);
}
......@@ -341,8 +342,8 @@ void executeScalarSubqueries(ASTPtr & query, const ASTSelectQuery * select_query
if (!select_query)
{
ExecuteScalarSubqueriesVisitor visitor(context, subquery_depth, log.stream());
visitor.visit(query);
ExecuteScalarSubqueriesVisitor::Data visitor_data{context, subquery_depth};
ExecuteScalarSubqueriesVisitor(visitor_data, log.stream()).visit(query);
}
else
{
......@@ -352,8 +353,8 @@ void executeScalarSubqueries(ASTPtr & query, const ASTSelectQuery * select_query
if (!typeid_cast<const ASTTableExpression *>(child.get())
&& !typeid_cast<const ASTSelectQuery *>(child.get()))
{
ExecuteScalarSubqueriesVisitor visitor(context, subquery_depth, log.stream());
visitor.visit(child);
ExecuteScalarSubqueriesVisitor::Data visitor_data{context, subquery_depth};
ExecuteScalarSubqueriesVisitor(visitor_data, log.stream()).visit(child);
}
}
}
......@@ -696,10 +697,10 @@ void getArrayJoinedColumns(ASTPtr & query, SyntaxAnalyzerResult & result, const
}
{
ArrayJoinedColumnsVisitor visitor(result.array_join_name_to_alias,
result.array_join_alias_to_name,
result.array_join_result_to_source);
visitor.visit(query);
ArrayJoinedColumnsVisitor::Data visitor_data{result.array_join_name_to_alias,
result.array_join_alias_to_name,
result.array_join_result_to_source};
ArrayJoinedColumnsVisitor(visitor_data).visit(query);
}
/// If the result of ARRAY JOIN is not used, it is necessary to ARRAY-JOIN any column,
......
#include <Interpreters/TranslateQualifiedNamesVisitor.h>
#include <Common/typeid_cast.h>
#include <Core/Names.h>
#include <Parsers/ASTIdentifier.h>
......@@ -15,10 +16,45 @@ namespace DB
namespace ErrorCodes
{
extern const int UNKNOWN_IDENTIFIER;
extern const int UNKNOWN_ELEMENT_IN_AST;
extern const int LOGICAL_ERROR;
}
void TranslateQualifiedNamesVisitor::visit(ASTIdentifier & identifier, ASTPtr & ast, const DumpASTNode & dump) const
bool TranslateQualifiedNamesMatcher::needChildVisit(ASTPtr & node, const ASTPtr & child)
{
/// Do not go to FROM, JOIN, subqueries.
if (typeid_cast<ASTTableExpression *>(child.get()) ||
typeid_cast<ASTSelectWithUnionQuery *>(child.get()))
return false;
/// Processed nodes. Do not go into children.
if (typeid_cast<ASTIdentifier *>(node.get()) ||
typeid_cast<ASTQualifiedAsterisk *>(node.get()) ||
typeid_cast<ASTTableJoin *>(node.get()))
return false;
/// ASTSelectQuery + others
return true;
}
std::vector<ASTPtr *> TranslateQualifiedNamesMatcher::visit(ASTPtr & ast, Data & data)
{
if (auto * t = typeid_cast<ASTIdentifier *>(ast.get()))
return visit(*t, ast, data);
if (auto * t = typeid_cast<ASTQualifiedAsterisk *>(ast.get()))
return visit(*t, ast, data);
if (auto * t = typeid_cast<ASTTableJoin *>(ast.get()))
return visit(*t, ast, data);
if (auto * t = typeid_cast<ASTSelectQuery *>(ast.get()))
return visit(*t, ast, data);
return {};
}
std::vector<ASTPtr *> TranslateQualifiedNamesMatcher::visit(const ASTIdentifier & identifier, ASTPtr & ast, Data & data)
{
const NameSet & source_columns = data.source_columns;
const std::vector<DatabaseAndTableWithAlias> & tables = data.tables;
if (identifier.general())
{
/// Select first table name with max number of qualifiers which can be stripped.
......@@ -38,23 +74,23 @@ void TranslateQualifiedNamesVisitor::visit(ASTIdentifier & identifier, ASTPtr &
}
if (max_num_qualifiers_to_strip)
{
dump.print(String("stripIdentifier ") + identifier.name, max_num_qualifiers_to_strip);
stripIdentifier(ast, max_num_qualifiers_to_strip);
}
/// In case if column from the joined table are in source columns, change it's name to qualified.
if (best_table_pos && source_columns.count(ast->getColumnName()))
{
const DatabaseAndTableWithAlias & table = tables[best_table_pos];
table.makeQualifiedName(ast);
dump.print("makeQualifiedName", table.database + '.' + table.table + ' ' + ast->getColumnName());
}
}
return {};
}
void TranslateQualifiedNamesVisitor::visit(ASTQualifiedAsterisk &, ASTPtr & ast, const DumpASTNode &) const
std::vector<ASTPtr *> TranslateQualifiedNamesMatcher::visit(const ASTQualifiedAsterisk & , const ASTPtr & ast, Data & data)
{
const std::vector<DatabaseAndTableWithAlias> & tables = data.tables;
if (ast->children.size() != 1)
throw Exception("Logical error: qualified asterisk must have exactly one child", ErrorCodes::LOGICAL_ERROR);
......@@ -76,51 +112,40 @@ void TranslateQualifiedNamesVisitor::visit(ASTQualifiedAsterisk &, ASTPtr & ast,
if (!table_names.database.empty() &&
db_and_table.database == table_names.database &&
db_and_table.table == table_names.table)
return;
return {};
}
else if (num_components == 0)
{
if ((!table_names.table.empty() && db_and_table.table == table_names.table) ||
(!table_names.alias.empty() && db_and_table.table == table_names.alias))
return;
return {};
}
}
throw Exception("Unknown qualified identifier: " + ident->getAliasOrColumnName(), ErrorCodes::UNKNOWN_IDENTIFIER);
}
void TranslateQualifiedNamesVisitor::visit(ASTTableJoin & join, ASTPtr &, const DumpASTNode &) const
std::vector<ASTPtr *> TranslateQualifiedNamesMatcher::visit(ASTTableJoin & join, const ASTPtr & , Data &)
{
/// Don't translate on_expression here in order to resolve equation parts later.
std::vector<ASTPtr *> out;
if (join.using_expression_list)
visit(join.using_expression_list);
out.push_back(&join.using_expression_list);
return out;
}
void TranslateQualifiedNamesVisitor::visit(ASTSelectQuery & select, ASTPtr & ast, const DumpASTNode &) const
std::vector<ASTPtr *> TranslateQualifiedNamesMatcher::visit(ASTSelectQuery & select, const ASTPtr & , Data &)
{
/// If the WHERE clause or HAVING consists of a single quailified column, the reference must be translated not only in children,
/// but also in where_expression and having_expression.
std::vector<ASTPtr *> out;
if (select.prewhere_expression)
visit(select.prewhere_expression);
out.push_back(&select.prewhere_expression);
if (select.where_expression)
visit(select.where_expression);
out.push_back(&select.where_expression);
if (select.having_expression)
visit(select.having_expression);
visitChildren(ast);
}
void TranslateQualifiedNamesVisitor::visitChildren(ASTPtr & ast) const
{
for (auto & child : ast->children)
{
/// Do not go to FROM, JOIN, subqueries.
if (!typeid_cast<const ASTTableExpression *>(child.get())
&& !typeid_cast<const ASTSelectWithUnionQuery *>(child.get()))
{
visit(child);
}
}
out.push_back(&select.having_expression);
return out;
}
}
#pragma once
#include <memory>
#include <vector>
#include <Common/typeid_cast.h>
#include <Parsers/DumpASTNode.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTQualifiedAsterisk.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Interpreters/DatabaseAndTableWithAlias.h>
#include <Interpreters/InDepthNodeVisitor.h>
namespace DB
{
/// Visitors consist of functions with unified interface 'void visit(Casted & x, ASTPtr & y)', there x is y, successfully casted to Casted.
/// Both types and fuction could have const specifiers. The second argument is used by visitor to replaces AST node (y) if needed.
class ASTIdentifier;
class ASTQualifiedAsterisk;
struct ASTTableJoin;
class ASTSelectQuery;
/// It visits nodes, find columns (general identifiers and asterisks) and translate their names according to tables' names.
class TranslateQualifiedNamesVisitor
/// Visit one node for names qualification. @sa InDepthNodeVisitor.
class TranslateQualifiedNamesMatcher
{
public:
TranslateQualifiedNamesVisitor(const NameSet & source_columns_, const std::vector<DatabaseAndTableWithAlias> & tables_,
std::ostream * ostr_ = nullptr)
: source_columns(source_columns_),
tables(tables_),
visit_depth(0),
ostr(ostr_)
{}
void visit(ASTPtr & ast) const
struct Data
{
if (!tryVisit<ASTIdentifier>(ast) &&
!tryVisit<ASTQualifiedAsterisk>(ast) &&
!tryVisit<ASTTableJoin>(ast) &&
!tryVisit<ASTSelectQuery>(ast))
visitChildren(ast); /// default: do nothing, visit children
}
private:
const NameSet & source_columns;
const std::vector<DatabaseAndTableWithAlias> & tables;
mutable size_t visit_depth;
std::ostream * ostr;
const NameSet & source_columns;
const std::vector<DatabaseAndTableWithAlias> & tables;
};
void visit(ASTIdentifier & node, ASTPtr & ast, const DumpASTNode & dump) const;
void visit(ASTQualifiedAsterisk & node, ASTPtr & ast, const DumpASTNode & dump) const;
void visit(ASTTableJoin & node, ASTPtr & ast, const DumpASTNode & dump) const;
void visit(ASTSelectQuery & ast, ASTPtr &, const DumpASTNode & dump) const;
static constexpr const char * label = "TranslateQualifiedNames";
void visitChildren(ASTPtr &) const;
static std::vector<ASTPtr *> visit(ASTPtr & ast, Data & data);
static bool needChildVisit(ASTPtr & node, const ASTPtr & child);
template <typename T>
bool tryVisit(ASTPtr & ast) const
{
if (T * t = typeid_cast<T *>(ast.get()))
{
DumpASTNode dump(*ast, ostr, visit_depth, "translateQualifiedNames");
visit(*t, ast, dump);
return true;
}
return false;
}
private:
static std::vector<ASTPtr *> visit(const ASTIdentifier & node, ASTPtr & ast, Data &);
static std::vector<ASTPtr *> visit(const ASTQualifiedAsterisk & node, const ASTPtr & ast, Data &);
static std::vector<ASTPtr *> visit(ASTTableJoin & node, const ASTPtr & ast, Data &);
static std::vector<ASTPtr *> visit(ASTSelectQuery & node, const ASTPtr & ast, Data &);
};
/// Visits AST for names qualification.
/// It finds columns (general identifiers and asterisks) and translate their names according to tables' names.
using TranslateQualifiedNamesVisitor = InDepthNodeVisitor<TranslateQualifiedNamesMatcher, true>;
}
......@@ -196,9 +196,9 @@ void ASTAlterCommandList::formatImpl(const FormatSettings & settings, FormatStat
/** Get the text that identifies this element. */
String ASTAlterQuery::getID() const
String ASTAlterQuery::getID(char delim) const
{
return "AlterQuery_" + database + "_" + table;
return "AlterQuery" + (delim + database) + delim + table;
}
ASTPtr ASTAlterQuery::clone() const
......
......@@ -97,7 +97,7 @@ public:
/// To distinguish REPLACE and ATTACH PARTITION partition FROM db.table
bool replace = true;
String getID() const override { return "AlterCommand_" + std::to_string(static_cast<int>(type)); }
String getID(char delim) const override { return "AlterCommand" + (delim + std::to_string(static_cast<int>(type))); }
ASTPtr clone() const override;
......@@ -116,7 +116,7 @@ public:
children.push_back(command);
}
String getID() const override { return "AlterCommandList"; }
String getID(char) const override { return "AlterCommandList"; }
ASTPtr clone() const override;
......@@ -129,7 +129,7 @@ class ASTAlterQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnCl
public:
ASTAlterCommandList * command_list = nullptr;
String getID() const override;
String getID(char) const override;
ASTPtr clone() const override;
......
......@@ -12,7 +12,7 @@ public:
String column_name;
ASTPtr expression;
String getID() const override { return "Assignment_" + column_name; }
String getID(char delim) const override { return "Assignment" + (delim + column_name); }
ASTPtr clone() const override
{
......
......@@ -9,7 +9,7 @@ namespace DB
class ASTAsterisk : public IAST
{
public:
String getID() const override { return "Asterisk"; }
String getID(char) const override { return "Asterisk"; }
ASTPtr clone() const override;
void appendColumnName(WriteBuffer & ostr) const override;
......
......@@ -8,7 +8,7 @@ namespace DB
struct ASTCheckQuery : public ASTQueryWithTableAndOutput
{
/** Get the text that identifies this element. */
String getID() const override { return ("CheckQuery_" + database + "_" + table); }
String getID(char delim) const override { return "CheckQuery" + (delim + database) + delim + table; }
ASTPtr clone() const override
{
......
......@@ -17,7 +17,7 @@ public:
ASTPtr default_expression;
ASTPtr comment;
String getID() const override { return "ColumnDeclaration_" + name; }
String getID(char delim) const override { return "ColumnDeclaration" + (delim + name); }
ASTPtr clone() const override
{
......
......@@ -21,7 +21,7 @@ public:
IAST * sample_by = nullptr;
ASTSetQuery * settings = nullptr;
String getID() const override { return "Storage definition"; }
String getID(char) const override { return "Storage definition"; }
ASTPtr clone() const override
{
......@@ -99,7 +99,7 @@ public:
ASTSelectWithUnionQuery * select = nullptr;
/** Get the text that identifies this element. */
String getID() const override { return (attach ? "AttachQuery_" : "CreateQuery_") + database + "_" + table; }
String getID(char delim) const override { return (attach ? "AttachQuery" : "CreateQuery") + (delim + database) + delim + table; }
ASTPtr clone() const override
{
......
......@@ -10,14 +10,14 @@ namespace ErrorCodes
}
String ASTDropQuery::getID() const
String ASTDropQuery::getID(char delim) const
{
if (kind == ASTDropQuery::Kind::Drop)
return "DropQuery_" + database + "_" + table;
return "DropQuery" + (delim + database) + delim + table;
else if (kind == ASTDropQuery::Kind::Detach)
return "DetachQuery_" + database + "_" + table;
return "DetachQuery" + (delim + database) + delim + table;
else if (kind == ASTDropQuery::Kind::Truncate)
return "TruncateQuery_" + database + "_" + table;
return "TruncateQuery" + (delim + database) + delim + table;
else
throw Exception("Not supported kind of drop query.", ErrorCodes::SYNTAX_ERROR);
}
......
......@@ -23,7 +23,7 @@ public:
bool if_exists{false};
/** Get the text that identifies this element. */
String getID() const override;
String getID(char) const override;
ASTPtr clone() const override;
ASTPtr getRewrittenASTWithoutOnCluster(const std::string & new_database) const override
......
......@@ -20,7 +20,7 @@ public:
: kind(kind_)
{}
String getID() const override { return "Explain_" + toString(kind); }
String getID(char delim) const override { return "Explain" + (delim + toString(kind)); }
ASTPtr clone() const override { return std::make_shared<ASTExplainQuery>(*this); }
protected:
......
......@@ -11,7 +11,7 @@ namespace DB
class ASTExpressionList : public IAST
{
public:
String getID() const override { return "ExpressionList"; }
String getID(char) const override { return "ExpressionList"; }
ASTPtr clone() const override;
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
......
......@@ -36,9 +36,9 @@ void ASTFunction::appendColumnNameImpl(WriteBuffer & ostr) const
}
/** Get the text that identifies this element. */
String ASTFunction::getID() const
String ASTFunction::getID(char delim) const
{
return "Function_" + name;
return "Function" + (delim + name);
}
ASTPtr ASTFunction::clone() const
......
......@@ -19,7 +19,7 @@ public:
public:
/** Get text identifying the AST node. */
String getID() const override;
String getID(char delim) const override;
ASTPtr clone() const override;
......
......@@ -24,7 +24,7 @@ public:
: name(name_), kind(kind_) { range = StringRange(name.data(), name.data() + name.size()); }
/** Get the text that identifies this element. */
String getID() const override { return "Identifier_" + name; }
String getID(char delim) const override { return "Identifier" + (delim + name); }
ASTPtr clone() const override { return std::make_shared<ASTIdentifier>(*this); }
......
......@@ -27,7 +27,7 @@ public:
const char * end = nullptr;
/** Get the text that identifies this element. */
String getID() const override { return "InsertQuery_" + database + "_" + table; }
String getID(char delim) const override { return "InsertQuery" + (delim + database) + delim + table; }
ASTPtr clone() const override
{
......
......@@ -3,9 +3,9 @@
namespace DB
{
String ASTKillQueryQuery::getID() const
String ASTKillQueryQuery::getID(char delim) const
{
return "KillQueryQuery_" + (where_expression ? where_expression->getID() : "") + "_" + String(sync ? "SYNC" : "ASYNC");
return String("KillQueryQuery") + delim + (where_expression ? where_expression->getID() : "") + delim + String(sync ? "SYNC" : "ASYNC");
}
void ASTKillQueryQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
......
......@@ -20,7 +20,7 @@ public:
return clone;
}
String getID() const override;
String getID(char) const override;
void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
......
......@@ -18,7 +18,7 @@ public:
ASTLiteral(const Field & value_) : value(value_) {}
/** Get the text that identifies this element. */
String getID() const override { return "Literal_" + applyVisitor(FieldVisitorDump(), value); }
String getID(char delim) const override { return "Literal" + (delim + applyVisitor(FieldVisitorDump(), value)); }
ASTPtr clone() const override { return std::make_shared<ASTLiteral>(*this); }
......
......@@ -17,7 +17,7 @@ public:
ASTPtr type;
/** Get the text that identifies this element. */
String getID() const override { return "NameTypePair_" + name; }
String getID(char delim) const override { return "NameTypePair" + (delim + name); }
ASTPtr clone() const override
{
......
......@@ -21,8 +21,10 @@ public:
bool deduplicate;
/** Get the text that identifies this element. */
String getID() const override
{ return "OptimizeQuery_" + database + "_" + table + (final ? "_final" : "") + (deduplicate ? "_deduplicate" : ""); }
String getID(char delim) const override
{
return "OptimizeQuery" + (delim + database) + delim + table + (final ? "_final" : "") + (deduplicate ? "_deduplicate" : "");
}
ASTPtr clone() const override
{
......
......@@ -27,10 +27,7 @@ public:
{
}
String getID() const override
{
return "OrderByElement";
}
String getID(char) const override { return "OrderByElement"; }
ASTPtr clone() const override
{
......
......@@ -4,12 +4,12 @@
namespace DB
{
String ASTPartition::getID() const
String ASTPartition::getID(char delim) const
{
if (value)
return "Partition";
else
return "Partition_ID_" + id;
return "Partition_ID" + (delim + id);
}
ASTPtr ASTPartition::clone() const
......
......@@ -17,7 +17,7 @@ public:
String id;
String getID() const override;
String getID(char) const override;
ASTPtr clone() const override;
protected:
......
......@@ -12,7 +12,7 @@ namespace DB
class ASTQualifiedAsterisk : public IAST
{
public:
String getID() const override { return "QualifiedAsterisk"; }
String getID(char) const override { return "QualifiedAsterisk"; }
ASTPtr clone() const override
{
auto clone = std::make_shared<ASTQualifiedAsterisk>(*this);
......
......@@ -34,7 +34,7 @@ template <typename ASTIDAndQueryNames>
class ASTQueryWithOutputImpl : public ASTQueryWithOutput
{
public:
String getID() const override { return ASTIDAndQueryNames::ID; }
String getID(char) const override { return ASTIDAndQueryNames::ID; }
ASTPtr clone() const override
{
......
......@@ -30,7 +30,7 @@ template <typename AstIDAndQueryNames>
class ASTQueryWithTableAndOutputImpl : public ASTQueryWithTableAndOutput
{
public:
String getID() const override { return AstIDAndQueryNames::ID + ("_" + database) + "_" + table; }
String getID(char delim) const override { return AstIDAndQueryNames::ID + (delim + database) + delim + table; }
ASTPtr clone() const override
{
......
......@@ -29,7 +29,7 @@ public:
Elements elements;
/** Get the text that identifies this element. */
String getID() const override { return "Rename"; }
String getID(char) const override { return "Rename"; }
ASTPtr clone() const override
{
......
......@@ -28,7 +28,7 @@ public:
ASTSampleRatio(Rational & ratio_) : ratio(ratio_) {}
String getID() const override { return "SampleRatio_" + toString(ratio); }
String getID(char delim) const override { return "SampleRatio" + (delim + toString(ratio)); }
ASTPtr clone() const override { return std::make_shared<ASTSampleRatio>(*this); }
......
......@@ -16,7 +16,7 @@ class ASTSelectQuery : public IAST
{
public:
/** Get the text that identifies this element. */
String getID() const override { return "SelectQuery"; }
String getID(char) const override { return "SelectQuery"; }
ASTPtr clone() const override;
......
......@@ -12,7 +12,7 @@ namespace DB
class ASTSelectWithUnionQuery : public ASTQueryWithOutput
{
public:
String getID() const override { return "SelectWithUnionQuery"; }
String getID(char) const override { return "SelectWithUnionQuery"; }
ASTPtr clone() const override;
void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
......
......@@ -26,7 +26,7 @@ public:
Changes changes;
/** Get the text that identifies this element. */
String getID() const override { return "Set"; }
String getID(char) const override { return "Set"; }
ASTPtr clone() const override { return std::make_shared<ASTSetQuery>(*this); }
......
......@@ -21,7 +21,7 @@ public:
bool not_like{false};
/** Get the text that identifies this element. */
String getID() const override { return "ShowTables"; }
String getID(char) const override { return "ShowTables"; }
ASTPtr clone() const override
{
......
......@@ -13,7 +13,7 @@ class ASTSubquery : public ASTWithAlias
{
public:
/** Get the text that identifies this element. */
String getID() const override { return "Subquery"; }
String getID(char) const override { return "Subquery"; }
ASTPtr clone() const override
{
......
......@@ -51,7 +51,7 @@ public:
String target_database;
String target_table;
String getID() const override { return "SYSTEM query"; }
String getID(char) const override { return "SYSTEM query"; }
ASTPtr clone() const override { return std::make_shared<ASTSystemQuery>(*this); }
......
......@@ -53,7 +53,7 @@ struct ASTTableExpression : public IAST
ASTPtr sample_offset;
using IAST::IAST;
String getID() const override { return "TableExpression"; }
String getID(char) const override { return "TableExpression"; }
ASTPtr clone() const override;
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
};
......@@ -98,7 +98,7 @@ struct ASTTableJoin : public IAST
ASTPtr on_expression;
using IAST::IAST;
String getID() const override { return "TableJoin"; }
String getID(char) const override { return "TableJoin"; }
ASTPtr clone() const override;
void formatImplBeforeTable(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const;
......@@ -122,7 +122,7 @@ struct ASTArrayJoin : public IAST
ASTPtr expression_list;
using IAST::IAST;
String getID() const override { return "ArrayJoin"; }
String getID(char) const override { return "ArrayJoin"; }
ASTPtr clone() const override;
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
};
......@@ -139,7 +139,7 @@ struct ASTTablesInSelectQueryElement : public IAST
ASTPtr array_join; /// Arrays to JOIN.
using IAST::IAST;
String getID() const override { return "TablesInSelectQueryElement"; }
String getID(char) const override { return "TablesInSelectQueryElement"; }
ASTPtr clone() const override;
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
};
......@@ -149,7 +149,7 @@ struct ASTTablesInSelectQueryElement : public IAST
struct ASTTablesInSelectQuery : public IAST
{
using IAST::IAST;
String getID() const override { return "TablesInSelectQuery"; }
String getID(char) const override { return "TablesInSelectQuery"; }
ASTPtr clone() const override;
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
};
......
......@@ -15,7 +15,7 @@ public:
String database;
/** Get the text that identifies this element. */
String getID() const override { return "UseQuery_" + database; }
String getID(char delim) const override { return "UseQuery" + (delim + database); }
ASTPtr clone() const override { return std::make_shared<ASTUseQuery>(*this); }
......
......@@ -62,12 +62,7 @@ private:
size_t & visit_depth; /// shared with children
const char * label;
String nodeId() const
{
String id = ast.getID();
std::replace(id.begin(), id.end(), '_', ' ');
return id;
}
String nodeId() const { return ast.getID(' '); }
void printNode() const
{
......@@ -78,7 +73,7 @@ private:
print("alias", aslias, " ");
if (!ast.children.empty())
print("/", ast.children.size(), " "); /// slash is just a short name for 'children' here
print("children", ast.children.size(), " ");
}
};
......
......@@ -67,7 +67,7 @@ public:
}
/** Get the text that identifies this element. */
virtual String getID() const = 0;
virtual String getID(char delimiter = '_') const = 0;
ASTPtr ptr() { return shared_from_this(); }
......
......@@ -48,7 +48,7 @@ class ASTDescribeQuery : public ASTQueryWithOutput
public:
ASTPtr table_expression;
String getID() const override { return "DescribeQuery"; }
String getID(char) const override { return "DescribeQuery"; }
ASTPtr clone() const override
{
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册