Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
bbe0245b
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
bbe0245b
编写于
5月 13, 2020
作者:
D
Dmitry
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
changes after review #1
上级
259de4aa
变更
13
隐藏空白更改
内联
并排
Showing
13 changed file
with
66 addition
and
172 deletion
+66
-172
src/Interpreters/Aggregator.cpp
src/Interpreters/Aggregator.cpp
+1
-1
src/Interpreters/InterpreterSelectQuery.cpp
src/Interpreters/InterpreterSelectQuery.cpp
+22
-26
src/Interpreters/InterpreterSelectQuery.h
src/Interpreters/InterpreterSelectQuery.h
+5
-5
src/Processors/Transforms/AggregatingInOrderTransform.cpp
src/Processors/Transforms/AggregatingInOrderTransform.cpp
+0
-62
src/Processors/Transforms/AggregatingInOrderTransform.h
src/Processors/Transforms/AggregatingInOrderTransform.h
+11
-14
src/Processors/Transforms/TotalsHavingTransform.h
src/Processors/Transforms/TotalsHavingTransform.h
+2
-1
src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
+7
-42
src/Storages/ReadInOrderOptimizer.cpp
src/Storages/ReadInOrderOptimizer.cpp
+2
-2
src/Storages/ReadInOrderOptimizer.h
src/Storages/ReadInOrderOptimizer.h
+1
-1
src/Storages/SelectQueryInfo.h
src/Storages/SelectQueryInfo.h
+7
-10
src/Storages/StorageBuffer.cpp
src/Storages/StorageBuffer.cpp
+2
-2
src/Storages/StorageMaterializedView.cpp
src/Storages/StorageMaterializedView.cpp
+2
-2
src/Storages/StorageMerge.cpp
src/Storages/StorageMerge.cpp
+4
-4
未找到文件。
src/Interpreters/Aggregator.cpp
浏览文件 @
bbe0245b
...
@@ -540,7 +540,7 @@ void NO_INLINE Aggregator::executeOnIntervalWithoutKeyImpl(
...
@@ -540,7 +540,7 @@ void NO_INLINE Aggregator::executeOnIntervalWithoutKeyImpl(
for
(
AggregateFunctionInstruction
*
inst
=
aggregate_instructions
;
inst
->
that
;
++
inst
)
for
(
AggregateFunctionInstruction
*
inst
=
aggregate_instructions
;
inst
->
that
;
++
inst
)
{
{
if
(
inst
->
offsets
)
if
(
inst
->
offsets
)
inst
->
batch_that
->
addBatchSinglePlaceFromInterval
(
inst
->
offsets
[
row_begin
],
inst
->
offsets
[
static_cast
<
ssize_t
>
(
row_end
-
1
)
],
res
+
inst
->
state_offset
,
inst
->
batch_arguments
,
arena
);
inst
->
batch_that
->
addBatchSinglePlaceFromInterval
(
inst
->
offsets
[
row_begin
],
inst
->
offsets
[
row_end
-
1
],
res
+
inst
->
state_offset
,
inst
->
batch_arguments
,
arena
);
else
else
inst
->
batch_that
->
addBatchSinglePlaceFromInterval
(
row_begin
,
row_end
,
res
+
inst
->
state_offset
,
inst
->
batch_arguments
,
arena
);
inst
->
batch_that
->
addBatchSinglePlaceFromInterval
(
row_begin
,
row_end
,
res
+
inst
->
state_offset
,
inst
->
batch_arguments
,
arena
);
}
}
...
...
src/Interpreters/InterpreterSelectQuery.cpp
浏览文件 @
bbe0245b
...
@@ -831,7 +831,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
...
@@ -831,7 +831,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
if
(
!
expressions
.
second_stage
&&
!
expressions
.
need_aggregate
&&
!
expressions
.
hasHaving
())
if
(
!
expressions
.
second_stage
&&
!
expressions
.
need_aggregate
&&
!
expressions
.
hasHaving
())
{
{
if
(
expressions
.
has_order_by
)
if
(
expressions
.
has_order_by
)
executeOrder
(
pipeline
,
query_info
.
input_
sorting
_info
);
executeOrder
(
pipeline
,
query_info
.
input_
order
_info
);
if
(
expressions
.
has_order_by
&&
query
.
limitLength
())
if
(
expressions
.
has_order_by
&&
query
.
limitLength
())
executeDistinct
(
pipeline
,
false
,
expressions
.
selected_columns
);
executeDistinct
(
pipeline
,
false
,
expressions
.
selected_columns
);
...
@@ -1025,7 +1025,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
...
@@ -1025,7 +1025,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
if
(
!
expressions
.
first_stage
&&
!
expressions
.
need_aggregate
&&
!
(
query
.
group_by_with_totals
&&
!
aggregate_final
))
if
(
!
expressions
.
first_stage
&&
!
expressions
.
need_aggregate
&&
!
(
query
.
group_by_with_totals
&&
!
aggregate_final
))
executeMergeSorted
(
pipeline
);
executeMergeSorted
(
pipeline
);
else
/// Otherwise, just sort.
else
/// Otherwise, just sort.
executeOrder
(
pipeline
,
query_info
.
input_
sorting
_info
);
executeOrder
(
pipeline
,
query_info
.
input_
order
_info
);
}
}
/** Optimization - if there are several sources and there is LIMIT, then first apply the preliminary LIMIT,
/** Optimization - if there are several sources and there is LIMIT, then first apply the preliminary LIMIT,
...
@@ -1424,25 +1424,21 @@ void InterpreterSelectQuery::executeFetchColumns(
...
@@ -1424,25 +1424,21 @@ void InterpreterSelectQuery::executeFetchColumns(
query_info
.
prewhere_info
=
prewhere_info
;
query_info
.
prewhere_info
=
prewhere_info
;
/// Create optimizer with prepared actions.
/// Create optimizer with prepared actions.
/// Maybe we will need to calc input_
sorting
_info later, e.g. while reading from StorageMerge.
/// Maybe we will need to calc input_
order
_info later, e.g. while reading from StorageMerge.
if
(
analysis_result
.
optimize_read_in_order
)
if
(
analysis_result
.
optimize_read_in_order
||
analysis_result
.
optimize_aggregation_in_order
)
{
{
query_info
.
order_by_optimizer
=
std
::
make_shared
<
ReadInOrderOptimizer
>
(
if
(
analysis_result
.
optimize_read_in_order
)
analysis_result
.
order_by_elements_actions
,
query_info
.
order_optimizer
=
std
::
make_shared
<
ReadInOrderOptimizer
>
(
getSortDescription
(
query
,
*
context
),
analysis_result
.
order_by_elements_actions
,
query_info
.
syntax_analyzer_result
);
getSortDescription
(
query
,
*
context
),
query_info
.
syntax_analyzer_result
);
query_info
.
input_sorting_info
=
query_info
.
order_by_optimizer
->
getInputOrder
(
storage
);
else
}
query_info
.
order_optimizer
=
std
::
make_shared
<
ReadInOrderOptimizer
>
(
if
(
analysis_result
.
optimize_aggregation_in_order
)
{
query_info
.
group_by_optimizer
=
std
::
make_shared
<
ReadInOrderOptimizer
>
(
analysis_result
.
group_by_elements_actions
,
analysis_result
.
group_by_elements_actions
,
getSortDescriptionFromGroupBy
(
query
,
*
context
),
getSortDescriptionFromGroupBy
(
query
,
*
context
),
query_info
.
syntax_analyzer_result
);
query_info
.
syntax_analyzer_result
);
query_info
.
group_by_info
=
query_info
.
group_by
_optimizer
->
getInputOrder
(
storage
);
query_info
.
input_order_info
=
query_info
.
order
_optimizer
->
getInputOrder
(
storage
);
}
}
...
@@ -1647,7 +1643,7 @@ void InterpreterSelectQuery::executeWhere(QueryPipeline & pipeline, const Expres
...
@@ -1647,7 +1643,7 @@ void InterpreterSelectQuery::executeWhere(QueryPipeline & pipeline, const Expres
});
});
}
}
void
InterpreterSelectQuery
::
executeAggregation
(
Pipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
,
bool
overflow_row
,
bool
final
,
Input
Sorting
InfoPtr
/*group_by_info*/
)
void
InterpreterSelectQuery
::
executeAggregation
(
Pipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
,
bool
overflow_row
,
bool
final
,
Input
Order
InfoPtr
/*group_by_info*/
)
{
{
pipeline
.
transform
([
&
](
auto
&
stream
)
pipeline
.
transform
([
&
](
auto
&
stream
)
{
{
...
@@ -1711,7 +1707,7 @@ void InterpreterSelectQuery::executeAggregation(Pipeline & pipeline, const Expre
...
@@ -1711,7 +1707,7 @@ void InterpreterSelectQuery::executeAggregation(Pipeline & pipeline, const Expre
}
}
void
InterpreterSelectQuery
::
executeAggregation
(
QueryPipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
,
bool
overflow_row
,
bool
final
,
Input
Sorting
InfoPtr
group_by_info
)
void
InterpreterSelectQuery
::
executeAggregation
(
QueryPipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
,
bool
overflow_row
,
bool
final
,
Input
Order
InfoPtr
group_by_info
)
{
{
pipeline
.
addSimpleTransform
([
&
](
const
Block
&
header
)
pipeline
.
addSimpleTransform
([
&
](
const
Block
&
header
)
{
{
...
@@ -1801,7 +1797,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const
...
@@ -1801,7 +1797,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const
{
{
pipeline
.
addSimpleTransform
([
&
](
const
Block
&
header
)
pipeline
.
addSimpleTransform
([
&
](
const
Block
&
header
)
{
{
return
std
::
make_shared
<
Finalizing
InOrder
Transform
>
(
header
,
transform_params
);
return
std
::
make_shared
<
Finalizing
Simple
Transform
>
(
header
,
transform_params
);
});
});
}
}
...
@@ -2075,7 +2071,7 @@ void InterpreterSelectQuery::executeExpression(QueryPipeline & pipeline, const E
...
@@ -2075,7 +2071,7 @@ void InterpreterSelectQuery::executeExpression(QueryPipeline & pipeline, const E
});
});
}
}
void
InterpreterSelectQuery
::
executeOrder
(
Pipeline
&
pipeline
,
Input
Sorting
InfoPtr
input_sorting_info
)
void
InterpreterSelectQuery
::
executeOrder
(
Pipeline
&
pipeline
,
Input
Order
InfoPtr
input_sorting_info
)
{
{
auto
&
query
=
getSelectQuery
();
auto
&
query
=
getSelectQuery
();
SortDescription
output_order_descr
=
getSortDescription
(
query
,
*
context
);
SortDescription
output_order_descr
=
getSortDescription
(
query
,
*
context
);
...
@@ -2138,7 +2134,7 @@ void InterpreterSelectQuery::executeOrder(Pipeline & pipeline, InputSortingInfoP
...
@@ -2138,7 +2134,7 @@ void InterpreterSelectQuery::executeOrder(Pipeline & pipeline, InputSortingInfoP
}
}
}
}
void
InterpreterSelectQuery
::
executeOrderOptimized
(
QueryPipeline
&
pipeline
,
Input
Sorting
InfoPtr
input_sorting_info
,
UInt64
limit
,
SortDescription
&
output_order_descr
)
void
InterpreterSelectQuery
::
executeOrderOptimized
(
QueryPipeline
&
pipeline
,
Input
Order
InfoPtr
input_sorting_info
,
UInt64
limit
,
SortDescription
&
output_order_descr
)
{
{
const
Settings
&
settings
=
context
->
getSettingsRef
();
const
Settings
&
settings
=
context
->
getSettingsRef
();
...
@@ -2176,7 +2172,7 @@ void InterpreterSelectQuery::executeOrderOptimized(QueryPipeline & pipeline, Inp
...
@@ -2176,7 +2172,7 @@ void InterpreterSelectQuery::executeOrderOptimized(QueryPipeline & pipeline, Inp
}
}
}
}
void
InterpreterSelectQuery
::
executeOrder
(
QueryPipeline
&
pipeline
,
Input
Sorting
InfoPtr
input_sorting_info
)
void
InterpreterSelectQuery
::
executeOrder
(
QueryPipeline
&
pipeline
,
Input
Order
InfoPtr
input_sorting_info
)
{
{
auto
&
query
=
getSelectQuery
();
auto
&
query
=
getSelectQuery
();
SortDescription
output_order_descr
=
getSortDescription
(
query
,
*
context
);
SortDescription
output_order_descr
=
getSortDescription
(
query
,
*
context
);
...
@@ -2649,11 +2645,11 @@ void InterpreterSelectQuery::executeExtremes(QueryPipeline & pipeline)
...
@@ -2649,11 +2645,11 @@ void InterpreterSelectQuery::executeExtremes(QueryPipeline & pipeline)
void
InterpreterSelectQuery
::
executeSubqueriesInSetsAndJoins
(
Pipeline
&
pipeline
,
const
SubqueriesForSets
&
subqueries_for_sets
)
void
InterpreterSelectQuery
::
executeSubqueriesInSetsAndJoins
(
Pipeline
&
pipeline
,
const
SubqueriesForSets
&
subqueries_for_sets
)
{
{
/// Merge streams to one. Use MergeSorting if data was read in sorted order, Union otherwise.
/// Merge streams to one. Use MergeSorting if data was read in sorted order, Union otherwise.
if
(
query_info
.
input_
sorting
_info
)
if
(
query_info
.
input_
order
_info
)
{
{
if
(
pipeline
.
stream_with_non_joined_data
)
if
(
pipeline
.
stream_with_non_joined_data
)
throw
Exception
(
"Using read in order optimization, but has stream with non-joined data in pipeline"
,
ErrorCodes
::
LOGICAL_ERROR
);
throw
Exception
(
"Using read in order optimization, but has stream with non-joined data in pipeline"
,
ErrorCodes
::
LOGICAL_ERROR
);
executeMergeSorted
(
pipeline
,
query_info
.
input_
sorting
_info
->
order_key_prefix_descr
,
0
);
executeMergeSorted
(
pipeline
,
query_info
.
input_
order
_info
->
order_key_prefix_descr
,
0
);
}
}
else
else
executeUnion
(
pipeline
,
{});
executeUnion
(
pipeline
,
{});
...
@@ -2664,8 +2660,8 @@ void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(Pipeline & pipeline
...
@@ -2664,8 +2660,8 @@ void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(Pipeline & pipeline
void
InterpreterSelectQuery
::
executeSubqueriesInSetsAndJoins
(
QueryPipeline
&
pipeline
,
const
SubqueriesForSets
&
subqueries_for_sets
)
void
InterpreterSelectQuery
::
executeSubqueriesInSetsAndJoins
(
QueryPipeline
&
pipeline
,
const
SubqueriesForSets
&
subqueries_for_sets
)
{
{
if
(
query_info
.
input_
sorting
_info
)
if
(
query_info
.
input_
order
_info
)
executeMergeSorted
(
pipeline
,
query_info
.
input_
sorting
_info
->
order_key_prefix_descr
,
0
);
executeMergeSorted
(
pipeline
,
query_info
.
input_
order
_info
->
order_key_prefix_descr
,
0
);
const
Settings
&
settings
=
context
->
getSettingsRef
();
const
Settings
&
settings
=
context
->
getSettingsRef
();
...
...
src/Interpreters/InterpreterSelectQuery.h
浏览文件 @
bbe0245b
...
@@ -173,12 +173,12 @@ private:
...
@@ -173,12 +173,12 @@ private:
QueryPipeline
&
save_context_and_storage
);
QueryPipeline
&
save_context_and_storage
);
void
executeWhere
(
Pipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
,
bool
remove_filter
);
void
executeWhere
(
Pipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
,
bool
remove_filter
);
void
executeAggregation
(
Pipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
,
bool
overflow_row
,
bool
final
,
Input
Sorting
InfoPtr
group_by_info
);
void
executeAggregation
(
Pipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
,
bool
overflow_row
,
bool
final
,
Input
Order
InfoPtr
group_by_info
);
void
executeMergeAggregated
(
Pipeline
&
pipeline
,
bool
overflow_row
,
bool
final
);
void
executeMergeAggregated
(
Pipeline
&
pipeline
,
bool
overflow_row
,
bool
final
);
void
executeTotalsAndHaving
(
Pipeline
&
pipeline
,
bool
has_having
,
const
ExpressionActionsPtr
&
expression
,
bool
overflow_row
,
bool
final
);
void
executeTotalsAndHaving
(
Pipeline
&
pipeline
,
bool
has_having
,
const
ExpressionActionsPtr
&
expression
,
bool
overflow_row
,
bool
final
);
void
executeHaving
(
Pipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
);
void
executeHaving
(
Pipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
);
static
void
executeExpression
(
Pipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
);
static
void
executeExpression
(
Pipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
);
void
executeOrder
(
Pipeline
&
pipeline
,
Input
Sorting
InfoPtr
sorting_info
);
void
executeOrder
(
Pipeline
&
pipeline
,
Input
Order
InfoPtr
sorting_info
);
void
executeWithFill
(
Pipeline
&
pipeline
);
void
executeWithFill
(
Pipeline
&
pipeline
);
void
executeMergeSorted
(
Pipeline
&
pipeline
);
void
executeMergeSorted
(
Pipeline
&
pipeline
);
void
executePreLimit
(
Pipeline
&
pipeline
);
void
executePreLimit
(
Pipeline
&
pipeline
);
...
@@ -191,13 +191,13 @@ private:
...
@@ -191,13 +191,13 @@ private:
void
executeSubqueriesInSetsAndJoins
(
Pipeline
&
pipeline
,
const
std
::
unordered_map
<
String
,
SubqueryForSet
>
&
subqueries_for_sets
);
void
executeSubqueriesInSetsAndJoins
(
Pipeline
&
pipeline
,
const
std
::
unordered_map
<
String
,
SubqueryForSet
>
&
subqueries_for_sets
);
void
executeMergeSorted
(
Pipeline
&
pipeline
,
const
SortDescription
&
sort_description
,
UInt64
limit
);
void
executeMergeSorted
(
Pipeline
&
pipeline
,
const
SortDescription
&
sort_description
,
UInt64
limit
);
void
executeWhere
(
QueryPipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
,
bool
remove_filter
);
void
executeWhere
(
QueryPipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
,
bool
remove_filter
);
void
executeAggregation
(
QueryPipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
,
bool
overflow_row
,
bool
final
,
Input
Sorting
InfoPtr
group_by_info
);
void
executeAggregation
(
QueryPipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
,
bool
overflow_row
,
bool
final
,
Input
Order
InfoPtr
group_by_info
);
void
executeMergeAggregated
(
QueryPipeline
&
pipeline
,
bool
overflow_row
,
bool
final
);
void
executeMergeAggregated
(
QueryPipeline
&
pipeline
,
bool
overflow_row
,
bool
final
);
void
executeTotalsAndHaving
(
QueryPipeline
&
pipeline
,
bool
has_having
,
const
ExpressionActionsPtr
&
expression
,
bool
overflow_row
,
bool
final
);
void
executeTotalsAndHaving
(
QueryPipeline
&
pipeline
,
bool
has_having
,
const
ExpressionActionsPtr
&
expression
,
bool
overflow_row
,
bool
final
);
void
executeHaving
(
QueryPipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
);
void
executeHaving
(
QueryPipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
);
static
void
executeExpression
(
QueryPipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
);
static
void
executeExpression
(
QueryPipeline
&
pipeline
,
const
ExpressionActionsPtr
&
expression
);
void
executeOrder
(
QueryPipeline
&
pipeline
,
Input
Sorting
InfoPtr
sorting_info
);
void
executeOrder
(
QueryPipeline
&
pipeline
,
Input
Order
InfoPtr
sorting_info
);
void
executeOrderOptimized
(
QueryPipeline
&
pipeline
,
Input
Sorting
InfoPtr
sorting_info
,
UInt64
limit
,
SortDescription
&
output_order_descr
);
void
executeOrderOptimized
(
QueryPipeline
&
pipeline
,
Input
Order
InfoPtr
sorting_info
,
UInt64
limit
,
SortDescription
&
output_order_descr
);
void
executeWithFill
(
QueryPipeline
&
pipeline
);
void
executeWithFill
(
QueryPipeline
&
pipeline
);
void
executeMergeSorted
(
QueryPipeline
&
pipeline
);
void
executeMergeSorted
(
QueryPipeline
&
pipeline
);
void
executePreLimit
(
QueryPipeline
&
pipeline
,
bool
do_not_skip_offset
);
void
executePreLimit
(
QueryPipeline
&
pipeline
,
bool
do_not_skip_offset
);
...
...
src/Processors/Transforms/AggregatingInOrderTransform.cpp
浏览文件 @
bbe0245b
#include <Processors/Transforms/AggregatingInOrderTransform.h>
#include <Processors/Transforms/AggregatingInOrderTransform.h>
#include <Processors/Transforms/TotalsHavingTransform.h>
#include <DataTypes/DataTypeLowCardinality.h>
#include <DataTypes/DataTypeLowCardinality.h>
namespace
DB
namespace
DB
{
{
...
@@ -250,65 +248,5 @@ void AggregatingInOrderTransform::generate()
...
@@ -250,65 +248,5 @@ void AggregatingInOrderTransform::generate()
need_generate
=
false
;
need_generate
=
false
;
}
}
FinalizingInOrderTransform
::
FinalizingInOrderTransform
(
Block
header
,
AggregatingTransformParamsPtr
params_
)
:
IProcessor
({
std
::
move
(
header
)},
{
params_
->
getHeader
(
true
)})
{
}
FinalizingInOrderTransform
::~
FinalizingInOrderTransform
()
=
default
;
void
FinalizingInOrderTransform
::
consume
(
Chunk
chunk
)
{
finalizeChunk
(
chunk
);
current_chunk
=
std
::
move
(
chunk
);
}
void
FinalizingInOrderTransform
::
work
()
{
consume
(
std
::
move
(
current_chunk
));
}
IProcessor
::
Status
FinalizingInOrderTransform
::
prepare
()
{
auto
&
output
=
outputs
.
front
();
auto
&
input
=
inputs
.
back
();
/// Check can output.
if
(
output
.
isFinished
())
{
input
.
close
();
return
Status
::
Finished
;
}
if
(
!
output
.
canPush
())
{
input
.
setNotNeeded
();
return
Status
::
PortFull
;
}
if
(
input
.
isFinished
())
{
output
.
push
(
std
::
move
(
current_chunk
));
output
.
finish
();
return
Status
::
Finished
;
}
if
(
!
current_chunk
.
empty
())
{
output
.
push
(
std
::
move
(
current_chunk
));
current_chunk
.
clear
();
return
Status
::
Ready
;
}
if
(
!
input
.
hasData
())
{
input
.
setNeeded
();
return
Status
::
NeedData
;
}
current_chunk
=
input
.
pull
(
true
);
return
Status
::
Ready
;
}
}
}
src/Processors/Transforms/AggregatingInOrderTransform.h
浏览文件 @
bbe0245b
...
@@ -2,7 +2,9 @@
...
@@ -2,7 +2,9 @@
#include <Core/SortDescription.h>
#include <Core/SortDescription.h>
#include <Interpreters/Aggregator.h>
#include <Interpreters/Aggregator.h>
#include <Processors/ISimpleTransform.h>
#include <Processors/Transforms/AggregatingTransform.h>
#include <Processors/Transforms/AggregatingTransform.h>
#include <Processors/Transforms/TotalsHavingTransform.h>
namespace
DB
namespace
DB
{
{
...
@@ -58,24 +60,19 @@ private:
...
@@ -58,24 +60,19 @@ private:
};
};
class
Finalizing
InOrderTransform
:
public
IProcessor
class
Finalizing
SimpleTransform
:
public
ISimpleTransform
{
{
public:
public:
FinalizingInOrderTransform
(
Block
header
,
AggregatingTransformParamsPtr
params
);
FinalizingSimpleTransform
(
Block
header
,
AggregatingTransformParamsPtr
params
)
:
ISimpleTransform
({
std
::
move
(
header
)},
{
params
->
getHeader
(
true
)},
true
)
{}
~
FinalizingInOrderTransform
()
override
;
void
transform
(
Chunk
&
chunk
)
override
{
finalizeChunk
(
chunk
);
}
String
getName
()
const
override
{
return
"FinalizingInOrderTransform"
;
}
String
getName
()
const
override
{
return
"FinalizingSimpleTransform"
;
}
};
/// TODO Simplify prepare
Status
prepare
()
override
;
void
work
()
override
;
void
consume
(
Chunk
chunk
);
private:
Chunk
current_chunk
;
Logger
*
log
=
&
Logger
::
get
(
"FinalizingInOrderTransform"
);
};
}
}
src/Processors/Transforms/TotalsHavingTransform.h
浏览文件 @
bbe0245b
#
include <Processors/ISimpleTransform.h>
#
pragma once
#include <Processors/ISimpleTransform.h>
#include <Common/Arena.h>
#include <Common/Arena.h>
namespace
DB
namespace
DB
...
...
src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
浏览文件 @
bbe0245b
...
@@ -625,30 +625,9 @@ Pipes MergeTreeDataSelectExecutor::readFromParts(
...
@@ -625,30 +625,9 @@ Pipes MergeTreeDataSelectExecutor::readFromParts(
settings
,
settings
,
reader_settings
);
reader_settings
);
}
}
else
if
(
settings
.
optimize_read_in_order
&&
query_info
.
input_sorting
_info
)
else
if
(
(
settings
.
optimize_read_in_order
||
settings
.
optimize_aggregation_in_order
)
&&
query_info
.
input_order
_info
)
{
{
size_t
prefix_size
=
query_info
.
input_sorting_info
->
order_key_prefix_descr
.
size
();
size_t
prefix_size
=
query_info
.
input_order_info
->
order_key_prefix_descr
.
size
();
auto
order_key_prefix_ast
=
data
.
sorting_key_expr_ast
->
clone
();
order_key_prefix_ast
->
children
.
resize
(
prefix_size
);
auto
syntax_result
=
SyntaxAnalyzer
(
context
).
analyze
(
order_key_prefix_ast
,
data
.
getColumns
().
getAllPhysical
());
auto
sorting_key_prefix_expr
=
ExpressionAnalyzer
(
order_key_prefix_ast
,
syntax_result
,
context
).
getActions
(
false
);
res
=
spreadMarkRangesAmongStreamsWithOrder
(
std
::
move
(
parts_with_ranges
),
num_streams
,
column_names_to_read
,
max_block_size
,
settings
.
use_uncompressed_cache
,
query_info
,
sorting_key_prefix_expr
,
virt_column_names
,
settings
,
reader_settings
);
}
else
if
(
settings
.
optimize_aggregation_in_order
&&
query_info
.
group_by_info
)
{
size_t
prefix_size
=
query_info
.
group_by_info
->
order_key_prefix_descr
.
size
();
auto
order_key_prefix_ast
=
data
.
sorting_key_expr_ast
->
clone
();
auto
order_key_prefix_ast
=
data
.
sorting_key_expr_ast
->
clone
();
order_key_prefix_ast
->
children
.
resize
(
prefix_size
);
order_key_prefix_ast
->
children
.
resize
(
prefix_size
);
...
@@ -855,8 +834,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
...
@@ -855,8 +834,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
const
MergeTreeReaderSettings
&
reader_settings
)
const
const
MergeTreeReaderSettings
&
reader_settings
)
const
{
{
size_t
sum_marks
=
0
;
size_t
sum_marks
=
0
;
const
InputSortingInfoPtr
&
input_sorting_info
=
query_info
.
input_sorting_info
;
const
InputOrderInfoPtr
&
input_order_info
=
query_info
.
input_order_info
;
const
InputSortingInfoPtr
&
group_by_info
=
query_info
.
group_by_info
;
size_t
adaptive_parts
=
0
;
size_t
adaptive_parts
=
0
;
std
::
vector
<
size_t
>
sum_marks_in_parts
(
parts
.
size
());
std
::
vector
<
size_t
>
sum_marks_in_parts
(
parts
.
size
());
...
@@ -1000,13 +978,9 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
...
@@ -1000,13 +978,9 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
}
}
parts
.
emplace_back
(
part
);
parts
.
emplace_back
(
part
);
}
}
/// TODO Better code
ranges_to_get_from_part
=
split_ranges
(
ranges_to_get_from_part
,
input_order_info
->
direction
);
if
(
group_by_info
)
ranges_to_get_from_part
=
split_ranges
(
ranges_to_get_from_part
,
group_by_info
->
direction
);
else
ranges_to_get_from_part
=
split_ranges
(
ranges_to_get_from_part
,
input_sorting_info
->
direction
);
if
(
group_by_info
||
input_sorting
_info
->
direction
==
1
)
if
(
input_order
_info
->
direction
==
1
)
{
{
pipes
.
emplace_back
(
std
::
make_shared
<
MergeTreeSelectProcessor
>
(
pipes
.
emplace_back
(
std
::
make_shared
<
MergeTreeSelectProcessor
>
(
data
,
part
.
data_part
,
max_block_size
,
settings
.
preferred_block_size_bytes
,
data
,
part
.
data_part
,
max_block_size
,
settings
.
preferred_block_size_bytes
,
...
@@ -1029,17 +1003,8 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
...
@@ -1029,17 +1003,8 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
if
(
pipes
.
size
()
>
1
)
if
(
pipes
.
size
()
>
1
)
{
{
SortDescription
sort_description
;
SortDescription
sort_description
;
/// TODO Better code
for
(
size_t
j
=
0
;
j
<
input_order_info
->
order_key_prefix_descr
.
size
();
++
j
)
if
(
group_by_info
)
sort_description
.
emplace_back
(
data
.
sorting_key_columns
[
j
],
input_order_info
->
direction
,
1
);
{
for
(
size_t
j
=
0
;
j
<
group_by_info
->
order_key_prefix_descr
.
size
();
++
j
)
sort_description
.
emplace_back
(
data
.
sorting_key_columns
[
j
],
group_by_info
->
direction
,
1
);
}
else
{
for
(
size_t
j
=
0
;
j
<
input_sorting_info
->
order_key_prefix_descr
.
size
();
++
j
)
sort_description
.
emplace_back
(
data
.
sorting_key_columns
[
j
],
input_sorting_info
->
direction
,
1
);
}
/// Project input columns to drop columns from sorting_key_prefix_expr
/// Project input columns to drop columns from sorting_key_prefix_expr
/// to allow execute the same expression later.
/// to allow execute the same expression later.
...
...
src/Storages/ReadInOrderOptimizer.cpp
浏览文件 @
bbe0245b
...
@@ -30,7 +30,7 @@ ReadInOrderOptimizer::ReadInOrderOptimizer(
...
@@ -30,7 +30,7 @@ ReadInOrderOptimizer::ReadInOrderOptimizer(
forbidden_columns
.
insert
(
elem
.
first
);
forbidden_columns
.
insert
(
elem
.
first
);
}
}
Input
Sorting
InfoPtr
ReadInOrderOptimizer
::
getInputOrder
(
const
StoragePtr
&
storage
)
const
Input
Order
InfoPtr
ReadInOrderOptimizer
::
getInputOrder
(
const
StoragePtr
&
storage
)
const
{
{
Names
sorting_key_columns
;
Names
sorting_key_columns
;
if
(
const
auto
*
merge_tree
=
dynamic_cast
<
const
MergeTreeData
*>
(
storage
.
get
()))
if
(
const
auto
*
merge_tree
=
dynamic_cast
<
const
MergeTreeData
*>
(
storage
.
get
()))
...
@@ -122,7 +122,7 @@ InputSortingInfoPtr ReadInOrderOptimizer::getInputOrder(const StoragePtr & stora
...
@@ -122,7 +122,7 @@ InputSortingInfoPtr ReadInOrderOptimizer::getInputOrder(const StoragePtr & stora
if
(
order_key_prefix_descr
.
empty
())
if
(
order_key_prefix_descr
.
empty
())
return
{};
return
{};
return
std
::
make_shared
<
Input
Sorting
Info
>
(
std
::
move
(
order_key_prefix_descr
),
read_direction
);
return
std
::
make_shared
<
Input
Order
Info
>
(
std
::
move
(
order_key_prefix_descr
),
read_direction
);
}
}
}
}
src/Storages/ReadInOrderOptimizer.h
浏览文件 @
bbe0245b
...
@@ -20,7 +20,7 @@ public:
...
@@ -20,7 +20,7 @@ public:
const
SortDescription
&
required_sort_description
,
const
SortDescription
&
required_sort_description
,
const
SyntaxAnalyzerResultPtr
&
syntax_result
);
const
SyntaxAnalyzerResultPtr
&
syntax_result
);
Input
Sorting
InfoPtr
getInputOrder
(
const
StoragePtr
&
storage
)
const
;
Input
Order
InfoPtr
getInputOrder
(
const
StoragePtr
&
storage
)
const
;
private:
private:
/// Actions for every element of order expression to analyze functions for monotonicity
/// Actions for every element of order expression to analyze functions for monotonicity
...
...
src/Storages/SelectQueryInfo.h
浏览文件 @
bbe0245b
...
@@ -36,25 +36,25 @@ struct FilterInfo
...
@@ -36,25 +36,25 @@ struct FilterInfo
bool
do_remove_column
=
false
;
bool
do_remove_column
=
false
;
};
};
struct
Input
Sorting
Info
struct
Input
Order
Info
{
{
SortDescription
order_key_prefix_descr
;
SortDescription
order_key_prefix_descr
;
int
direction
;
int
direction
;
Input
Sorting
Info
(
const
SortDescription
&
order_key_prefix_descr_
,
int
direction_
)
Input
Order
Info
(
const
SortDescription
&
order_key_prefix_descr_
,
int
direction_
)
:
order_key_prefix_descr
(
order_key_prefix_descr_
),
direction
(
direction_
)
{}
:
order_key_prefix_descr
(
order_key_prefix_descr_
),
direction
(
direction_
)
{}
bool
operator
==
(
const
Input
Sorting
Info
&
other
)
const
bool
operator
==
(
const
Input
Order
Info
&
other
)
const
{
{
return
order_key_prefix_descr
==
other
.
order_key_prefix_descr
&&
direction
==
other
.
direction
;
return
order_key_prefix_descr
==
other
.
order_key_prefix_descr
&&
direction
==
other
.
direction
;
}
}
bool
operator
!=
(
const
Input
Sorting
Info
&
other
)
const
{
return
!
(
*
this
==
other
);
}
bool
operator
!=
(
const
Input
Order
Info
&
other
)
const
{
return
!
(
*
this
==
other
);
}
};
};
using
PrewhereInfoPtr
=
std
::
shared_ptr
<
PrewhereInfo
>
;
using
PrewhereInfoPtr
=
std
::
shared_ptr
<
PrewhereInfo
>
;
using
FilterInfoPtr
=
std
::
shared_ptr
<
FilterInfo
>
;
using
FilterInfoPtr
=
std
::
shared_ptr
<
FilterInfo
>
;
using
Input
SortingInfoPtr
=
std
::
shared_ptr
<
const
InputSorting
Info
>
;
using
Input
OrderInfoPtr
=
std
::
shared_ptr
<
const
InputOrder
Info
>
;
struct
SyntaxAnalyzerResult
;
struct
SyntaxAnalyzerResult
;
using
SyntaxAnalyzerResultPtr
=
std
::
shared_ptr
<
const
SyntaxAnalyzerResult
>
;
using
SyntaxAnalyzerResultPtr
=
std
::
shared_ptr
<
const
SyntaxAnalyzerResult
>
;
...
@@ -75,12 +75,9 @@ struct SelectQueryInfo
...
@@ -75,12 +75,9 @@ struct SelectQueryInfo
PrewhereInfoPtr
prewhere_info
;
PrewhereInfoPtr
prewhere_info
;
ReadInOrderOptimizerPtr
order_by_optimizer
;
ReadInOrderOptimizerPtr
order_optimizer
;
ReadInOrderOptimizerPtr
group_by_optimizer
;
/// We can modify it while reading from storage
/// We can modify it while reading from storage
mutable
InputSortingInfoPtr
input_sorting_info
;
mutable
InputOrderInfoPtr
input_order_info
;
InputSortingInfoPtr
group_by_info
;
/// Prepared sets are used for indices by storage engine.
/// Prepared sets are used for indices by storage engine.
/// Example: x IN (1, 2, 3)
/// Example: x IN (1, 2, 3)
...
...
src/Storages/StorageBuffer.cpp
浏览文件 @
bbe0245b
...
@@ -171,8 +171,8 @@ Pipes StorageBuffer::read(
...
@@ -171,8 +171,8 @@ Pipes StorageBuffer::read(
if
(
dst_has_same_structure
)
if
(
dst_has_same_structure
)
{
{
if
(
query_info
.
order_
by_
optimizer
)
if
(
query_info
.
order_optimizer
)
query_info
.
input_
sorting_info
=
query_info
.
order_by
_optimizer
->
getInputOrder
(
destination
);
query_info
.
input_
order_info
=
query_info
.
order
_optimizer
->
getInputOrder
(
destination
);
/// The destination table has the same structure of the requested columns and we can simply read blocks from there.
/// The destination table has the same structure of the requested columns and we can simply read blocks from there.
pipes_from_dst
=
destination
->
read
(
column_names
,
query_info
,
context
,
processed_stage
,
max_block_size
,
num_streams
);
pipes_from_dst
=
destination
->
read
(
column_names
,
query_info
,
context
,
processed_stage
,
max_block_size
,
num_streams
);
...
...
src/Storages/StorageMaterializedView.cpp
浏览文件 @
bbe0245b
...
@@ -180,8 +180,8 @@ Pipes StorageMaterializedView::read(
...
@@ -180,8 +180,8 @@ Pipes StorageMaterializedView::read(
auto
lock
=
storage
->
lockStructureForShare
(
auto
lock
=
storage
->
lockStructureForShare
(
false
,
context
.
getCurrentQueryId
(),
context
.
getSettingsRef
().
lock_acquire_timeout
);
false
,
context
.
getCurrentQueryId
(),
context
.
getSettingsRef
().
lock_acquire_timeout
);
if
(
query_info
.
order_
by_
optimizer
)
if
(
query_info
.
order_optimizer
)
query_info
.
input_
sorting_info
=
query_info
.
order_by
_optimizer
->
getInputOrder
(
storage
);
query_info
.
input_
order_info
=
query_info
.
order
_optimizer
->
getInputOrder
(
storage
);
Pipes
pipes
=
storage
->
read
(
column_names
,
query_info
,
context
,
processed_stage
,
max_block_size
,
num_streams
);
Pipes
pipes
=
storage
->
read
(
column_names
,
query_info
,
context
,
processed_stage
,
max_block_size
,
num_streams
);
...
...
src/Storages/StorageMerge.cpp
浏览文件 @
bbe0245b
...
@@ -177,12 +177,12 @@ Pipes StorageMerge::read(
...
@@ -177,12 +177,12 @@ Pipes StorageMerge::read(
num_streams
*=
num_streams_multiplier
;
num_streams
*=
num_streams_multiplier
;
size_t
remaining_streams
=
num_streams
;
size_t
remaining_streams
=
num_streams
;
Input
Sorting
InfoPtr
input_sorting_info
;
Input
Order
InfoPtr
input_sorting_info
;
if
(
query_info
.
order_
by_
optimizer
)
if
(
query_info
.
order_optimizer
)
{
{
for
(
auto
it
=
selected_tables
.
begin
();
it
!=
selected_tables
.
end
();
++
it
)
for
(
auto
it
=
selected_tables
.
begin
();
it
!=
selected_tables
.
end
();
++
it
)
{
{
auto
current_info
=
query_info
.
order_
by_
optimizer
->
getInputOrder
(
std
::
get
<
0
>
(
*
it
));
auto
current_info
=
query_info
.
order_optimizer
->
getInputOrder
(
std
::
get
<
0
>
(
*
it
));
if
(
it
==
selected_tables
.
begin
())
if
(
it
==
selected_tables
.
begin
())
input_sorting_info
=
current_info
;
input_sorting_info
=
current_info
;
else
if
(
!
current_info
||
(
input_sorting_info
&&
*
current_info
!=
*
input_sorting_info
))
else
if
(
!
current_info
||
(
input_sorting_info
&&
*
current_info
!=
*
input_sorting_info
))
...
@@ -192,7 +192,7 @@ Pipes StorageMerge::read(
...
@@ -192,7 +192,7 @@ Pipes StorageMerge::read(
break
;
break
;
}
}
query_info
.
input_
sorting
_info
=
input_sorting_info
;
query_info
.
input_
order
_info
=
input_sorting_info
;
}
}
for
(
const
auto
&
table
:
selected_tables
)
for
(
const
auto
&
table
:
selected_tables
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录