Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
bc09b513
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
bc09b513
编写于
5月 18, 2019
作者:
A
Anastasiya Rodigina
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add reading in pk_order
上级
af4fff97
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
95 addition
and
5 deletion
+95
-5
dbms/src/Storages/MergeTree/MergeTreeData.cpp
dbms/src/Storages/MergeTree/MergeTreeData.cpp
+3
-1
dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
+8
-2
dbms/src/Storages/MergeTree/MergeTreeDataPart.h
dbms/src/Storages/MergeTree/MergeTreeDataPart.h
+10
-2
dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
+65
-0
dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h
dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h
+9
-0
未找到文件。
dbms/src/Storages/MergeTree/MergeTreeData.cpp
浏览文件 @
bc09b513
...
...
@@ -764,13 +764,15 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
auto
lock
=
lockParts
();
data_parts_indexes
.
clear
();
size_t
data_parts_counter
=
0
;
for
(
const
String
&
file_name
:
part_file_names
)
{
MergeTreePartInfo
part_info
;
if
(
!
MergeTreePartInfo
::
tryParsePartName
(
file_name
,
&
part_info
,
format_version
))
continue
;
MutableDataPartPtr
part
=
std
::
make_shared
<
DataPart
>
(
*
this
,
file_name
,
part_info
);
MutableDataPartPtr
part
=
std
::
make_shared
<
DataPart
>
(
*
this
,
file_name
,
part_info
,
data_parts_counter
);
++
data_parts_counter
;
part
->
relative_path
=
file_name
;
bool
broken
=
false
;
...
...
dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
浏览文件 @
bc09b513
...
...
@@ -135,17 +135,23 @@ void MergeTreeDataPart::MinMaxIndex::merge(const MinMaxIndex & other)
}
MergeTreeDataPart
::
MergeTreeDataPart
(
MergeTreeData
&
storage_
,
const
String
&
name_
)
MergeTreeDataPart
::
MergeTreeDataPart
(
MergeTreeData
&
storage_
,
const
String
&
name_
,
size_t
data_part_id_
)
:
storage
(
storage_
)
,
name
(
name_
)
,
info
(
MergeTreePartInfo
::
fromPartName
(
name_
,
storage
.
format_version
))
,
data_part_id
(
data_part_id_
)
{
}
MergeTreeDataPart
::
MergeTreeDataPart
(
const
MergeTreeData
&
storage_
,
const
String
&
name_
,
const
MergeTreePartInfo
&
info_
)
MergeTreeDataPart
::
MergeTreeDataPart
(
const
MergeTreeData
&
storage_
,
const
String
&
name_
,
const
MergeTreePartInfo
&
info_
,
size_t
data_part_id_
)
:
storage
(
storage_
)
,
name
(
name_
)
,
info
(
info_
)
,
data_part_id
(
data_part_id_
)
{
}
...
...
dbms/src/Storages/MergeTree/MergeTreeDataPart.h
浏览文件 @
bc09b513
...
...
@@ -30,9 +30,9 @@ struct MergeTreeDataPart
using
Checksums
=
MergeTreeDataPartChecksums
;
using
Checksum
=
MergeTreeDataPartChecksums
::
Checksum
;
MergeTreeDataPart
(
const
MergeTreeData
&
storage_
,
const
String
&
name_
,
const
MergeTreePartInfo
&
info_
);
MergeTreeDataPart
(
const
MergeTreeData
&
storage_
,
const
String
&
name_
,
const
MergeTreePartInfo
&
info_
,
size_t
data_part_id_
=
0
);
MergeTreeDataPart
(
MergeTreeData
&
storage_
,
const
String
&
name_
);
MergeTreeDataPart
(
MergeTreeData
&
storage_
,
const
String
&
name_
,
size_t
data_part_id_
=
0
);
/// Returns the name of a column with minimum compressed size (as returned by getColumnSize()).
/// If no checksums are present returns the name of the first physically existing column.
...
...
@@ -92,6 +92,9 @@ struct MergeTreeDataPart
/// Examples: 'detached/tmp_fetch_<name>', 'tmp_<name>', '<name>'
mutable
String
relative_path
;
/// Is used in reading in pk_order
size_t
data_part_id
=
0
;
size_t
rows_count
=
0
;
std
::
atomic
<
UInt64
>
bytes_on_disk
{
0
};
/// 0 - if not counted;
/// Is used from several threads without locks (it is changed with ALTER).
...
...
@@ -144,6 +147,11 @@ struct MergeTreeDataPart
return
name
+
" (state "
+
stateString
()
+
")"
;
}
void
setDataPartId
(
size_t
data_part_id_
)
{
data_part_id
=
data_part_id_
;
}
/// Returns true if state of part is one of affordable_states
bool
checkState
(
const
std
::
initializer_list
<
State
>
&
affordable_states
)
const
{
...
...
dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
浏览文件 @
bc09b513
...
...
@@ -587,6 +587,28 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts(
virt_column_names
,
settings
);
}
else
if
(
settings
.
optimize_pk_order
)
{
std
::
vector
<
String
>
add_columns
=
data
.
sorting_key_expr
->
getRequiredColumns
();
column_names_to_read
.
insert
(
column_names_to_read
.
end
(),
add_columns
.
begin
(),
add_columns
.
end
());
if
(
!
data
.
merging_params
.
sign_column
.
empty
())
column_names_to_read
.
push_back
(
data
.
merging_params
.
sign_column
);
if
(
!
data
.
merging_params
.
version_column
.
empty
())
column_names_to_read
.
push_back
(
data
.
merging_params
.
version_column
);
std
::
sort
(
column_names_to_read
.
begin
(),
column_names_to_read
.
end
());
column_names_to_read
.
erase
(
std
::
unique
(
column_names_to_read
.
begin
(),
column_names_to_read
.
end
()),
column_names_to_read
.
end
());
res
=
spreadMarkRangesAmongStreamsPKOrder
(
std
::
move
(
parts_with_ranges
),
column_names_to_read
,
max_block_size
,
settings
.
use_uncompressed_cache
,
query_info
,
virt_column_names
,
settings
);
}
else
{
res
=
spreadMarkRangesAmongStreams
(
...
...
@@ -790,6 +812,49 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams(
return
res
;
}
BlockInputStreams
MergeTreeDataSelectExecutor
::
spreadMarkRangesAmongStreamsPKOrder
(
RangesInDataParts
&&
parts
,
const
Names
&
column_names
,
UInt64
max_block_size
,
bool
use_uncompressed_cache
,
const
SelectQueryInfo
&
query_info
,
const
Names
&
virt_columns
,
const
Settings
&
settings
)
const
{
const
PrewhereInfoPtr
prewhere_info
=
query_info
.
prewhere_info
;
const
size_t
max_marks_to_use_cache
=
roundRowsOrBytesToMarks
(
settings
.
merge_tree_max_rows_to_use_cache
,
settings
.
merge_tree_max_bytes_to_use_cache
,
data
.
index_granularity_info
);
size_t
sum_marks
=
0
;
for
(
size_t
i
=
0
;
i
<
parts
.
size
();
++
i
)
for
(
size_t
j
=
0
;
j
<
parts
[
i
].
ranges
.
size
();
++
j
)
sum_marks
+=
parts
[
i
].
ranges
[
j
].
end
-
parts
[
i
].
ranges
[
j
].
begin
;
if
(
sum_marks
>
max_marks_to_use_cache
)
use_uncompressed_cache
=
false
;
BlockInputStreams
to_merge
;
for
(
size_t
part_index
=
0
;
part_index
<
parts
.
size
();
++
part_index
)
{
RangesInDataPart
&
part
=
parts
[
part_index
];
BlockInputStreamPtr
source_stream
=
std
::
make_shared
<
MergeTreeSelectBlockInputStream
>
(
data
,
part
.
data_part
,
max_block_size
,
settings
.
preferred_block_size_bytes
,
settings
.
preferred_max_column_in_block_size_bytes
,
column_names
,
part
.
ranges
,
use_uncompressed_cache
,
prewhere_info
,
true
,
settings
.
min_bytes_to_use_direct_io
,
settings
.
max_read_buffer_size
,
true
,
virt_columns
,
part
.
part_index_in_query
);
to_merge
.
emplace_back
(
std
::
make_shared
<
ExpressionBlockInputStream
>
(
source_stream
,
data
.
sorting_key_expr
));
}
return
to_merge
;
}
BlockInputStreams
MergeTreeDataSelectExecutor
::
spreadMarkRangesAmongStreamsFinal
(
RangesInDataParts
&&
parts
,
const
Names
&
column_names
,
...
...
dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h
浏览文件 @
bc09b513
...
...
@@ -56,6 +56,15 @@ private:
const
Names
&
virt_columns
,
const
Settings
&
settings
)
const
;
BlockInputStreams
spreadMarkRangesAmongStreamsPKOrder
(
RangesInDataParts
&&
parts
,
const
Names
&
column_names
,
UInt64
max_block_size
,
bool
use_uncompressed_cache
,
const
SelectQueryInfo
&
query_info
,
const
Names
&
virt_columns
,
const
Settings
&
settings
)
const
;
BlockInputStreams
spreadMarkRangesAmongStreamsFinal
(
RangesInDataParts
&&
parts
,
const
Names
&
column_names
,
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录