Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
0b5645db
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
0b5645db
编写于
4月 18, 2017
作者:
A
Alexey Milovidov
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Better code for parsing part names [#CLICKHOUSE-2].
上级
47d2ca41
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
58 addition
and
40 deletion
+58
-40
dbms/src/Storages/MergeTree/ActiveDataPartSet.cpp
dbms/src/Storages/MergeTree/ActiveDataPartSet.cpp
+45
-27
dbms/src/Storages/MergeTree/ActiveDataPartSet.h
dbms/src/Storages/MergeTree/ActiveDataPartSet.h
+5
-3
dbms/src/Storages/MergeTree/MergeTreeData.cpp
dbms/src/Storages/MergeTree/MergeTreeData.cpp
+6
-8
dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
+2
-2
未找到文件。
dbms/src/Storages/MergeTree/ActiveDataPartSet.cpp
浏览文件 @
0b5645db
#include <Storages/MergeTree/ActiveDataPartSet.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>
#include <IO/ReadBufferFromString.h>
namespace
DB
...
...
@@ -141,50 +142,67 @@ String ActiveDataPartSet::getPartName(DayNum_t left_date, DayNum_t right_date, I
}
bool
ActiveDataPartSet
::
isPartDirectory
(
const
String
&
dir_name
,
Poco
::
RegularExpression
::
MatchVec
*
out_matches
)
bool
ActiveDataPartSet
::
isPartDirectory
(
const
String
&
dir_name
)
{
Poco
::
RegularExpression
::
MatchVec
matches
;
static
Poco
::
RegularExpression
file_name_regexp
(
"^(
\\
d{8})_(
\\
d{8})_(-?
\\
d+)_(-?
\\
d+)_(
\\
d+)"
);
bool
res
=
(
file_name_regexp
.
match
(
dir_name
,
0
,
matches
)
&&
6
==
matches
.
size
());
if
(
out_matches
)
*
out_matches
=
matches
;
return
res
;
return
parsePartNameImpl
(
dir_name
,
nullptr
);
}
void
ActiveDataPartSet
::
parsePartName
(
const
String
&
file_name
,
Part
&
part
,
const
Poco
::
RegularExpression
::
MatchVec
*
matches_p
)
bool
ActiveDataPartSet
::
parsePartNameImpl
(
const
String
&
dir_name
,
Part
*
part
)
{
Poco
::
RegularExpression
::
MatchVec
match_vec
;
if
(
!
matches_p
)
UInt32
min_yyyymmdd
=
0
;
UInt32
max_yyyymmdd
=
0
;
Int64
min_block_num
=
0
;
Int64
max_block_num
=
0
;
UInt32
level
=
0
;
ReadBufferFromString
in
(
dir_name
);
if
(
!
tryReadIntText
(
min_yyyymmdd
,
in
)
||
!
checkChar
(
'_'
,
in
)
||
!
tryReadIntText
(
max_yyyymmdd
,
in
)
||
!
checkChar
(
'_'
,
in
)
||
!
tryReadIntText
(
min_block_num
,
in
)
||
!
checkChar
(
'_'
,
in
)
||
!
tryReadIntText
(
max_block_num
,
in
)
||
!
checkChar
(
'_'
,
in
)
||
!
tryReadIntText
(
level
,
in
)
||
!
in
.
eof
())
{
if
(
!
isPartDirectory
(
file_name
,
&
match_vec
))
throw
Exception
(
"Unexpected part name: "
+
file_name
,
ErrorCodes
::
BAD_DATA_PART_NAME
);
matches_p
=
&
match_vec
;
return
false
;
}
const
Poco
::
RegularExpression
::
MatchVec
&
matches
=
*
matches_p
;
if
(
part
)
{
const
auto
&
date_lut
=
DateLUT
::
instance
();
const
auto
&
date_lut
=
DateLUT
::
instance
();
part
->
left_date
=
date_lut
.
YYYYMMDDToDayNum
(
min_yyyymmdd
);
part
->
right_date
=
date_lut
.
YYYYMMDDToDayNum
(
max_yyyymmdd
);
part
->
left
=
min_block_num
;
part
->
right
=
max_block_num
;
part
->
level
=
level
;
part
.
left_date
=
date_lut
.
YYYYMMDDToDayNum
(
parse
<
UInt32
>
(
file_name
.
substr
(
matches
[
1
].
offset
,
matches
[
1
].
length
)));
part
.
right_date
=
date_lut
.
YYYYMMDDToDayNum
(
parse
<
UInt32
>
(
file_name
.
substr
(
matches
[
2
].
offset
,
matches
[
2
].
length
)));
part
.
left
=
parse
<
Int64
>
(
file_name
.
substr
(
matches
[
3
].
offset
,
matches
[
3
].
length
));
part
.
right
=
parse
<
Int64
>
(
file_name
.
substr
(
matches
[
4
].
offset
,
matches
[
4
].
length
));
part
.
level
=
parse
<
UInt32
>
(
file_name
.
substr
(
matches
[
5
].
offset
,
matches
[
5
].
length
));
DayNum_t
left_month
=
date_lut
.
toFirstDayNumOfMonth
(
part
->
left_date
);
DayNum_t
right_month
=
date_lut
.
toFirstDayNumOfMonth
(
part
->
right_date
);
DayNum_t
left_month
=
date_lut
.
toFirstDayNumOfMonth
(
part
.
left_date
);
DayNum_t
right_month
=
date_lut
.
toFirstDayNumOfMonth
(
part
.
right_date
);
if
(
left_month
!=
right_month
)
throw
Exception
(
"Part name "
+
dir_name
+
" contains different months"
,
ErrorCodes
::
BAD_DATA_PART_NAME
);
if
(
left_month
!=
right_month
)
throw
Exception
(
"Part name "
+
file_name
+
" contains different months"
,
ErrorCodes
::
BAD_DATA_PART_NAME
);
part
->
month
=
left_month
;
}
part
.
month
=
left_month
;
return
true
;
}
void
ActiveDataPartSet
::
parsePartName
(
const
String
&
dir_name
,
Part
&
part
)
{
if
(
!
parsePartNameImpl
(
dir_name
,
&
part
))
throw
Exception
(
"Unexpected part name: "
+
dir_name
,
ErrorCodes
::
BAD_DATA_PART_NAME
);
}
bool
ActiveDataPartSet
::
contains
(
const
String
&
outer_part_name
,
const
String
&
inner_part_name
)
{
Part
outer
,
inner
;
Part
outer
;
Part
inner
;
parsePartName
(
outer_part_name
,
outer
);
parsePartName
(
inner_part_name
,
inner
);
return
outer
.
contains
(
inner
);
...
...
dbms/src/Storages/MergeTree/ActiveDataPartSet.h
浏览文件 @
0b5645db
#pragma once
#include <mutex>
#include <Poco/RegularExpression.h>
#include <common/DateLUT.h>
#include <Core/Types.h>
#include <set>
namespace
DB
{
...
...
@@ -69,10 +69,12 @@ public:
static
String
getPartName
(
DayNum_t
left_date
,
DayNum_t
right_date
,
Int64
left_id
,
Int64
right_id
,
UInt64
level
);
/// Returns true if the directory name matches the format of the directory name of the parts
static
bool
isPartDirectory
(
const
String
&
dir_name
,
Poco
::
RegularExpression
::
MatchVec
*
out_matches
=
nullptr
);
static
bool
isPartDirectory
(
const
String
&
dir_name
);
static
bool
parsePartNameImpl
(
const
String
&
dir_name
,
Part
*
part
);
/// Put data in DataPart from the name of the part.
static
void
parsePartName
(
const
String
&
file_name
,
Part
&
part
,
const
Poco
::
RegularExpression
::
MatchVec
*
matches
=
nullptr
);
static
void
parsePartName
(
const
String
&
dir_name
,
Part
&
part
);
static
bool
contains
(
const
String
&
outer_part_name
,
const
String
&
inner_part_name
);
...
...
dbms/src/Storages/MergeTree/MergeTreeData.cpp
浏览文件 @
0b5645db
...
...
@@ -306,16 +306,13 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
DataPartsVector
broken_parts_to_detach
;
size_t
suspicious_broken_parts
=
0
;
Poco
::
RegularExpression
::
MatchVec
matches
;
for
(
const
String
&
file_name
:
part_file_names
)
{
if
(
!
ActiveDataPartSet
::
isPartDirectory
(
file_name
,
&
matches
))
MutableDataPartPtr
part
=
std
::
make_shared
<
DataPart
>
(
*
this
);
if
(
!
ActiveDataPartSet
::
parsePartNameImpl
(
file_name
,
part
.
get
()))
continue
;
MutableDataPartPtr
part
=
std
::
make_shared
<
DataPart
>
(
*
this
);
ActiveDataPartSet
::
parsePartName
(
file_name
,
*
part
,
&
matches
);
part
->
name
=
file_name
;
bool
broken
=
false
;
try
...
...
@@ -365,10 +362,11 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
{
if
(
contained_name
==
file_name
)
continue
;
if
(
!
ActiveDataPartSet
::
isPartDirectory
(
contained_name
,
&
matches
))
continue
;
DataPart
contained_part
(
*
this
);
ActiveDataPartSet
::
parsePartName
(
contained_name
,
contained_part
,
&
matches
);
if
(
!
ActiveDataPartSet
::
parsePartNameImpl
(
contained_name
,
&
contained_part
))
continue
;
if
(
part
->
contains
(
contained_part
))
{
LOG_ERROR
(
log
,
"Found part "
<<
full_path
+
contained_name
);
...
...
dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
浏览文件 @
0b5645db
...
...
@@ -468,8 +468,8 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
return
false
;
/** When the corresponding action is completed, then `shouldExecuteLogEntry` next time, will succeed,
* and queue element will be processed.
Immediately in the `executeLogEntry` function it will be found that we already have a part,
,
* and queue element will be processed.
* Immediately in the `executeLogEntry` function it will be found that we already have a part
,
* and queue element will be immediately treated as processed.
*/
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录