Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
ee6a6472
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
ee6a6472
编写于
4月 14, 2014
作者:
M
Michael Kolupaev
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Merge
上级
b9c4a341
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
46 addition
and
11 deletion
+46
-11
dbms/include/DB/Storages/StorageReplicatedMergeTree.h
dbms/include/DB/Storages/StorageReplicatedMergeTree.h
+7
-2
dbms/src/Storages/StorageReplicatedMergeTree.cpp
dbms/src/Storages/StorageReplicatedMergeTree.cpp
+39
-9
未找到文件。
dbms/include/DB/Storages/StorageReplicatedMergeTree.h
浏览文件 @
ee6a6472
...
...
@@ -246,9 +246,10 @@ private:
/// Поток, выбирающий куски для слияния.
std
::
thread
merge_selecting_thread
;
/// Поток, удаляющий информацию о старых блоках из ZooKeeper.
std
::
thread
clear_old_blocks_thread
;
/// Когда последний раз выбрасывали старые данные из ZooKeeper.
time_t
clear_old_blocks_time
=
0
;
/// Когда последний раз выбрасывали старые логи из ZooKeeper.
time_t
clear_old_logs_time
=
0
;
Logger
*
log
;
...
...
@@ -347,6 +348,10 @@ private:
*/
void
mergeSelectingThread
();
/** В бесконечном цикле вызывает clearOldBlocks.
*/
void
clearOldBlocksThread
();
/// Вызывается во время выбора кусков для слияния.
bool
canMergeParts
(
const
MergeTreeData
::
DataPartPtr
&
left
,
const
MergeTreeData
::
DataPartPtr
&
right
);
...
...
dbms/src/Storages/StorageReplicatedMergeTree.cpp
浏览文件 @
ee6a6472
...
...
@@ -369,10 +369,15 @@ void StorageReplicatedMergeTree::clearOldBlocks()
if
(
!
zookeeper
.
exists
(
zookeeper_path
+
"/blocks"
,
&
stat
))
throw
Exception
(
zookeeper_path
+
"/blocks doesn't exist"
,
ErrorCodes
::
NOT_FOUND_NODE
);
int
children_count
=
stat
.
getnumChildren
();
/// Чтобы делать "асимптотически" меньше запросов exists, будем ждать, пока накопятся в 1.1 раза больше блоков, чем нужно.
if
(
static_cast
<
double
>
(
stat
.
getnumChildren
()
)
<
data
.
settings
.
replicated_deduplication_window
*
1.1
)
if
(
static_cast
<
double
>
(
children_count
)
<
data
.
settings
.
replicated_deduplication_window
*
1.1
)
return
;
LOG_TRACE
(
log
,
"Clearing about "
<<
static_cast
<
size_t
>
(
children_count
)
-
data
.
settings
.
replicated_deduplication_window
<<
" old blocks from ZooKeeper"
);
Strings
blocks
=
zookeeper
.
getChildren
(
zookeeper_path
+
"/blocks"
);
std
::
vector
<
std
::
pair
<
Int64
,
String
>
>
timed_blocks
;
...
...
@@ -384,10 +389,14 @@ void StorageReplicatedMergeTree::clearOldBlocks()
timed_blocks
.
push_back
(
std
::
make_pair
(
stat
.
getczxid
(),
block
));
}
std
::
sort
(
timed_blocks
.
begin
(),
timed_blocks
.
end
());
std
::
sort
(
timed_blocks
.
begin
(),
timed_blocks
.
end
()
,
std
::
greater
<
std
::
pair
<
Int64
,
String
>>
()
);
for
(
size_t
i
=
data
.
settings
.
replicated_deduplication_window
;
i
<
timed_blocks
.
size
();
++
i
)
{
zookeeper
.
remove
(
zookeeper_path
+
"/blocks/"
+
timed_blocks
[
i
].
second
);
zkutil
::
Ops
ops
;
ops
.
push_back
(
new
zkutil
::
Op
::
Remove
(
zookeeper_path
+
"/blocks/"
+
timed_blocks
[
i
].
second
+
"/number"
,
-
1
));
ops
.
push_back
(
new
zkutil
::
Op
::
Remove
(
zookeeper_path
+
"/blocks/"
+
timed_blocks
[
i
].
second
+
"/checksums"
,
-
1
));
ops
.
push_back
(
new
zkutil
::
Op
::
Remove
(
zookeeper_path
+
"/blocks/"
+
timed_blocks
[
i
].
second
,
-
1
));
zookeeper
.
multi
(
ops
);
}
LOG_TRACE
(
log
,
"Cleared "
<<
blocks
.
size
()
-
data
.
settings
.
replicated_deduplication_window
<<
" old blocks from ZooKeeper"
);
...
...
@@ -882,18 +891,35 @@ void StorageReplicatedMergeTree::mergeSelectingThread()
tryLogCurrentException
(
__PRETTY_FUNCTION__
);
}
if
(
shutdown_called
)
if
(
shutdown_called
||
!
is_leader_node
)
break
;
/// Каждую минуту выбрасываем старые блоки.
if
(
time
(
0
)
-
clear_old_blocks_time
>
60
)
if
(
!
success
)
std
::
this_thread
::
sleep_for
(
MERGE_SELECTING_SLEEP
);
}
}
void
StorageReplicatedMergeTree
::
clearOldBlocksThread
()
{
while
(
!
shutdown_called
&&
is_leader_node
)
{
try
{
clear_old_blocks_time
=
time
(
0
);
clearOldBlocks
();
}
catch
(...)
{
tryLogCurrentException
(
__PRETTY_FUNCTION__
);
}
if
(
!
success
)
std
::
this_thread
::
sleep_for
(
MERGE_SELECTING_SLEEP
);
/// Спим минуту, но проверяем, нужно ли завершиться, каждую секунду.
/// TODO: Лучше во всех подобных местах использовать condition variable.
for
(
size_t
i
=
0
;
i
<
60
;
++
i
)
{
if
(
shutdown_called
||
!
is_leader_node
)
break
;
std
::
this_thread
::
sleep_for
(
std
::
chrono
::
seconds
(
1
));
}
}
}
...
...
@@ -925,6 +951,7 @@ void StorageReplicatedMergeTree::becomeLeader()
LOG_INFO
(
log
,
"Became leader"
);
is_leader_node
=
true
;
merge_selecting_thread
=
std
::
thread
(
&
StorageReplicatedMergeTree
::
mergeSelectingThread
,
this
);
clear_old_blocks_thread
=
std
::
thread
(
&
StorageReplicatedMergeTree
::
clearOldBlocksThread
,
this
);
}
String
StorageReplicatedMergeTree
::
findReplicaHavingPart
(
const
String
&
part_name
,
bool
active
)
...
...
@@ -992,7 +1019,10 @@ void StorageReplicatedMergeTree::shutdown()
LOG_TRACE
(
log
,
"Waiting for threads to finish"
);
if
(
is_leader_node
)
{
merge_selecting_thread
.
join
();
clear_old_blocks_thread
.
join
();
}
queue_updating_thread
.
join
();
for
(
auto
&
thread
:
queue_threads
)
thread
.
join
();
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录