Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
bc387e86
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
bc387e86
编写于
6月 14, 2019
作者:
A
alexey-milovidov
提交者:
GitHub
6月 14, 2019
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #5549 from yandex/fix_deduplication_bug
Fix complex bug in deduplication
上级
864e1370
947dbeff
变更
3
显示空白变更内容
内联
并排
Showing
3 changed file
with
77 addition
and
1 deletion
+77
-1
dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp
...c/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp
+10
-1
dbms/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.reference
...teless/00953_zookeeper_suetin_deduplication_bug.reference
+4
-0
dbms/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh
...s/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh
+63
-0
未找到文件。
dbms/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp
浏览文件 @
bc387e86
...
...
@@ -319,10 +319,17 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks()
{
/// Can happen if there are leftover block nodes with children created by previous server versions.
zookeeper
->
removeRecursive
(
path
);
cached_block_stats
.
erase
(
first_outdated_block
->
node
);
}
else
if
(
rc
)
LOG_WARNING
(
log
,
"Error while deleting ZooKeeper path `"
<<
path
<<
"`: "
+
zkutil
::
ZooKeeper
::
error2string
(
rc
)
<<
", ignoring."
);
else
{
/// Successfully removed blocks have to be removed from cache
cached_block_stats
.
erase
(
first_outdated_block
->
node
);
}
first_outdated_block
++
;
}
auto
num_nodes_to_delete
=
timed_blocks
.
end
()
-
first_outdated_block
;
...
...
@@ -340,7 +347,9 @@ void ReplicatedMergeTreeCleanupThread::getBlocksSortedByTime(zkutil::ZooKeeper &
if
(
zookeeper
.
tryGetChildren
(
storage
.
zookeeper_path
+
"/blocks"
,
blocks
,
&
stat
))
throw
Exception
(
storage
.
zookeeper_path
+
"/blocks doesn't exist"
,
ErrorCodes
::
NOT_FOUND_NODE
);
/// Clear already deleted blocks from the cache, cached_block_ctime should be subset of blocks
/// Seems like this code is obsolete, because we delete blocks from cache
/// when they are deleted from zookeeper. But we don't know about all (maybe future) places in code
/// where they can be removed, so just to be sure that cache would not leak we check it here.
{
NameSet
blocks_set
(
blocks
.
begin
(),
blocks
.
end
());
for
(
auto
it
=
cached_block_stats
.
begin
();
it
!=
cached_block_stats
.
end
();)
...
...
dbms/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.reference
0 → 100644
浏览文件 @
bc387e86
3
4
5
5
dbms/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh
0 → 100755
浏览文件 @
bc387e86
#!/usr/bin/env bash
set
-e
CURDIR
=
$(
cd
"
$(
dirname
"
${
BASH_SOURCE
[0]
}
"
)
"
&&
pwd
)
.
$CURDIR
/../shell_config.sh
$CLICKHOUSE_CLIENT
--query
=
"DROP TABLE IF EXISTS elog;"
$CLICKHOUSE_CLIENT
--query
=
"
CREATE TABLE elog (
date Date,
engine_id UInt32,
referrer String
)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/elog', 'test')
PARTITION BY date
ORDER BY (engine_id)
SETTINGS replicated_deduplication_window = 2, cleanup_delay_period=4, cleanup_delay_period_random_add=0;"
$CLICKHOUSE_CLIENT
--query
=
"INSERT INTO elog VALUES (toDate('2018-10-01'), 1, 'hello')"
sleep
1
$CLICKHOUSE_CLIENT
--query
=
"INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')"
sleep
1
$CLICKHOUSE_CLIENT
--query
=
"INSERT INTO elog VALUES (toDate('2018-10-01'), 3, 'hello')"
$CLICKHOUSE_CLIENT
--query
=
"SELECT count(*) from elog"
# 3 rows
count
=
`
$CLICKHOUSE_CLIENT
--query
=
"SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/elog/blocks'"
`
while
[[
$count
!=
2
]]
do
sleep
1
count
=
`
$CLICKHOUSE_CLIENT
--query
=
"SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/elog/blocks'"
`
done
$CLICKHOUSE_CLIENT
--query
=
"INSERT INTO elog VALUES (toDate('2018-10-01'), 1, 'hello')"
$CLICKHOUSE_CLIENT
--query
=
"SELECT count(*) from elog"
# 4 rows
count
=
`
$CLICKHOUSE_CLIENT
--query
=
"SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/elog/blocks'"
`
while
[[
$count
!=
2
]]
do
sleep
1
count
=
`
$CLICKHOUSE_CLIENT
--query
=
"SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/elog/blocks'"
`
done
$CLICKHOUSE_CLIENT
--query
=
"INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')"
$CLICKHOUSE_CLIENT
--query
=
"SELECT count(*) from elog"
# 5 rows
count
=
`
$CLICKHOUSE_CLIENT
--query
=
"SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/elog/blocks'"
`
while
[[
$count
!=
2
]]
do
sleep
1
count
=
`
$CLICKHOUSE_CLIENT
--query
=
"SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/elog/blocks'"
`
done
$CLICKHOUSE_CLIENT
--query
=
"INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')"
$CLICKHOUSE_CLIENT
--query
=
"SELECT count(*) from elog"
# still 5 rows
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录