Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
c49eefda
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
c49eefda
编写于
12月 28, 2019
作者:
N
Nikita Mikhaylov
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
better integration tests
上级
62f26557
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
188 addition
and
49 deletion
+188
-49
dbms/src/Dictionaries/CacheDictionary.inc.h
dbms/src/Dictionaries/CacheDictionary.inc.h
+20
-24
dbms/tests/integration/helpers/cluster.py
dbms/tests/integration/helpers/cluster.py
+20
-0
dbms/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py
...ictionary_allow_read_expired_keys/test_default_reading.py
+64
-0
dbms/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py
.../test_dictionary_allow_read_expired_keys/test_dict_get.py
+24
-25
dbms/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py
...onary_allow_read_expired_keys/test_dict_get_or_default.py
+60
-0
未找到文件。
dbms/src/Dictionaries/CacheDictionary.inc.h
浏览文件 @
c49eefda
...
...
@@ -361,30 +361,26 @@ void CacheDictionary::prepareAnswer(
continue
;
}
/// TODO: understand properly this code and remove
// if (error_count)
// {
// if (find_result.outdated)
// {
// /// We have expired data for that `id` so we can continue using it.
// bool was_default = cell.isDefault();
//
// if (was_default)
// LOG_FATAL(log, "WAS DEFAULT");
//
// cell.setExpiresAt(backoff_end_time);
// if (was_default)
// cell.setDefault();
//
// if (was_default)
// on_id_not_found(id, cell_idx);
// else
// on_cell_updated(id, cell_idx);
// continue;
// }
// /// We don't have expired data for that `id` so all we can do is to rethrow `last_exception`.
// std::rethrow_exception(last_exception);
// }
if
(
error_count
)
{
if
(
find_result
.
outdated
)
{
/// We have expired data for that `id` so we can continue using it.
bool
was_default
=
cell
.
isDefault
();
cell
.
setExpiresAt
(
backoff_end_time
);
if
(
was_default
)
cell
.
setDefault
();
if
(
was_default
)
on_id_not_found
(
id
,
cell_idx
);
else
on_cell_updated
(
id
,
cell_idx
);
continue
;
}
/// We don't have expired data for that `id` so all we can do is to rethrow `last_exception`.
std
::
rethrow_exception
(
last_exception
);
}
/// Check if cell had not been occupied before and increment element counter if it hadn't
if
(
cell
.
id
==
0
&&
cell_idx
!=
zero_cell_idx
)
...
...
dbms/tests/integration/helpers/cluster.py
浏览文件 @
c49eefda
...
...
@@ -650,6 +650,15 @@ class ClickHouseInstance:
self
.
exec_in_container
([
"bash"
,
"-c"
,
"kill -9 {}"
.
format
(
pid
)],
user
=
'root'
)
time
.
sleep
(
stop_start_wait_sec
)
def
restore_clickhouse
(
self
,
retries
=
100
):
pid
=
self
.
get_process_pid
(
"clickhouse"
)
if
pid
:
raise
Exception
(
"ClickHouse has already started"
)
self
.
exec_in_container
([
"bash"
,
"-c"
,
"{} --daemon"
.
format
(
CLICKHOUSE_START_COMMAND
)],
user
=
str
(
os
.
getuid
()))
from
helpers.test_tools
import
assert_eq_with_retry
# wait start
assert_eq_with_retry
(
self
,
"select 1"
,
"1"
,
retry_count
=
retries
)
def
restart_clickhouse
(
self
,
stop_start_wait_sec
=
5
,
kill
=
False
):
if
not
self
.
stay_alive
:
raise
Exception
(
"clickhouse can be restarted only with stay_alive=True instance"
)
...
...
@@ -956,3 +965,14 @@ class ClickHouseInstance:
def
destroy_dir
(
self
):
if
p
.
exists
(
self
.
path
):
shutil
.
rmtree
(
self
.
path
)
class
ClickHouseKiller
(
object
):
def
__init__
(
self
,
clickhouse_node
):
self
.
clickhouse_node
=
clickhouse_node
def
__enter__
(
self
):
self
.
clickhouse_node
.
kill_clickhouse
()
def
__exit__
(
self
,
exc_type
,
exc_val
,
exc_tb
):
self
.
clickhouse_node
.
restore_clickhouse
()
\ No newline at end of file
dbms/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py
0 → 100644
浏览文件 @
c49eefda
from
__future__
import
print_function
import
pytest
import
time
import
os
from
contextlib
import
contextmanager
from
helpers.cluster
import
ClickHouseCluster
from
helpers.cluster
import
ClickHouseKiller
from
helpers.network
import
PartitionManager
SCRIPT_DIR
=
os
.
path
.
dirname
(
os
.
path
.
realpath
(
__file__
))
cluster
=
ClickHouseCluster
(
__file__
,
base_configs_dir
=
os
.
path
.
join
(
SCRIPT_DIR
,
'configs'
))
dictionary_node
=
cluster
.
add_instance
(
'dictionary_node'
,
stay_alive
=
True
)
main_node
=
cluster
.
add_instance
(
'main_node'
,
main_configs
=
[
'configs/dictionaries/cache_ints_dictionary.xml'
])
@
pytest
.
fixture
(
scope
=
"module"
)
def
started_cluster
():
try
:
cluster
.
start
()
dictionary_node
.
query
(
"create database if not exists test;"
)
dictionary_node
.
query
(
"drop table if exists test.ints;"
)
dictionary_node
.
query
(
"create table test.ints "
"(key UInt64, "
"i8 Int8, i16 Int16, i32 Int32, i64 Int64, "
"u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64) "
"Engine = Memory;"
)
dictionary_node
.
query
(
"insert into test.ints values (7, 7, 7, 7, 7, 7, 7, 7, 7);"
)
dictionary_node
.
query
(
"insert into test.ints values (5, 5, 5, 5, 5, 5, 5, 5, 5);"
)
yield
cluster
finally
:
cluster
.
shutdown
()
# @pytest.mark.skip(reason="debugging")
def
test_default_reading
(
started_cluster
):
assert
None
!=
dictionary_node
.
get_process_pid
(
"clickhouse"
),
"ClickHouse must be alive"
# Key 0 is not in dictionary, so default value will be returned
def
test_helper
():
assert
'42'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'i8', toUInt64(13), toInt8(42));"
).
rstrip
()
assert
'42'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'i16', toUInt64(13), toInt16(42));"
).
rstrip
()
assert
'42'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'i32', toUInt64(13), toInt32(42));"
).
rstrip
()
assert
'42'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'i64', toUInt64(13), toInt64(42));"
).
rstrip
()
assert
'42'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'u8', toUInt64(13), toUInt8(42));"
).
rstrip
()
assert
'42'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'u16', toUInt64(13), toUInt16(42));"
).
rstrip
()
assert
'42'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'u32', toUInt64(13), toUInt32(42));"
).
rstrip
()
assert
'42'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'u64', toUInt64(13), toUInt64(42));"
).
rstrip
()
test_helper
()
with
PartitionManager
()
as
pm
,
ClickHouseKiller
(
dictionary_node
):
assert
None
==
dictionary_node
.
get_process_pid
(
"clickhouse"
),
"CLickHouse must be alive"
# Remove connection between main_node and dictionary for sure
pm
.
heal_all
()
pm
.
partition_instances
(
main_node
,
dictionary_node
)
# Dictionary max lifetime is 2 seconds.
time
.
sleep
(
3
)
test_helper
()
dbms/tests/integration/test_dictionary_allow_read_expired_keys/test.py
→
dbms/tests/integration/test_dictionary_allow_read_expired_keys/test
_dict_get
.py
浏览文件 @
c49eefda
...
...
@@ -2,8 +2,10 @@ from __future__ import print_function
import
pytest
import
time
import
os
from
contextlib
import
contextmanager
from
helpers.cluster
import
ClickHouseCluster
from
helpers.cluster
import
ClickHouseKiller
from
helpers.network
import
PartitionManager
from
helpers.network
import
PartitionManagerDisabler
...
...
@@ -12,7 +14,6 @@ cluster = ClickHouseCluster(__file__, base_configs_dir=os.path.join(SCRIPT_DIR,
dictionary_node
=
cluster
.
add_instance
(
'dictionary_node'
,
stay_alive
=
True
)
main_node
=
cluster
.
add_instance
(
'main_node'
,
main_configs
=
[
'configs/dictionaries/cache_ints_dictionary.xml'
])
pm
=
PartitionManager
()
@
pytest
.
fixture
(
scope
=
"module"
)
def
started_cluster
():
...
...
@@ -20,40 +21,24 @@ def started_cluster():
cluster
.
start
()
dictionary_node
.
query
(
"create database if not exists test;"
)
dictionary_node
.
query
(
"drop table if exists test.ints;"
)
dictionary_node
.
query
(
"create table test.ints (key UInt64, i8 Int8, i16 Int16, i32 Int32, "
"i64 Int64, u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64) "
dictionary_node
.
query
(
"create table test.ints "
"(key UInt64, "
"i8 Int8, i16 Int16, i32 Int32, i64 Int64, "
"u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64) "
"Engine = Memory;"
)
dictionary_node
.
query
(
"insert into test.ints values (7, 7, 7, 7, 7, 7, 7, 7, 7);"
)
print
(
dictionary_node
.
query
(
"describe test.ints"
))
print
(
dictionary_node
.
query
(
"select * from test.ints"
))
dictionary_node
.
query
(
"insert into test.ints values (5, 5, 5, 5, 5, 5, 5, 5, 5);"
)
yield
cluster
finally
:
cluster
.
shutdown
()
# @pytest.mark.skip(reason="debugging")
def
test_simple_dict_get
(
started_cluster
):
assert
None
!=
dictionary_node
.
get_process_pid
(
"clickhouse"
),
"ClickHouse must be alive"
assert
'7'
==
main_node
.
query
(
"select dictGet('anime_dict', 'i8', toUInt64(7));"
).
rstrip
(),
"Wrong answer."
assert
'7'
==
main_node
.
query
(
"select dictGet('anime_dict', 'i16', toUInt64(7));"
).
rstrip
(),
"Wrong answer."
assert
'7'
==
main_node
.
query
(
"select dictGet('anime_dict', 'i32', toUInt64(7));"
).
rstrip
(),
"Wrong answer."
assert
'7'
==
main_node
.
query
(
"select dictGet('anime_dict', 'i64', toUInt64(7));"
).
rstrip
(),
"Wrong answer."
assert
'7'
==
main_node
.
query
(
"select dictGet('anime_dict', 'u8', toUInt64(7));"
).
rstrip
(),
"Wrong answer."
assert
'7'
==
main_node
.
query
(
"select dictGet('anime_dict', 'u16', toUInt64(7));"
).
rstrip
(),
"Wrong answer."
assert
'7'
==
main_node
.
query
(
"select dictGet('anime_dict', 'u32', toUInt64(7));"
).
rstrip
(),
"Wrong answer."
assert
'7'
==
main_node
.
query
(
"select dictGet('anime_dict', 'u64', toUInt64(7));"
).
rstrip
(),
"Wrong answer."
with
PartitionManager
()
as
pm
:
pm
.
partition_instances
(
main_node
,
dictionary_node
,
port
=
9000
)
# Kill the instance for sure
dictionary_node
.
kill_clickhouse
()
assert
None
==
dictionary_node
.
get_process_pid
(
"clickhouse"
)
pm
.
heal_all
()
pm
.
partition_instances
(
main_node
,
dictionary_node
,
port
=
9000
)
# print(dictionary_node.query("select * from test.ints"))
def
test_helper
():
assert
'7'
==
main_node
.
query
(
"select dictGet('anime_dict', 'i8', toUInt64(7));"
).
rstrip
(),
"Wrong answer."
assert
'7'
==
main_node
.
query
(
"select dictGet('anime_dict', 'i16', toUInt64(7));"
).
rstrip
(),
"Wrong answer."
assert
'7'
==
main_node
.
query
(
"select dictGet('anime_dict', 'i32', toUInt64(7));"
).
rstrip
(),
"Wrong answer."
...
...
@@ -62,3 +47,17 @@ def test_simple_dict_get(started_cluster):
assert
'7'
==
main_node
.
query
(
"select dictGet('anime_dict', 'u16', toUInt64(7));"
).
rstrip
(),
"Wrong answer."
assert
'7'
==
main_node
.
query
(
"select dictGet('anime_dict', 'u32', toUInt64(7));"
).
rstrip
(),
"Wrong answer."
assert
'7'
==
main_node
.
query
(
"select dictGet('anime_dict', 'u64', toUInt64(7));"
).
rstrip
(),
"Wrong answer."
test_helper
()
with
PartitionManager
()
as
pm
,
ClickHouseKiller
(
dictionary_node
):
assert
None
==
dictionary_node
.
get_process_pid
(
"clickhouse"
)
# Remove connection between main_node and dictionary for sure
pm
.
heal_all
()
pm
.
partition_instances
(
main_node
,
dictionary_node
)
# Dictionary max lifetime is 2 seconds.
time
.
sleep
(
3
)
test_helper
()
dbms/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py
0 → 100644
浏览文件 @
c49eefda
from
__future__
import
print_function
import
pytest
import
time
import
os
from
contextlib
import
contextmanager
from
helpers.cluster
import
ClickHouseCluster
from
helpers.cluster
import
ClickHouseKiller
from
helpers.network
import
PartitionManager
SCRIPT_DIR
=
os
.
path
.
dirname
(
os
.
path
.
realpath
(
__file__
))
cluster
=
ClickHouseCluster
(
__file__
,
base_configs_dir
=
os
.
path
.
join
(
SCRIPT_DIR
,
'configs'
))
dictionary_node
=
cluster
.
add_instance
(
'dictionary_node'
,
stay_alive
=
True
)
main_node
=
cluster
.
add_instance
(
'main_node'
,
main_configs
=
[
'configs/dictionaries/cache_ints_dictionary.xml'
])
@
pytest
.
fixture
(
scope
=
"module"
)
def
started_cluster
():
try
:
cluster
.
start
()
dictionary_node
.
query
(
"create database if not exists test;"
)
dictionary_node
.
query
(
"drop table if exists test.ints;"
)
dictionary_node
.
query
(
"create table test.ints "
"(key UInt64, "
"i8 Int8, i16 Int16, i32 Int32, i64 Int64, "
"u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64) "
"Engine = Memory;"
)
dictionary_node
.
query
(
"insert into test.ints values (7, 7, 7, 7, 7, 7, 7, 7, 7);"
)
dictionary_node
.
query
(
"insert into test.ints values (5, 5, 5, 5, 5, 5, 5, 5, 5);"
)
yield
cluster
finally
:
cluster
.
shutdown
()
# @pytest.mark.skip(reason="debugging")
def
test_simple_dict_get_or_default
(
started_cluster
):
assert
None
!=
dictionary_node
.
get_process_pid
(
"clickhouse"
),
"ClickHouse must be alive"
def
test_helper
():
assert
'5'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'i8', toUInt64(5), toInt8(42));"
).
rstrip
()
assert
'5'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'i16', toUInt64(5), toInt16(42));"
).
rstrip
()
assert
'5'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'i32', toUInt64(5), toInt32(42));"
).
rstrip
()
assert
'5'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'i64', toUInt64(5), toInt64(42));"
).
rstrip
()
assert
'5'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'u8', toUInt64(5), toUInt8(42));"
).
rstrip
()
assert
'5'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'u16', toUInt64(5), toUInt16(42));"
).
rstrip
()
assert
'5'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'u32', toUInt64(5), toUInt32(42));"
).
rstrip
()
assert
'5'
==
main_node
.
query
(
"select dictGetOrDefault('anime_dict', 'u64', toUInt64(5), toUInt64(42));"
).
rstrip
()
test_helper
()
with
PartitionManager
()
as
pm
,
ClickHouseKiller
(
dictionary_node
):
assert
None
==
dictionary_node
.
get_process_pid
(
"clickhouse"
)
# Remove connection between main_node and dictionary for sure
pm
.
partition_instances
(
main_node
,
dictionary_node
)
# Dictionary max lifetime is 2 seconds.
time
.
sleep
(
3
)
test_helper
()
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录