Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
9493532f
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
9493532f
编写于
9月 07, 2020
作者:
A
Anton Popov
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
rework test for redis dictionary
上级
c4861155
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
184 addition
and
134 deletion
+184
-134
docker/test/integration/runner/compose/docker_compose_redis.yml
.../test/integration/runner/compose/docker_compose_redis.yml
+1
-1
tests/integration/helpers/dictionary.py
tests/integration/helpers/dictionary.py
+0
-0
tests/integration/helpers/external_sources.py
tests/integration/helpers/external_sources.py
+3
-11
tests/integration/helpers/fake_cert.pem
tests/integration/helpers/fake_cert.pem
+0
-0
tests/integration/helpers/http_server.py
tests/integration/helpers/http_server.py
+0
-0
tests/integration/test_dictionaries_all_layouts_and_sources/test.py
...gration/test_dictionaries_all_layouts_and_sources/test.py
+4
-122
tests/integration/test_dictionaries_redis/__init__.py
tests/integration/test_dictionaries_redis/__init__.py
+0
-0
tests/integration/test_dictionaries_redis/test.py
tests/integration/test_dictionaries_redis/test.py
+176
-0
未找到文件。
docker/test/integration/runner/compose/docker_compose_redis.yml
浏览文件 @
9493532f
...
...
@@ -5,4 +5,4 @@ services:
restart
:
always
ports
:
-
6380:6379
command
:
redis-server --requirepass "clickhouse"
command
:
redis-server --requirepass "clickhouse"
--databases
32
tests/integration/
test_dictionaries_all_layouts_and_source
s/dictionary.py
→
tests/integration/
helper
s/dictionary.py
浏览文件 @
9493532f
文件已移动
tests/integration/
test_dictionaries_all_layouts_and_source
s/external_sources.py
→
tests/integration/
helper
s/external_sources.py
浏览文件 @
9493532f
...
...
@@ -477,13 +477,13 @@ class SourceCassandra(ExternalSource):
class
SourceRedis
(
ExternalSource
):
def
__init__
(
self
,
name
,
internal_hostname
,
internal_port
,
docker_hostname
,
docker_port
,
user
,
password
,
storage_type
self
,
name
,
internal_hostname
,
internal_port
,
docker_hostname
,
docker_port
,
user
,
password
,
db_index
,
storage_type
):
super
(
SourceRedis
,
self
).
__init__
(
name
,
internal_hostname
,
internal_port
,
docker_hostname
,
docker_port
,
user
,
password
)
self
.
storage_type
=
storage_type
self
.
db_index
=
1
self
.
db_index
=
db_index
def
get_source_str
(
self
,
table_name
):
return
'''
...
...
@@ -513,21 +513,13 @@ class SourceRedis(ExternalSource):
values
=
[]
for
name
in
self
.
ordered_names
:
values
.
append
(
str
(
row
.
data
[
name
]))
print
'values: '
,
values
if
len
(
values
)
==
2
:
self
.
client
.
set
(
*
values
)
print
'kek: '
,
self
.
client
.
get
(
values
[
0
])
else
:
self
.
client
.
hset
(
*
values
)
def
compatible_with_layout
(
self
,
layout
):
if
(
layout
.
is_simple
and
self
.
storage_type
==
"simple"
or
layout
.
is_complex
and
self
.
storage_type
==
"simple"
and
layout
.
name
==
"complex_key_hashed_one_key"
or
layout
.
is_complex
and
self
.
storage_type
==
"hash_map"
and
layout
.
name
==
"complex_key_hashed_two_keys"
):
return
True
return
False
return
layout
.
is_simple
and
self
.
storage_type
==
"simple"
or
layout
.
is_complex
and
self
.
storage_type
==
"hash_map"
class
SourceAerospike
(
ExternalSource
):
def
__init__
(
self
,
name
,
internal_hostname
,
internal_port
,
...
...
tests/integration/
test_dictionaries_all_layouts_and_source
s/fake_cert.pem
→
tests/integration/
helper
s/fake_cert.pem
浏览文件 @
9493532f
文件已移动
tests/integration/
test_dictionaries_all_layouts_and_source
s/http_server.py
→
tests/integration/
helper
s/http_server.py
浏览文件 @
9493532f
文件已移动
tests/integration/test_dictionaries_all_layouts_and_sources/test.py
浏览文件 @
9493532f
...
...
@@ -2,9 +2,9 @@ import pytest
import
os
from
helpers.cluster
import
ClickHouseCluster
from
dictionary
import
Field
,
Row
,
Dictionary
,
DictionaryStructure
,
Layout
from
external_sources
import
SourceMySQL
,
SourceClickHouse
,
SourceFile
,
SourceExecutableCache
,
SourceExecutableHashed
from
external_sources
import
SourceMongo
,
SourceMongoURI
,
SourceHTTP
,
SourceHTTPS
,
SourceRedis
,
SourceCassandra
from
helpers.
dictionary
import
Field
,
Row
,
Dictionary
,
DictionaryStructure
,
Layout
from
helpers.
external_sources
import
SourceMySQL
,
SourceClickHouse
,
SourceFile
,
SourceExecutableCache
,
SourceExecutableHashed
from
helpers.
external_sources
import
SourceMongo
,
SourceMongoURI
,
SourceHTTP
,
SourceHTTPS
,
SourceRedis
,
SourceCassandra
import
math
SCRIPT_DIR
=
os
.
path
.
dirname
(
os
.
path
.
realpath
(
__file__
))
...
...
@@ -132,14 +132,6 @@ SOURCES = [
DICTIONARIES
=
[]
# Key-value dictionaries with only one possible field for key
SOURCES_KV
=
[
SourceRedis
(
"RedisSimple"
,
"localhost"
,
"6380"
,
"redis1"
,
"6379"
,
""
,
"clickhouse"
,
storage_type
=
"simple"
),
SourceRedis
(
"RedisHash"
,
"localhost"
,
"6380"
,
"redis1"
,
"6379"
,
""
,
"clickhouse"
,
storage_type
=
"hash_map"
),
]
DICTIONARIES_KV
=
[]
cluster
=
None
node
=
None
...
...
@@ -170,17 +162,6 @@ def setup_module(module):
else
:
print
"Source"
,
source
.
name
,
"incompatible with layout"
,
layout
.
name
for
layout
in
LAYOUTS
:
field_keys
=
list
(
filter
(
lambda
x
:
x
.
is_key
,
FIELDS
[
layout
.
layout_type
]))
for
source
in
SOURCES_KV
:
if
not
source
.
compatible_with_layout
(
layout
):
print
"Source"
,
source
.
name
,
"incompatible with layout"
,
layout
.
name
continue
for
field
in
FIELDS
[
layout
.
layout_type
]:
if
not
(
field
.
is_key
or
field
.
is_range
or
field
.
is_range_key
):
DICTIONARIES_KV
.
append
(
get_dict
(
source
,
layout
,
field_keys
+
[
field
],
field
.
name
))
cluster
=
ClickHouseCluster
(
__file__
)
main_configs
=
[]
...
...
@@ -199,7 +180,7 @@ def setup_module(module):
def
started_cluster
():
try
:
cluster
.
start
()
for
dictionary
in
DICTIONARIES
+
DICTIONARIES_KV
:
for
dictionary
in
DICTIONARIES
:
print
"Preparing"
,
dictionary
.
name
dictionary
.
prepare_source
(
cluster
)
print
"Prepared"
...
...
@@ -363,102 +344,3 @@ def test_ranged_dictionaries(started_cluster, fold):
for
query
,
answer
in
queries_with_answers
:
print
query
assert
node
.
query
(
query
)
==
str
(
answer
)
+
'
\n
'
@
pytest
.
mark
.
parametrize
(
"fold"
,
list
(
range
(
10
)))
def
test_key_value_simple_dictionaries
(
started_cluster
,
fold
):
fields
=
FIELDS
[
"simple"
]
values
=
VALUES
[
"simple"
]
data
=
[
Row
(
fields
,
vals
)
for
vals
in
values
]
all_simple_dicts
=
[
d
for
d
in
DICTIONARIES_KV
if
d
.
structure
.
layout
.
layout_type
==
"simple"
]
simple_dicts
=
get_dictionaries
(
fold
,
10
,
all_simple_dicts
)
for
dct
in
simple_dicts
:
queries_with_answers
=
[]
local_data
=
[]
for
row
in
data
:
local_fields
=
dct
.
get_fields
()
local_values
=
[
row
.
get_value_by_name
(
field
.
name
)
for
field
in
local_fields
if
row
.
has_field
(
field
.
name
)]
local_data
.
append
(
Row
(
local_fields
,
local_values
))
dct
.
load_data
(
local_data
)
node
.
query
(
"system reload dictionary {}"
.
format
(
dct
.
name
))
print
'name: '
,
dct
.
name
for
row
in
local_data
:
print
dct
.
get_fields
()
for
field
in
dct
.
get_fields
():
print
field
.
name
,
field
.
is_key
if
not
field
.
is_key
:
for
query
in
dct
.
get_select_get_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
row
.
get_value_by_name
(
field
.
name
)))
for
query
in
dct
.
get_select_has_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
1
))
for
query
in
dct
.
get_select_get_or_default_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
field
.
default_value_for_get
))
if
dct
.
structure
.
has_hierarchy
:
for
query
in
dct
.
get_hierarchical_queries
(
data
[
0
]):
queries_with_answers
.
append
((
query
,
[
1
]))
for
query
in
dct
.
get_hierarchical_queries
(
data
[
1
]):
queries_with_answers
.
append
((
query
,
[
2
,
1
]))
for
query
in
dct
.
get_is_in_queries
(
data
[
0
],
data
[
1
]):
queries_with_answers
.
append
((
query
,
0
))
for
query
in
dct
.
get_is_in_queries
(
data
[
1
],
data
[
0
]):
queries_with_answers
.
append
((
query
,
1
))
for
query
,
answer
in
queries_with_answers
:
print
query
if
isinstance
(
answer
,
list
):
answer
=
str
(
answer
).
replace
(
' '
,
''
)
assert
node
.
query
(
query
)
==
str
(
answer
)
+
'
\n
'
@
pytest
.
mark
.
parametrize
(
"fold"
,
list
(
range
(
10
)))
def
test_key_value_complex_dictionaries
(
started_cluster
,
fold
):
fields
=
FIELDS
[
"complex"
]
values
=
VALUES
[
"complex"
]
data
=
[
Row
(
fields
,
vals
)
for
vals
in
values
]
all_complex_dicts
=
[
d
for
d
in
DICTIONARIES_KV
if
d
.
structure
.
layout
.
layout_type
==
"complex"
]
complex_dicts
=
get_dictionaries
(
fold
,
10
,
all_complex_dicts
)
for
dct
in
complex_dicts
:
dct
.
load_data
(
data
)
node
.
query
(
"system reload dictionaries"
)
for
dct
in
complex_dicts
:
queries_with_answers
=
[]
local_data
=
[]
for
row
in
data
:
local_fields
=
dct
.
get_fields
()
local_values
=
[
row
.
get_value_by_name
(
field
.
name
)
for
field
in
local_fields
if
row
.
has_field
(
field
.
name
)]
local_data
.
append
(
Row
(
local_fields
,
local_values
))
dct
.
load_data
(
local_data
)
node
.
query
(
"system reload dictionary {}"
.
format
(
dct
.
name
))
for
row
in
local_data
:
for
field
in
dct
.
get_fields
():
if
not
field
.
is_key
:
for
query
in
dct
.
get_select_get_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
row
.
get_value_by_name
(
field
.
name
)))
for
query
in
dct
.
get_select_has_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
1
))
for
query
in
dct
.
get_select_get_or_default_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
field
.
default_value_for_get
))
for
query
,
answer
in
queries_with_answers
:
print
query
assert
node
.
query
(
query
)
==
str
(
answer
)
+
'
\n
'
tests/integration/test_dictionaries_redis/__init__.py
0 → 100644
浏览文件 @
9493532f
tests/integration/test_dictionaries_redis/test.py
0 → 100644
浏览文件 @
9493532f
import
os
import
pytest
import
redis
from
helpers.cluster
import
ClickHouseCluster
from
helpers.dictionary
import
Field
,
Row
,
Dictionary
,
DictionaryStructure
,
Layout
from
helpers.external_sources
import
SourceRedis
cluster
=
None
SCRIPT_DIR
=
os
.
path
.
dirname
(
os
.
path
.
realpath
(
__file__
))
dict_configs_path
=
os
.
path
.
join
(
SCRIPT_DIR
,
'configs/dictionaries'
)
node
=
None
KEY_FIELDS
=
{
"simple"
:
[
Field
(
"KeyField"
,
'UInt64'
,
is_key
=
True
,
default_value_for_get
=
9999999
)
],
"complex"
:
[
Field
(
"KeyField1"
,
'UInt64'
,
is_key
=
True
,
default_value_for_get
=
9999999
),
Field
(
"KeyField2"
,
'String'
,
is_key
=
True
,
default_value_for_get
=
'xxxxxxxxx'
),
]
}
KEY_VALUES
=
{
"simple"
:
[
[
1
],
[
2
]
],
"complex"
:
[
[
1
,
'world'
],
[
2
,
'qwerty2'
]
]
}
FIELDS
=
[
Field
(
"UInt8_"
,
'UInt8'
,
default_value_for_get
=
55
),
Field
(
"UInt16_"
,
'UInt16'
,
default_value_for_get
=
66
),
Field
(
"UInt32_"
,
'UInt32'
,
default_value_for_get
=
77
),
Field
(
"UInt64_"
,
'UInt64'
,
default_value_for_get
=
88
),
Field
(
"Int8_"
,
'Int8'
,
default_value_for_get
=-
55
),
Field
(
"Int16_"
,
'Int16'
,
default_value_for_get
=-
66
),
Field
(
"Int32_"
,
'Int32'
,
default_value_for_get
=-
77
),
Field
(
"Int64_"
,
'Int64'
,
default_value_for_get
=-
88
),
Field
(
"UUID_"
,
'UUID'
,
default_value_for_get
=
'550e8400-0000-0000-0000-000000000000'
),
Field
(
"Date_"
,
'Date'
,
default_value_for_get
=
'2018-12-30'
),
Field
(
"DateTime_"
,
'DateTime'
,
default_value_for_get
=
'2018-12-30 00:00:00'
),
Field
(
"String_"
,
'String'
,
default_value_for_get
=
'hi'
),
Field
(
"Float32_"
,
'Float32'
,
default_value_for_get
=
555.11
),
Field
(
"Float64_"
,
'Float64'
,
default_value_for_get
=
777.11
),
]
VALUES
=
[
[
22
,
3
],
[
333
,
4
],
[
4444
,
5
],
[
55555
,
6
],
[
-
6
,
-
7
],
[
-
77
,
-
8
],
[
-
888
,
-
9
],
[
-
999
,
-
10
],
[
'550e8400-e29b-41d4-a716-446655440003'
,
'550e8400-e29b-41d4-a716-446655440002'
],
[
'1973-06-28'
,
'1978-06-28'
],
[
'1985-02-28 23:43:25'
,
'1986-02-28 23:42:25'
],
[
'hello'
,
'hello'
],
[
22.543
,
21.543
],
[
3332154213.4
,
3222154213.4
],
]
LAYOUTS
=
[
Layout
(
"flat"
),
Layout
(
"hashed"
),
Layout
(
"cache"
),
Layout
(
"complex_key_hashed"
),
# Layout("complex_key_cache"), # Currently not supported
Layout
(
"direct"
),
# Layout("complex_key_direct") # Currently not supported
]
DICTIONARIES
=
[]
def
get_dict
(
source
,
layout
,
fields
,
suffix_name
=
''
):
global
dict_configs_path
structure
=
DictionaryStructure
(
layout
,
fields
)
dict_name
=
source
.
name
+
"_"
+
layout
.
name
+
'_'
+
suffix_name
dict_path
=
os
.
path
.
join
(
dict_configs_path
,
dict_name
+
'.xml'
)
dictionary
=
Dictionary
(
dict_name
,
structure
,
source
,
dict_path
,
"table_"
+
dict_name
,
fields
)
dictionary
.
generate_config
()
return
dictionary
def
setup_module
(
module
):
global
DICTIONARIES
global
cluster
global
node
global
dict_configs_path
for
f
in
os
.
listdir
(
dict_configs_path
):
os
.
remove
(
os
.
path
.
join
(
dict_configs_path
,
f
))
for
i
,
field
in
enumerate
(
FIELDS
):
DICTIONARIES
.
append
([])
sources
=
[]
sources
.
append
(
SourceRedis
(
"RedisSimple"
,
"localhost"
,
"6380"
,
"redis1"
,
"6379"
,
""
,
"clickhouse"
,
i
*
2
,
storage_type
=
"simple"
))
sources
.
append
(
SourceRedis
(
"RedisHash"
,
"localhost"
,
"6380"
,
"redis1"
,
"6379"
,
""
,
"clickhouse"
,
i
*
2
+
1
,
storage_type
=
"hash_map"
))
for
source
in
sources
:
for
layout
in
LAYOUTS
:
if
not
source
.
compatible_with_layout
(
layout
):
print
"Source"
,
source
.
name
,
"incompatible with layout"
,
layout
.
name
continue
fields
=
KEY_FIELDS
[
layout
.
layout_type
]
+
[
field
]
DICTIONARIES
[
i
].
append
(
get_dict
(
source
,
layout
,
fields
,
field
.
name
))
main_configs
=
[]
dictionaries
=
[]
for
fname
in
os
.
listdir
(
dict_configs_path
):
dictionaries
.
append
(
os
.
path
.
join
(
dict_configs_path
,
fname
))
cluster
=
ClickHouseCluster
(
__file__
)
node
=
cluster
.
add_instance
(
'node'
,
main_configs
=
main_configs
,
dictionaries
=
dictionaries
,
with_redis
=
True
)
@
pytest
.
fixture
(
scope
=
"module"
,
autouse
=
True
)
def
started_cluster
():
try
:
cluster
.
start
()
assert
len
(
FIELDS
)
==
len
(
VALUES
)
for
dicts
in
DICTIONARIES
:
for
dictionary
in
dicts
:
print
"Preparing"
,
dictionary
.
name
dictionary
.
prepare_source
(
cluster
)
print
"Prepared"
yield
cluster
finally
:
cluster
.
shutdown
()
@
pytest
.
mark
.
parametrize
(
"id"
,
range
(
len
(
FIELDS
)))
def
test_redis_dictionaries
(
started_cluster
,
id
):
print
'id:'
,
id
dicts
=
DICTIONARIES
[
id
]
values
=
VALUES
[
id
]
field
=
FIELDS
[
id
]
node
.
query
(
"system reload dictionaries"
)
for
dct
in
dicts
:
data
=
[]
dict_type
=
dct
.
structure
.
layout
.
layout_type
key_fields
=
KEY_FIELDS
[
dict_type
]
key_values
=
KEY_VALUES
[
dict_type
]
for
key_value
,
value
in
zip
(
key_values
,
values
):
data
.
append
(
Row
(
key_fields
+
[
field
],
key_value
+
[
value
]))
dct
.
load_data
(
data
)
queries_with_answers
=
[]
for
row
in
data
:
for
query
in
dct
.
get_select_get_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
row
.
get_value_by_name
(
field
.
name
)))
for
query
in
dct
.
get_select_has_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
1
))
for
query
in
dct
.
get_select_get_or_default_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
field
.
default_value_for_get
))
node
.
query
(
"system reload dictionary {}"
.
format
(
dct
.
name
))
for
query
,
answer
in
queries_with_answers
:
print
query
assert
node
.
query
(
query
)
==
str
(
answer
)
+
'
\n
'
# Checks, that dictionaries can be reloaded.
node
.
query
(
"system reload dictionaries"
)
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录