Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
2bb54b0b
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
2bb54b0b
编写于
2月 22, 2019
作者:
A
alesapin
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add http tests and different queries
上级
0d2e562b
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
264 addition
and
72 deletion
+264
-72
dbms/tests/integration/helpers/cluster.py
dbms/tests/integration/helpers/cluster.py
+2
-2
dbms/tests/integration/test_external_dictionaries/configs/dictionaries/.gitkeep
.../test_external_dictionaries/configs/dictionaries/.gitkeep
+0
-0
dbms/tests/integration/test_external_dictionaries/dictionary.py
...ests/integration/test_external_dictionaries/dictionary.py
+78
-11
dbms/tests/integration/test_external_dictionaries/external_sources.py
...ntegration/test_external_dictionaries/external_sources.py
+23
-8
dbms/tests/integration/test_external_dictionaries/fake_cert.pem
...ests/integration/test_external_dictionaries/fake_cert.pem
+49
-0
dbms/tests/integration/test_external_dictionaries/http_server.py
...sts/integration/test_external_dictionaries/http_server.py
+43
-0
dbms/tests/integration/test_external_dictionaries/test.py
dbms/tests/integration/test_external_dictionaries/test.py
+69
-51
未找到文件。
dbms/tests/integration/helpers/cluster.py
浏览文件 @
2bb54b0b
...
...
@@ -457,10 +457,10 @@ class ClickHouseInstance:
return
self
.
client
.
get_query_request
(
*
args
,
**
kwargs
)
def
exec_in_container
(
self
,
cmd
,
**
kwargs
):
def
exec_in_container
(
self
,
cmd
,
detach
=
False
,
**
kwargs
):
container
=
self
.
get_docker_handle
()
exec_id
=
self
.
docker_client
.
api
.
exec_create
(
container
.
id
,
cmd
,
**
kwargs
)
output
=
self
.
docker_client
.
api
.
exec_start
(
exec_id
,
detach
=
False
)
output
=
self
.
docker_client
.
api
.
exec_start
(
exec_id
,
detach
=
detach
)
output
=
output
.
decode
(
'utf8'
)
exit_code
=
self
.
docker_client
.
api
.
exec_inspect
(
exec_id
)[
'ExitCode'
]
...
...
dbms/tests/integration/test_external_dictionaries/configs/dictionaries/.gitkeep
0 → 100644
浏览文件 @
2bb54b0b
dbms/tests/integration/test_external_dictionaries/dictionary.py
浏览文件 @
2bb54b0b
...
...
@@ -48,7 +48,7 @@ class Row(object):
class
Field
(
object
):
def
__init__
(
self
,
name
,
field_type
,
is_key
=
False
,
is_range_key
=
False
,
default
=
None
,
hierarchical
=
False
,
range_hash_type
=
None
):
def
__init__
(
self
,
name
,
field_type
,
is_key
=
False
,
is_range_key
=
False
,
default
=
None
,
hierarchical
=
False
,
range_hash_type
=
None
,
default_value_for_get
=
None
):
self
.
name
=
name
self
.
field_type
=
field_type
self
.
is_key
=
is_key
...
...
@@ -57,6 +57,7 @@ class Field(object):
self
.
range_hash_type
=
range_hash_type
self
.
is_range
=
self
.
range_hash_type
is
not
None
self
.
is_range_key
=
is_range_key
self
.
default_value_for_get
=
default_value_for_get
def
get_attribute_str
(
self
):
return
'''
...
...
@@ -157,21 +158,26 @@ class DictionaryStructure(object):
return
fields_strs
def
get_dict_get_expression
(
self
,
dict_name
,
field
,
row
):
def
_get_dict_get_common_expression
(
self
,
dict_name
,
field
,
row
,
or_default
,
with_type
,
has
):
if
field
in
self
.
keys
:
raise
Exception
(
"Trying to receive key field {} from dictionary"
.
format
(
field
.
name
))
if
not
self
.
layout
.
is_complex
:
key_expr
=
'toUInt64({})'
.
format
(
row
.
data
[
self
.
keys
[
0
].
name
])
if
not
or_default
:
key_expr
=
', toUInt64({})'
.
format
(
row
.
data
[
self
.
keys
[
0
].
name
])
else
:
key_expr
=
', toUInt64({})'
.
format
(
self
.
keys
[
0
].
default_value_for_get
)
else
:
key_exprs_strs
=
[]
for
key
in
self
.
keys
:
val
=
row
.
data
[
key
.
name
]
if
not
or_default
:
val
=
row
.
data
[
key
.
name
]
else
:
val
=
key
.
default_value_for_get
if
isinstance
(
val
,
str
):
val
=
"'"
+
val
+
"'"
key_exprs_strs
.
append
(
'to{type}({value})'
.
format
(
type
=
key
.
field_type
,
value
=
val
))
key_expr
=
'('
+
','
.
join
(
key_exprs_strs
)
+
')'
key_expr
=
'
,
('
+
','
.
join
(
key_exprs_strs
)
+
')'
date_expr
=
''
if
self
.
layout
.
is_ranged
:
...
...
@@ -182,14 +188,69 @@ class DictionaryStructure(object):
date_expr
=
', '
+
val
return
"dictGet{field_type}('{dict_name}', '{field_name}', {key_expr}{date_expr})"
.
format
(
field_type
=
field
.
field_type
,
if
or_default
:
raise
Exception
(
"Can create 'dictGetOrDefault' query for ranged dictionary"
)
if
or_default
:
or_default_expr
=
'OrDefault'
if
field
.
default_value_for_get
is
None
:
raise
Exception
(
"Can create 'dictGetOrDefault' query for field {} without default_value_for_get"
.
format
(
field
.
name
))
val
=
field
.
default_value_for_get
if
isinstance
(
val
,
str
):
val
=
"'"
+
val
+
"'"
default_value_for_get
=
', to{type}({value})'
.
format
(
type
=
field
.
field_type
,
value
=
val
)
else
:
or_default_expr
=
''
default_value_for_get
=
''
if
with_type
:
field_type
=
field
.
field_type
else
:
field_type
=
''
field_name
=
", '"
+
field
.
name
+
"'"
if
has
:
what
=
"Has"
field_type
=
''
or_default
=
''
field_name
=
''
date_expr
=
''
def_for_get
=
''
else
:
what
=
"Get"
return
"dict{what}{field_type}{or_default}('{dict_name}'{field_name}{key_expr}{date_expr}{def_for_get})"
.
format
(
what
=
what
,
field_type
=
field_type
,
dict_name
=
dict_name
,
field_name
=
field
.
name
,
field_name
=
field
_
name
,
key_expr
=
key_expr
,
date_expr
=
date_expr
,
or_default
=
or_default_expr
,
def_for_get
=
default_value_for_get
,
)
def
get_get_expressions
(
self
,
dict_name
,
field
,
row
):
return
[
self
.
_get_dict_get_common_expression
(
dict_name
,
field
,
row
,
or_default
=
False
,
with_type
=
False
,
has
=
False
),
self
.
_get_dict_get_common_expression
(
dict_name
,
field
,
row
,
or_default
=
False
,
with_type
=
True
,
has
=
False
),
]
def
get_get_or_default_expressions
(
self
,
dict_name
,
field
,
row
):
if
not
self
.
layout
.
is_ranged
:
return
[
self
.
_get_dict_get_common_expression
(
dict_name
,
field
,
row
,
or_default
=
True
,
with_type
=
False
,
has
=
False
),
self
.
_get_dict_get_common_expression
(
dict_name
,
field
,
row
,
or_default
=
True
,
with_type
=
True
,
has
=
False
),
]
return
[]
def
get_has_expressions
(
self
,
dict_name
,
field
,
row
):
if
not
self
.
layout
.
is_ranged
:
return
[
self
.
_get_dict_get_common_expression
(
dict_name
,
field
,
row
,
or_default
=
False
,
with_type
=
False
,
has
=
True
)]
return
[]
class
Dictionary
(
object
):
def
__init__
(
self
,
name
,
structure
,
source
,
config_path
,
table_name
):
...
...
@@ -230,8 +291,14 @@ class Dictionary(object):
self
.
source
.
load_data
(
data
,
self
.
table_name
)
def
get_select_query
(
self
,
field
,
row
):
return
'select {}'
.
format
(
self
.
structure
.
get_dict_get_expression
(
self
.
name
,
field
,
row
))
def
get_select_get_queries
(
self
,
field
,
row
):
return
[
'select {}'
.
format
(
expr
)
for
expr
in
self
.
structure
.
get_get_expressions
(
self
.
name
,
field
,
row
)]
def
get_select_get_or_default_queries
(
self
,
field
,
row
):
return
[
'select {}'
.
format
(
expr
)
for
expr
in
self
.
structure
.
get_get_or_default_expressions
(
self
.
name
,
field
,
row
)]
def
get_select_has_queries
(
self
,
field
,
row
):
return
[
'select {}'
.
format
(
expr
)
for
expr
in
self
.
structure
.
get_has_expressions
(
self
.
name
,
field
,
row
)]
def
is_complex
(
self
):
return
self
.
structure
.
layout
.
is_complex
dbms/tests/integration/test_external_dictionaries/external_sources.py
浏览文件 @
2bb54b0b
...
...
@@ -2,9 +2,8 @@
import
warnings
import
pymysql.cursors
import
pymongo
import
subprocess
import
copy
import
base64
import
os
class
ExternalSource
(
object
):
def
__init__
(
self
,
name
,
internal_hostname
,
internal_port
,
...
...
@@ -305,9 +304,11 @@ class SourceExecutableHashed(_SourceExecutableBase):
class
SourceHTTPBase
(
ExternalSource
):
PORT_COUNTER
=
5555
def
get_source_str
(
self
,
table_name
):
self
.
http_port
=
5555
self
.
http_port
=
SourceHTTPBase
.
PORT_COUNTER
url
=
"{schema}://{host}:{port}/"
.
format
(
schema
=
self
.
_get_schema
(),
host
=
self
.
docker_hostname
,
port
=
self
.
http_port
)
SourceHTTPBase
.
PORT_COUNTER
+=
1
return
'''
<http>
<url>{url}</url>
...
...
@@ -317,14 +318,18 @@ class SourceHTTPBase(ExternalSource):
def
prepare
(
self
,
structure
,
table_name
,
cluster
):
self
.
node
=
cluster
.
instances
[
self
.
docker_hostname
]
path
=
"/"
+
table_name
+
".tsv"
self
.
node
.
exec_in_container
([
"bash"
,
"-c"
,
"touch {}"
.
format
(
path
)])
self
.
node
.
copy_file_to_container
(
'./http_server.py'
,
'/http_server.py'
)
script_dir
=
os
.
path
.
dirname
(
os
.
path
.
realpath
(
__file__
))
self
.
node
.
copy_file_to_container
(
os
.
path
.
join
(
script_dir
,
'./http_server.py'
),
'/http_server.py'
)
self
.
node
.
copy_file_to_container
(
os
.
path
.
join
(
script_dir
,
'./fake_cert.pem'
),
'/fake_cert.pem'
)
self
.
node
.
exec_in_container
([
"bash"
,
"-c"
,
"python
/http_server.py --data-path /{tbl}.tsv --schema={schema} --host={host} --port={port}
"
.
format
(
tbl
=
table_name
,
schema
=
self
.
_get_schema
(),
host
=
self
.
docker_hostname
,
port
=
self
.
http_port
)
])
"python
2 /http_server.py --data-path={tbl} --schema={schema} --host={host} --port={port} --cert-path=/fake_cert.pem
"
.
format
(
tbl
=
path
,
schema
=
self
.
_get_schema
(),
host
=
self
.
docker_hostname
,
port
=
self
.
http_port
)
]
,
detach
=
True
)
self
.
ordered_names
=
structure
.
get_ordered_names
()
self
.
prepared
=
True
...
...
@@ -339,3 +344,13 @@ class SourceHTTPBase(ExternalSource):
str_data
=
'
\t
'
.
join
(
sorted_row
)
self
.
node
.
exec_in_container
([
"bash"
,
"-c"
,
"echo
\"
{row}
\"
>> {fname}"
.
format
(
row
=
str_data
,
fname
=
path
)])
class
SourceHTTP
(
SourceHTTPBase
):
def
_get_schema
(
self
):
return
"http"
class
SourceHTTPS
(
SourceHTTPBase
):
def
_get_schema
(
self
):
return
"https"
dbms/tests/integration/test_external_dictionaries/fake_cert.pem
0 → 100644
浏览文件 @
2bb54b0b
-----BEGIN PRIVATE KEY-----
MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDDHnGYqN/ztiFE
rMQizbYiEpI/q/91bCDQ+xRes+gucKrr4qvQbosANYfpXgsaGizH24CpAXDvnFwC
oHqPmotHunJvG9uKiVvshy+tx1SNLZEN9DySri+8V+8fetn5PFxWQsKclMGCypyE
REV6H0vflPWmZRZWvAb5aaIxcRa2m3bTVUZPuY0wzCtc+ELPQ/sRc62gWH4bMlBo
0Wdai4+wcmpdcSR+rlZVDPt+ysxF/PcJFMAQ9CIRJRhXuK7Q/XCmAkagpH9tPPwY
SDMONTPhumXY7gCX4lmV9CflGJ6IpGmpEL04Rpr3gAcvz/w4JiMXgGpvtDjiJku9
qOdCYS/FAgMBAAECggEBAL/miULjlJ9VWZL5eE3ilGcebMhCmZUbK4td5cLenlRO
a0xkOydcEUm7XFihLboWVEScFgYibLi8x6Gtw9zI2oNJVJMCiwHN5qLSsonvqbDQ
SAG5XHnG5xwOQBht80O1ofsU3eKyS0AflaBgpRRfA3h6QL/OXBIiC5nx0ptd5kDh
HR0IHUcleBHt8I0d/PZbQE9oMOBlnMf8v2jGe80JXscQt2UabA/quCalDihhDt5J
qySfh4mDOrBOQEsmO/C1JCztQ6WZ2FVwRiITb/fRmsPadKJsIiMyy2w6NmP96v2a
V2ZqMvz9OZym8M2is4HR2pbn8XJ6vmW52fwNQhpWDgECgYEA8aiqF5df3j8YEDAX
XVAhIaubSLcS50qSk/p0/ZS9ETR1Uv8zjJDs6xBVBd4xXe/G2/XvvV6sGp4JcW3V
U66Ll3S1veMlnvCTjZUEi931EJbIdoyGACEG19QIVteSEhQkoSOk/Zx1lFSVm9UZ
hUV4JvWifQvLetS/v6MhnxSbTdUCgYEAzrK7+0gVT0a0szMs7CbeQVm80EWcqPea
p5jyLQHu+7vzcC8c9RRlqBPkxeG9BTt0sbBBJTrtvls15QaFoKCtTyjnrrLEHqu3
VZfIpjjrIhhvoRWP3A3r4DFMDGm/TOTUWEMSPJPXKe3uVm3buwVXWj4ipvhnAdr5
kJ+x1YqNIjECgYEAo0ISHzv53Vh8tjr3HehLacbYcmiUEcOUgPo8XTBGBsCM3pRg
S/+Av1FaT0uLyG17yBA/dYzm8liAAqxz6UPLNHf5bB5vxQ+8b3MUDjXWIO3s4gIP
aTjmuZqaQ6kBGsuW73H4PgmceagnJo7x3dJP2OoraxUz03i1Tg80YJd4UD0CgYBC
dzL/gJRpo6DjpuchIPaDKSoQBvJzWvt+PS5SzrZceHm1b1DudhqiS5NbFlXD4vSJ
VtX79NESTx4rgUdi+YgBVnP5tz5dZnZTrbU1zkO9+QGcWOSjrE5XD0MXEsITJdoq
b5bjp96eewYTAMyRfQwz1psp+eKVtCZgHRoAQsdTYQKBgQC7yBABJ4LDTie2C2n0
itO7SRT1tMfkNx8gK9RrgGawBUhD1EokmOKk+O1Ht6Cx7hqCd3Hsa4zc9se++jV1
Er+T8LW8FOFfAwtv8xggJtA8h6U8n6gIoq0EsSsWREJ4m9fDfZQnVTj8IPYvPHMr
Jv++IPqtFGG4O8IeWG+HY8mHxQ==
-----END PRIVATE KEY-----
-----BEGIN CERTIFICATE-----
MIIDYDCCAkigAwIBAgIJAKSJ3I0ORzjtMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV
BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
aWRnaXRzIFB0eSBMdGQwHhcNMTkwMjIyMDgxNTIzWhcNMjAwMjIyMDgxNTIzWjBF
MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50
ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
CgKCAQEAwx5xmKjf87YhRKzEIs22IhKSP6v/dWwg0PsUXrPoLnCq6+Kr0G6LADWH
6V4LGhosx9uAqQFw75xcAqB6j5qLR7pybxvbiolb7IcvrcdUjS2RDfQ8kq4vvFfv
H3rZ+TxcVkLCnJTBgsqchERFeh9L35T1pmUWVrwG+WmiMXEWtpt201VGT7mNMMwr
XPhCz0P7EXOtoFh+GzJQaNFnWouPsHJqXXEkfq5WVQz7fsrMRfz3CRTAEPQiESUY
V7iu0P1wpgJGoKR/bTz8GEgzDjUz4bpl2O4Al+JZlfQn5RieiKRpqRC9OEaa94AH
L8/8OCYjF4Bqb7Q44iZLvajnQmEvxQIDAQABo1MwUTAdBgNVHQ4EFgQU6P39PMY3
jRgJM0svz9XpHH8z7xUwHwYDVR0jBBgwFoAU6P39PMY3jRgJM0svz9XpHH8z7xUw
DwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAdIKBKlCIprCDGTtn
xatBlcpkbys4hQhHwkWn5aAPKE2oZlUOTEe90xxLJuciK+vCXTwQ3mgjGFc+ioAo
B7m3VL1DLmHCw5DQ2T/g8TjVjlKoaCj+9SZZPga5ygYJChx5HKFO4eek9stWo6hA
BmXndKhdX7mphUoSqUnQ+RwQ9XA0n6eTPqXAThWVqvLQgDj7Msz1XeFfqFqyD9MN
RocFg87aASTtwxYneG3IZCOQudlbHaRuEflHjlty2V5mNPjzcS2QK598i/5vmIoD
ZiUBXg+P8n+dklEa4qnQplDKERD20GtDgWtgYrfmpspLWNv8/bZ4h4gmGsH0+3uz
dHQNQA==
-----END CERTIFICATE-----
dbms/tests/integration/test_external_dictionaries/http_server.py
0 → 100644
浏览文件 @
2bb54b0b
# -*- coding: utf-8 -*-
import
argparse
from
BaseHTTPServer
import
BaseHTTPRequestHandler
,
HTTPServer
import
ssl
import
csv
import
os
def
start_server
(
server_address
,
cert_path
,
data_path
,
schema
):
class
TSVHTTPHandler
(
BaseHTTPRequestHandler
):
def
_set_headers
(
self
):
self
.
send_response
(
200
)
self
.
send_header
(
'Content-type'
,
'text/tsv'
)
self
.
end_headers
()
def
do_GET
(
self
):
self
.
_set_headers
()
with
open
(
data_path
,
'r'
)
as
fl
:
reader
=
csv
.
reader
(
fl
,
delimiter
=
'
\t
'
)
for
row
in
reader
:
self
.
wfile
.
write
(
'
\t
'
.
join
(
row
)
+
'
\n
'
)
return
def
do_POST
(
self
):
return
self
.
do_GET
()
httpd
=
HTTPServer
(
server_address
,
TSVHTTPHandler
)
if
schema
==
'https'
:
httpd
.
socket
=
ssl
.
wrap_socket
(
httpd
.
socket
,
certfile
=
cert_path
,
server_side
=
True
)
httpd
.
serve_forever
()
if
__name__
==
"__main__"
:
parser
=
argparse
.
ArgumentParser
(
description
=
"Simple HTTP server returns data from file"
)
parser
.
add_argument
(
"--data-path"
,
required
=
True
)
parser
.
add_argument
(
"--schema"
,
choices
=
(
"http"
,
"https"
),
required
=
True
)
parser
.
add_argument
(
"--host"
,
default
=
"localhost"
)
parser
.
add_argument
(
"--port"
,
default
=
5555
,
type
=
int
)
parser
.
add_argument
(
"--cert-path"
,
default
=
"./fake_cert.pem"
)
args
=
parser
.
parse_args
()
start_server
((
args
.
host
,
args
.
port
),
args
.
cert_path
,
args
.
data_path
,
args
.
schema
)
dbms/tests/integration/test_external_dictionaries/test.py
浏览文件 @
2bb54b0b
...
...
@@ -5,64 +5,65 @@ import time
from
helpers.cluster
import
ClickHouseCluster
from
dictionary
import
Field
,
Row
,
Dictionary
,
DictionaryStructure
,
Layout
from
external_sources
import
SourceMySQL
,
SourceMongo
,
SourceClickHouse
,
SourceFile
,
SourceExecutableCache
,
SourceExecutableHashed
from
external_sources
import
SourceHTTP
,
SourceHTTPS
SCRIPT_DIR
=
os
.
path
.
dirname
(
os
.
path
.
realpath
(
__file__
))
FIELDS
=
{
"simple"
:
[
Field
(
"KeyField"
,
'UInt64'
,
is_key
=
True
),
Field
(
"UInt8_"
,
'UInt8'
),
Field
(
"UInt16_"
,
'UInt16'
),
Field
(
"UInt32_"
,
'UInt32'
),
Field
(
"UInt64_"
,
'UInt64'
),
Field
(
"Int8_"
,
'Int8'
),
Field
(
"Int16_"
,
'Int16'
),
Field
(
"Int32_"
,
'Int32'
),
Field
(
"Int64_"
,
'Int64'
),
Field
(
"UUID_"
,
'UUID'
),
Field
(
"Date_"
,
'Date'
),
Field
(
"DateTime_"
,
'DateTime'
),
Field
(
"String_"
,
'String'
),
Field
(
"Float32_"
,
'Float32'
),
Field
(
"Float64_"
,
'Float64'
),
Field
(
"KeyField"
,
'UInt64'
,
is_key
=
True
,
default_value_for_get
=
9999999
),
Field
(
"UInt8_"
,
'UInt8'
,
default_value_for_get
=
55
),
Field
(
"UInt16_"
,
'UInt16'
,
default_value_for_get
=
66
),
Field
(
"UInt32_"
,
'UInt32'
,
default_value_for_get
=
77
),
Field
(
"UInt64_"
,
'UInt64'
,
default_value_for_get
=
88
),
Field
(
"Int8_"
,
'Int8'
,
default_value_for_get
=-
55
),
Field
(
"Int16_"
,
'Int16'
,
default_value_for_get
=-
66
),
Field
(
"Int32_"
,
'Int32'
,
default_value_for_get
=-
77
),
Field
(
"Int64_"
,
'Int64'
,
default_value_for_get
=-
88
),
Field
(
"UUID_"
,
'UUID'
,
default_value_for_get
=
'550e8400-0000-0000-0000-000000000000'
),
Field
(
"Date_"
,
'Date'
,
default_value_for_get
=
'2018-12-30'
),
Field
(
"DateTime_"
,
'DateTime'
,
default_value_for_get
=
'2018-12-30 00:00:00'
),
Field
(
"String_"
,
'String'
,
default_value_for_get
=
'hi'
),
Field
(
"Float32_"
,
'Float32'
,
default_value_for_get
=
555.11
),
Field
(
"Float64_"
,
'Float64'
,
default_value_for_get
=
777.11
),
],
"complex"
:
[
Field
(
"KeyField1"
,
'UInt64'
,
is_key
=
True
),
Field
(
"KeyField2"
,
'String'
,
is_key
=
True
),
Field
(
"UInt8_"
,
'UInt8'
),
Field
(
"UInt16_"
,
'UInt16'
),
Field
(
"UInt32_"
,
'UInt32'
),
Field
(
"UInt64_"
,
'UInt64'
),
Field
(
"Int8_"
,
'Int8'
),
Field
(
"Int16_"
,
'Int16'
),
Field
(
"Int32_"
,
'Int32'
),
Field
(
"Int64_"
,
'Int64'
),
Field
(
"UUID_"
,
'UUID'
),
Field
(
"Date_"
,
'Date'
),
Field
(
"DateTime_"
,
'DateTime'
),
Field
(
"String_"
,
'String'
),
Field
(
"Float32_"
,
'Float32'
),
Field
(
"Float64_"
,
'Float64'
),
Field
(
"KeyField1"
,
'UInt64'
,
is_key
=
True
,
default_value_for_get
=
9999999
),
Field
(
"KeyField2"
,
'String'
,
is_key
=
True
,
default_value_for_get
=
'xxxxxxxxx'
),
Field
(
"UInt8_"
,
'UInt8'
,
default_value_for_get
=
55
),
Field
(
"UInt16_"
,
'UInt16'
,
default_value_for_get
=
66
),
Field
(
"UInt32_"
,
'UInt32'
,
default_value_for_get
=
77
),
Field
(
"UInt64_"
,
'UInt64'
,
default_value_for_get
=
88
),
Field
(
"Int8_"
,
'Int8'
,
default_value_for_get
=-
55
),
Field
(
"Int16_"
,
'Int16'
,
default_value_for_get
=-
66
),
Field
(
"Int32_"
,
'Int32'
,
default_value_for_get
=-
77
),
Field
(
"Int64_"
,
'Int64'
,
default_value_for_get
=-
88
),
Field
(
"UUID_"
,
'UUID'
,
default_value_for_get
=
'550e8400-0000-0000-0000-000000000000'
),
Field
(
"Date_"
,
'Date'
,
default_value_for_get
=
'2018-12-30'
),
Field
(
"DateTime_"
,
'DateTime'
,
default_value_for_get
=
'2018-12-30 00:00:00'
),
Field
(
"String_"
,
'String'
,
default_value_for_get
=
'hi'
),
Field
(
"Float32_"
,
'Float32'
,
default_value_for_get
=
555.11
),
Field
(
"Float64_"
,
'Float64'
,
default_value_for_get
=
777.11
),
],
"ranged"
:
[
Field
(
"KeyField1"
,
'UInt64'
,
is_key
=
True
),
Field
(
"KeyField2"
,
'Date'
,
is_range_key
=
True
),
Field
(
"StartDate"
,
'Date'
,
range_hash_type
=
'min'
),
Field
(
"EndDate"
,
'Date'
,
range_hash_type
=
'max'
),
Field
(
"UInt8_"
,
'UInt8'
),
Field
(
"UInt16_"
,
'UInt16'
),
Field
(
"UInt32_"
,
'UInt32'
),
Field
(
"UInt64_"
,
'UInt64'
),
Field
(
"Int8_"
,
'Int8'
),
Field
(
"Int16_"
,
'Int16'
),
Field
(
"Int32_"
,
'Int32'
),
Field
(
"Int64_"
,
'Int64'
),
Field
(
"UUID_"
,
'UUID'
),
Field
(
"Date_"
,
'Date'
),
Field
(
"DateTime_"
,
'DateTime'
),
Field
(
"String_"
,
'String'
),
Field
(
"Float32_"
,
'Float32'
),
Field
(
"Float64_"
,
'Float64'
),
Field
(
"UInt8_"
,
'UInt8'
,
default_value_for_get
=
55
),
Field
(
"UInt16_"
,
'UInt16'
,
default_value_for_get
=
66
),
Field
(
"UInt32_"
,
'UInt32'
,
default_value_for_get
=
77
),
Field
(
"UInt64_"
,
'UInt64'
,
default_value_for_get
=
88
),
Field
(
"Int8_"
,
'Int8'
,
default_value_for_get
=-
55
),
Field
(
"Int16_"
,
'Int16'
,
default_value_for_get
=-
66
),
Field
(
"Int32_"
,
'Int32'
,
default_value_for_get
=-
77
),
Field
(
"Int64_"
,
'Int64'
,
default_value_for_get
=-
88
),
Field
(
"UUID_"
,
'UUID'
,
default_value_for_get
=
'550e8400-0000-0000-0000-000000000000'
),
Field
(
"Date_"
,
'Date'
,
default_value_for_get
=
'2018-12-30'
),
Field
(
"DateTime_"
,
'DateTime'
,
default_value_for_get
=
'2018-12-30 00:00:00'
),
Field
(
"String_"
,
'String'
,
default_value_for_get
=
'hi'
),
Field
(
"Float32_"
,
'Float32'
,
default_value_for_get
=
555.11
),
Field
(
"Float64_"
,
'Float64'
,
default_value_for_get
=
777.11
),
]
}
...
...
@@ -77,6 +78,7 @@ LAYOUTS = [
]
SOURCES
=
[
# some kind of troubles with that dictionary
#SourceMongo("MongoDB", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"),
SourceMySQL
(
"MySQL"
,
"localhost"
,
"3308"
,
"mysql1"
,
"3306"
,
"root"
,
"clickhouse"
),
SourceClickHouse
(
"RemoteClickHouse"
,
"localhost"
,
"9000"
,
"clickhouse1"
,
"9000"
,
"default"
,
""
),
...
...
@@ -84,6 +86,8 @@ SOURCES = [
SourceFile
(
"File"
,
"localhost"
,
"9000"
,
"node"
,
"9000"
,
""
,
""
),
SourceExecutableHashed
(
"ExecutableHashed"
,
"localhost"
,
"9000"
,
"node"
,
"9000"
,
""
,
""
),
SourceExecutableCache
(
"ExecutableCache"
,
"localhost"
,
"9000"
,
"node"
,
"9000"
,
""
,
""
),
SourceHTTP
(
"SourceHTTP"
,
"localhost"
,
"9000"
,
"clickhouse1"
,
"9000"
,
""
,
""
),
SourceHTTPS
(
"SourceHTTPS"
,
"localhost"
,
"9000"
,
"clickhouse1"
,
"9000"
,
""
,
""
),
]
DICTIONARIES
=
[]
...
...
@@ -116,8 +120,8 @@ def setup_module(module):
for
fname
in
os
.
listdir
(
dict_configs_path
):
main_configs
.
append
(
os
.
path
.
join
(
dict_configs_path
,
fname
))
cluster
=
ClickHouseCluster
(
__file__
,
base_configs_dir
=
os
.
path
.
join
(
SCRIPT_DIR
,
'configs'
))
node
=
cluster
.
add_instance
(
'node'
,
main_configs
=
main_configs
,
with_mysql
=
True
,
with_mongo
=
True
)
cluster
.
add_instance
(
'clickhouse1'
)
node
=
cluster
.
add_instance
(
'node'
,
main_configs
=
main_configs
,
with_mysql
=
True
)
cluster
.
add_instance
(
'clickhouse1'
,
image
=
"python"
)
@
pytest
.
fixture
(
scope
=
"module"
)
def
started_cluster
():
...
...
@@ -131,7 +135,6 @@ def started_cluster():
yield
cluster
finally
:
pass
cluster
.
shutdown
()
...
...
@@ -153,7 +156,14 @@ def test_simple_dictionaries(started_cluster):
for
row
in
data
:
for
field
in
fields
:
if
not
field
.
is_key
:
queries_with_answers
.
append
((
dct
.
get_select_query
(
field
,
row
),
row
.
get_value_by_name
(
field
.
name
)))
for
query
in
dct
.
get_select_get_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
row
.
get_value_by_name
(
field
.
name
)))
for
query
in
dct
.
get_select_has_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
1
))
for
query
in
dct
.
get_select_get_or_default_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
field
.
default_value_for_get
))
for
query
,
answer
in
queries_with_answers
:
print
query
...
...
@@ -178,7 +188,14 @@ def test_complex_dictionaries(started_cluster):
for
row
in
data
:
for
field
in
fields
:
if
not
field
.
is_key
:
queries_with_answers
.
append
((
dct
.
get_select_query
(
field
,
row
),
row
.
get_value_by_name
(
field
.
name
)))
for
query
in
dct
.
get_select_get_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
row
.
get_value_by_name
(
field
.
name
)))
for
query
in
dct
.
get_select_has_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
1
))
for
query
in
dct
.
get_select_get_or_default_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
field
.
default_value_for_get
))
for
query
,
answer
in
queries_with_answers
:
print
query
...
...
@@ -205,7 +222,8 @@ def test_ranged_dictionaries(started_cluster):
for
row
in
data
:
for
field
in
fields
:
if
not
field
.
is_key
and
not
field
.
is_range
:
queries_with_answers
.
append
((
dct
.
get_select_query
(
field
,
row
),
row
.
get_value_by_name
(
field
.
name
)))
for
query
in
dct
.
get_select_get_queries
(
field
,
row
):
queries_with_answers
.
append
((
query
,
row
.
get_value_by_name
(
field
.
name
)))
for
query
,
answer
in
queries_with_answers
:
print
query
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录