Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
a38c7ff2
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
a38c7ff2
编写于
9月 30, 2020
作者:
V
Vladimir Chebotarev
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Somehow uncompressed PUT works and gzipped PUT doesn't, in S3 storage.
上级
b7a6f15f
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
50 addition
and
3 deletion
+50
-3
src/Storages/StorageS3.cpp
src/Storages/StorageS3.cpp
+13
-2
src/Storages/StorageS3.h
src/Storages/StorageS3.h
+1
-1
tests/integration/test_storage_s3/test.py
tests/integration/test_storage_s3/test.py
+36
-0
未找到文件。
src/Storages/StorageS3.cpp
浏览文件 @
a38c7ff2
...
...
@@ -197,7 +197,7 @@ StorageS3::StorageS3(
const
ColumnsDescription
&
columns_
,
const
ConstraintsDescription
&
constraints_
,
Context
&
context_
,
const
String
&
compression_method_
=
""
)
const
String
&
compression_method_
)
:
IStorage
(
table_id_
)
,
uri
(
uri_
)
,
context_global
(
context_
)
...
...
@@ -372,7 +372,18 @@ void registerStorageS3Impl(const String & name, StorageFactory & factory)
format_name
=
engine_args
.
back
()
->
as
<
ASTLiteral
&>
().
value
.
safeGet
<
String
>
();
}
return
StorageS3
::
create
(
s3_uri
,
access_key_id
,
secret_access_key
,
args
.
table_id
,
format_name
,
min_upload_part_size
,
args
.
columns
,
args
.
constraints
,
args
.
context
);
return
StorageS3
::
create
(
s3_uri
,
access_key_id
,
secret_access_key
,
args
.
table_id
,
format_name
,
min_upload_part_size
,
args
.
columns
,
args
.
constraints
,
args
.
context
,
compression_method
);
},
{
.
source_access_type
=
AccessType
::
S3
,
...
...
src/Storages/StorageS3.h
浏览文件 @
a38c7ff2
...
...
@@ -34,7 +34,7 @@ public:
const
ColumnsDescription
&
columns_
,
const
ConstraintsDescription
&
constraints_
,
Context
&
context_
,
const
String
&
compression_method_
);
const
String
&
compression_method_
=
""
);
String
getName
()
const
override
{
...
...
tests/integration/test_storage_s3/test.py
浏览文件 @
a38c7ff2
...
...
@@ -392,6 +392,41 @@ def test_storage_s3_get_gzip(cluster):
run_query
(
instance
,
"DROP TABLE {}"
.
format
(
name
))
def
test_storage_s3_put_uncompressed
(
cluster
):
bucket
=
cluster
.
minio_bucket
instance
=
cluster
.
instances
[
"dummy"
]
filename
=
"test_put_uncompressed.bin"
name
=
"test_put_uncompressed"
data
=
[
"'Gloria Thompson',99"
,
"'Matthew Tang',98"
,
"'Patsy Anderson',23"
,
"'Nancy Badillo',93"
,
"'Roy Hunt',5"
,
"'Adam Kirk',51"
,
"'Joshua Douds',28"
,
"'Jolene Ryan',0"
,
"'Roxanne Padilla',50"
,
"'Howard Roberts',41"
,
"'Ricardo Broughton',13"
,
"'Roland Speer',83"
,
"'Cathy Cohan',58"
,
"'Kathie Dawson',100"
,
"'Gregg Mcquistion',11"
,
]
try
:
run_query
(
instance
,
"CREATE TABLE {} (name String, id UInt32) ENGINE = S3('http://{}:{}/{}/{}', 'CSV')"
.
format
(
name
,
cluster
.
minio_host
,
cluster
.
minio_port
,
bucket
,
filename
))
run_query
(
instance
,
"INSERT INTO {} VALUES ({})"
.
format
(
name
,
"),("
.
join
(
data
)))
run_query
(
instance
,
"SELECT sum(id) FROM {}"
.
format
(
name
)).
splitlines
()
==
[
"753"
]
uncompressed_content
=
get_s3_file_content
(
cluster
,
bucket
,
filename
)
assert
sum
([
int
(
i
.
split
(
','
)[
1
])
for
i
in
uncompressed_content
.
splitlines
()
])
==
753
finally
:
run_query
(
instance
,
"DROP TABLE {}"
.
format
(
name
))
def
test_storage_s3_put_gzip
(
cluster
):
bucket
=
cluster
.
minio_bucket
instance
=
cluster
.
instances
[
"dummy"
]
...
...
@@ -419,6 +454,7 @@ def test_storage_s3_put_gzip(cluster):
name
,
cluster
.
minio_host
,
cluster
.
minio_port
,
bucket
,
filename
))
run_query
(
instance
,
"INSERT INTO {} VALUES ({})"
.
format
(
name
,
"),("
.
join
(
data
)))
run_query
(
instance
,
"SELECT sum(id) FROM {}"
.
format
(
name
)).
splitlines
()
==
[
"708"
]
buf
=
StringIO
.
StringIO
(
get_s3_file_content
(
cluster
,
bucket
,
filename
))
f
=
gzip
.
GzipFile
(
fileobj
=
buf
,
mode
=
"rb"
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录