Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
c2849d08
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
c2849d08
编写于
11月 20, 2019
作者:
A
Andrei Bodrov
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
move to helpers
上级
53b02227
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
44 addition
and
36 deletion
+44
-36
dbms/src/DataStreams/IBlockInputStream.h
dbms/src/DataStreams/IBlockInputStream.h
+0
-13
dbms/src/DataStreams/IBlockOutputStream.h
dbms/src/DataStreams/IBlockOutputStream.h
+0
-13
dbms/src/IO/ReadHelpers.h
dbms/src/IO/ReadHelpers.h
+13
-0
dbms/src/IO/WriteHelpers.h
dbms/src/IO/WriteHelpers.h
+14
-0
dbms/src/Storages/StorageFile.cpp
dbms/src/Storages/StorageFile.cpp
+5
-4
dbms/src/Storages/StorageHDFS.cpp
dbms/src/Storages/StorageHDFS.cpp
+4
-2
dbms/src/Storages/StorageS3.cpp
dbms/src/Storages/StorageS3.cpp
+4
-2
dbms/src/Storages/StorageURL.cpp
dbms/src/Storages/StorageURL.cpp
+4
-2
未找到文件。
dbms/src/DataStreams/IBlockInputStream.h
浏览文件 @
c2849d08
...
...
@@ -5,9 +5,7 @@
#include <DataStreams/IBlockStream_fwd.h>
#include <DataStreams/SizeLimits.h>
#include <DataStreams/ExecutionSpeedLimits.h>
#include <IO/CompressionMethod.h>
#include <IO/Progress.h>
#include <IO/ZlibInflatingReadBuffer.h>
#include <Storages/TableStructureLockHolder.h>
#include <Common/TypePromotion.h>
...
...
@@ -230,17 +228,6 @@ public:
/// Enable calculation of minimums and maximums by the result columns.
void
enableExtremes
()
{
enabled_extremes
=
true
;
}
template
<
class
TReadBuffer
,
class
...
Types
>
std
::
unique_ptr
<
ReadBuffer
>
getBuffer
(
const
DB
::
CompressionMethod
method
,
Types
...
args
)
{
if
(
method
==
DB
::
CompressionMethod
::
Gzip
)
{
auto
read_buf
=
std
::
make_unique
<
TReadBuffer
>
(
args
...);
return
std
::
make_unique
<
ZlibInflatingReadBuffer
>
(
std
::
move
(
read_buf
),
method
);
}
return
std
::
make_unique
<
TReadBuffer
>
(
args
...);
}
protected:
/// Order is important: `table_locks` must be destroyed after `children` so that tables from
/// which child streams read are protected by the locks during the lifetime of the child streams.
...
...
dbms/src/DataStreams/IBlockOutputStream.h
浏览文件 @
c2849d08
...
...
@@ -3,8 +3,6 @@
#include <Core/Block.h>
#include <DataStreams/IBlockStream_fwd.h>
#include <Storages/TableStructureLockHolder.h>
#include <IO/CompressionMethod.h>
#include <IO/ZlibDeflatingWriteBuffer.h>
#include <boost/noncopyable.hpp>
...
...
@@ -65,17 +63,6 @@ public:
*/
void
addTableLock
(
const
TableStructureReadLockHolder
&
lock
)
{
table_locks
.
push_back
(
lock
);
}
template
<
class
TWriteBuffer
,
class
...
Types
>
std
::
unique_ptr
<
WriteBuffer
>
getBuffer
(
const
DB
::
CompressionMethod
method
,
Types
...
args
)
{
if
(
method
==
DB
::
CompressionMethod
::
Gzip
)
{
auto
write_buf
=
std
::
make_unique
<
TWriteBuffer
>
(
args
...);
return
std
::
make_unique
<
ZlibDeflatingWriteBuffer
>
(
std
::
move
(
write_buf
),
method
,
1
/* compression level */
);
}
return
std
::
make_unique
<
TWriteBuffer
>
(
args
...);
}
virtual
void
finalize
()
{}
private:
...
...
dbms/src/IO/ReadHelpers.h
浏览文件 @
c2849d08
...
...
@@ -23,9 +23,11 @@
#include <Formats/FormatSettings.h>
#include <IO/CompressionMethod.h>
#include <IO/ReadBuffer.h>
#include <IO/ReadBufferFromMemory.h>
#include <IO/VarInt.h>
#include <IO/ZlibInflatingReadBuffer.h>
#ifdef __clang__
#pragma clang diagnostic push
...
...
@@ -911,4 +913,15 @@ void skipToNextLineOrEOF(ReadBuffer & buf);
/// Skip to next character after next unescaped \n. If no \n in stream, skip to end. Does not throw on invalid escape sequences.
void
skipToUnescapedNextLineOrEOF
(
ReadBuffer
&
buf
);
template
<
class
TReadBuffer
,
class
...
Types
>
std
::
unique_ptr
<
ReadBuffer
>
getReadBuffer
(
const
DB
::
CompressionMethod
method
,
Types
...
args
)
{
if
(
method
==
DB
::
CompressionMethod
::
Gzip
)
{
auto
read_buf
=
std
::
make_unique
<
TReadBuffer
>
(
args
...);
return
std
::
make_unique
<
ZlibInflatingReadBuffer
>
(
std
::
move
(
read_buf
),
method
);
}
return
std
::
make_unique
<
TReadBuffer
>
(
args
...);
}
}
dbms/src/IO/WriteHelpers.h
浏览文件 @
c2849d08
...
...
@@ -20,11 +20,13 @@
#include <Common/UInt128.h>
#include <Common/intExp.h>
#include <IO/CompressionMethod.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteIntText.h>
#include <IO/VarInt.h>
#include <IO/DoubleConverter.h>
#include <IO/WriteBufferFromString.h>
#include <IO/ZlibDeflatingWriteBuffer.h>
#include <Formats/FormatSettings.h>
...
...
@@ -905,4 +907,16 @@ inline String toString(const T & x)
writeText
(
x
,
buf
);
return
buf
.
str
();
}
template
<
class
TWriteBuffer
,
class
...
Types
>
std
::
unique_ptr
<
WriteBuffer
>
getWriteBuffer
(
const
DB
::
CompressionMethod
method
,
Types
...
args
)
{
if
(
method
==
DB
::
CompressionMethod
::
Gzip
)
{
auto
write_buf
=
std
::
make_unique
<
TWriteBuffer
>
(
args
...);
return
std
::
make_unique
<
ZlibDeflatingWriteBuffer
>
(
std
::
move
(
write_buf
),
method
,
1
/* compression level */
);
}
return
std
::
make_unique
<
TWriteBuffer
>
(
args
...);
}
}
dbms/src/Storages/StorageFile.cpp
浏览文件 @
c2849d08
...
...
@@ -8,6 +8,7 @@
#include <Parsers/ASTIdentifier.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/WriteHelpers.h>
...
...
@@ -203,12 +204,12 @@ public:
}
storage
->
table_fd_was_used
=
true
;
read_buf
=
getBuffer
<
ReadBufferFromFileDescriptor
>
(
compression_method
,
storage
->
table_fd
);
read_buf
=
get
Read
Buffer
<
ReadBufferFromFileDescriptor
>
(
compression_method
,
storage
->
table_fd
);
}
else
{
shared_lock
=
std
::
shared_lock
(
storage
->
rwlock
);
read_buf
=
getBuffer
<
ReadBufferFromFile
>
(
compression_method
,
file_path
);
read_buf
=
get
Read
Buffer
<
ReadBufferFromFile
>
(
compression_method
,
file_path
);
}
reader
=
FormatFactory
::
instance
().
getInput
(
storage
->
format_name
,
*
read_buf
,
storage
->
getSampleBlock
(),
context
,
max_block_size
);
...
...
@@ -285,13 +286,13 @@ public:
* INSERT data; SELECT *; last SELECT returns only insert_data
*/
storage
.
table_fd_was_used
=
true
;
write_buf
=
getBuffer
<
WriteBufferFromFileDescriptor
>
(
compression_method
,
storage
.
table_fd
);
write_buf
=
get
Write
Buffer
<
WriteBufferFromFileDescriptor
>
(
compression_method
,
storage
.
table_fd
);
}
else
{
if
(
storage
.
paths
.
size
()
!=
1
)
throw
Exception
(
"Table '"
+
storage
.
table_name
+
"' is in readonly mode because of globs in filepath"
,
ErrorCodes
::
DATABASE_ACCESS_DENIED
);
write_buf
=
getBuffer
<
WriteBufferFromFile
>
(
compression_method
,
storage
.
paths
[
0
],
DBMS_DEFAULT_BUFFER_SIZE
,
O_WRONLY
|
O_APPEND
|
O_CREAT
);
write_buf
=
get
Write
Buffer
<
WriteBufferFromFile
>
(
compression_method
,
storage
.
paths
[
0
],
DBMS_DEFAULT_BUFFER_SIZE
,
O_WRONLY
|
O_APPEND
|
O_CREAT
);
}
writer
=
FormatFactory
::
instance
().
getOutput
(
storage
.
format_name
,
*
write_buf
,
storage
.
getSampleBlock
(),
storage
.
context_global
);
...
...
dbms/src/Storages/StorageHDFS.cpp
浏览文件 @
c2849d08
...
...
@@ -7,8 +7,10 @@
#include <Interpreters/Context.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <Parsers/ASTLiteral.h>
#include <IO/ReadHelpers.h>
#include <IO/ReadBufferFromHDFS.h>
#include <IO/WriteBufferFromHDFS.h>
#include <IO/WriteHelpers.h>
#include <IO/HDFSCommon.h>
#include <Formats/FormatFactory.h>
#include <DataStreams/IBlockOutputStream.h>
...
...
@@ -61,7 +63,7 @@ public:
UInt64
max_block_size
,
const
CompressionMethod
compression_method
)
{
auto
read_buf
=
getBuffer
<
ReadBufferFromHDFS
>
(
compression_method
,
uri
);
auto
read_buf
=
get
Read
Buffer
<
ReadBufferFromHDFS
>
(
compression_method
,
uri
);
auto
input_stream
=
FormatFactory
::
instance
().
getInput
(
format
,
*
read_buf
,
sample_block
,
context
,
max_block_size
);
reader
=
std
::
make_shared
<
OwningBlockInputStream
<
ReadBuffer
>>
(
input_stream
,
std
::
move
(
read_buf
));
...
...
@@ -106,7 +108,7 @@ public:
const
CompressionMethod
compression_method
)
:
sample_block
(
sample_block_
)
{
write_buf
=
getBuffer
<
WriteBufferFromHDFS
>
(
compression_method
,
uri
);
write_buf
=
get
Write
Buffer
<
WriteBufferFromHDFS
>
(
compression_method
,
uri
);
writer
=
FormatFactory
::
instance
().
getOutput
(
format
,
*
write_buf
,
sample_block
,
context
);
}
...
...
dbms/src/Storages/StorageS3.cpp
浏览文件 @
c2849d08
...
...
@@ -6,7 +6,9 @@
#include <Parsers/ASTLiteral.h>
#include <IO/ReadBufferFromS3.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromS3.h>
#include <IO/WriteHelpers.h>
#include <Formats/FormatFactory.h>
...
...
@@ -39,7 +41,7 @@ namespace
const
CompressionMethod
compression_method
)
:
name
(
name_
)
{
read_buf
=
getBuffer
<
ReadBufferFromS3
>
(
compression_method
,
uri
,
timeouts
);
read_buf
=
get
Read
Buffer
<
ReadBufferFromS3
>
(
compression_method
,
uri
,
timeouts
);
reader
=
FormatFactory
::
instance
().
getInput
(
format
,
*
read_buf
,
sample_block
,
context
,
max_block_size
);
}
...
...
@@ -86,7 +88,7 @@ namespace
const
CompressionMethod
compression_method
)
:
sample_block
(
sample_block_
)
{
write_buf
=
getBuffer
<
WriteBufferFromS3
>
(
compression_method
,
uri
,
min_upload_part_size
,
timeouts
);
write_buf
=
get
Write
Buffer
<
WriteBufferFromS3
>
(
compression_method
,
uri
,
min_upload_part_size
,
timeouts
);
writer
=
FormatFactory
::
instance
().
getOutput
(
format
,
*
write_buf
,
sample_block
,
context
);
}
...
...
dbms/src/Storages/StorageURL.cpp
浏览文件 @
c2849d08
...
...
@@ -5,8 +5,10 @@
#include <Interpreters/evaluateConstantExpression.h>
#include <Parsers/ASTLiteral.h>
#include <IO/ReadHelpers.h>
#include <IO/ReadWriteBufferFromHTTP.h>
#include <IO/WriteBufferFromHTTP.h>
#include <IO/WriteHelpers.h>
#include <Formats/FormatFactory.h>
...
...
@@ -56,7 +58,7 @@ namespace
const
CompressionMethod
compression_method
)
:
name
(
name_
)
{
read_buf
=
getBuffer
<
ReadWriteBufferFromHTTP
>
(
compression_method
,
uri
,
method
,
callback
,
timeouts
,
context
.
getSettingsRef
().
max_http_get_redirects
);
read_buf
=
get
Read
Buffer
<
ReadWriteBufferFromHTTP
>
(
compression_method
,
uri
,
method
,
callback
,
timeouts
,
context
.
getSettingsRef
().
max_http_get_redirects
);
reader
=
FormatFactory
::
instance
().
getInput
(
format
,
*
read_buf
,
sample_block
,
context
,
max_block_size
);
}
...
...
@@ -102,7 +104,7 @@ namespace
const
CompressionMethod
compression_method
)
:
sample_block
(
sample_block_
)
{
write_buf
=
getBuffer
<
WriteBufferFromHTTP
>
(
compression_method
,
uri
,
Poco
::
Net
::
HTTPRequest
::
HTTP_POST
,
timeouts
);
write_buf
=
get
Write
Buffer
<
WriteBufferFromHTTP
>
(
compression_method
,
uri
,
Poco
::
Net
::
HTTPRequest
::
HTTP_POST
,
timeouts
);
writer
=
FormatFactory
::
instance
().
getOutput
(
format
,
*
write_buf
,
sample_block
,
context
);
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录