Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
7e398f47
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
7e398f47
编写于
12月 19, 2019
作者:
N
Nikolai Kochetov
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Change implementation of copyData for multiple streams.
上级
c61e2aef
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
70 addition
and
57 deletion
+70
-57
dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h
dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h
+4
-0
dbms/src/DataStreams/copyData.cpp
dbms/src/DataStreams/copyData.cpp
+66
-57
未找到文件。
dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h
浏览文件 @
7e398f47
...
...
@@ -49,6 +49,10 @@ public:
protected:
Block
readImpl
()
override
{
/// We do not use cancel flag here.
/// If query was cancelled, it will be processed by child streams.
/// Part of the data will be processed.
if
(
input_streams
.
size
()
==
1
&&
output_streams
.
size
()
==
1
)
copyData
(
*
input_streams
.
at
(
0
),
*
output_streams
.
at
(
0
));
else
...
...
dbms/src/DataStreams/copyData.cpp
浏览文件 @
7e398f47
...
...
@@ -2,6 +2,7 @@
#include <DataStreams/IBlockInputStream.h>
#include <DataStreams/IBlockOutputStream.h>
#include <DataStreams/copyData.h>
#include <DataStreams/ParallelInputsProcessor.h>
#include <Common/ConcurrentBoundedQueue.h>
#include <Common/ThreadPool.h>
...
...
@@ -54,69 +55,77 @@ void copyDataImpl(IBlockInputStream & from, IBlockOutputStream & to, TCancelCall
inline
void
doNothing
(
const
Block
&
)
{}
namespace
{
static
void
copyDataImpl
(
BlockInputStreams
&
froms
,
BlockOutputStreams
&
tos
)
struct
ParallelInsertsHandler
{
if
(
froms
.
size
()
==
tos
.
size
())
using
CencellationHook
=
std
::
function
<
void
()
>
;
explicit
ParallelInsertsHandler
(
BlockOutputStreams
&
output_streams
,
CencellationHook
cancellation_hook_
)
:
outputs
(
output_streams
.
size
()),
cancellation_hook
(
std
::
move
(
cancellation_hook_
))
{
exceptions
.
resize
(
output_streams
.
size
());
for
(
auto
&
output
:
output_streams
)
outputs
.
push
(
output
.
get
());
}
void
onBlock
(
Block
&
block
,
size_t
/*thread_num*/
)
{
IBlockOutputStream
*
out
=
nullptr
;
outputs
.
pop
(
out
);
out
->
write
(
block
);
outputs
.
push
(
out
);
}
void
onFinishThread
(
size_t
/*thread_num*/
)
{}
void
onFinish
()
{}
void
onException
(
std
::
exception_ptr
&
exception
,
size_t
thread_num
)
{
std
::
vector
<
ThreadFromGlobalPool
>
threads
;
threads
.
reserve
(
froms
.
size
());
for
(
size_t
i
=
0
;
i
<
froms
.
size
();
i
++
)
{
threads
.
emplace_back
([
from
=
froms
.
at
(
i
),
to
=
tos
.
at
(
i
)]()
{
from
->
readPrefix
();
to
->
writePrefix
();
while
(
Block
block
=
from
->
read
())
to
->
write
(
block
);
from
->
readSuffix
();
to
->
writeSuffix
();
});
}
for
(
auto
&
thread
:
threads
)
thread
.
join
();
exceptions
[
thread_num
]
=
std
::
move
(
exception
);
cancellation_hook
();
}
else
void
rethrowFirstException
()
{
ConcurrentBoundedQueue
<
Block
>
queue
(
froms
.
size
());
std
::
vector
<
ThreadFromGlobalPool
>
to_threads
;
for
(
auto
&
to
:
tos
)
{
to_threads
.
emplace_back
([
&
queue
,
to
]()
{
to
->
writePrefix
();
Block
block
;
while
(
true
)
{
queue
.
pop
(
block
);
if
(
!
block
)
break
;
to
->
write
(
block
);
}
to
->
writeSuffix
();
});
}
std
::
vector
<
ThreadFromGlobalPool
>
from_threads_
;
from_threads_
.
reserve
(
froms
.
size
());
for
(
auto
&
from
:
froms
)
{
from_threads_
.
emplace_back
([
&
queue
,
from
]()
{
from
->
readPrefix
();
while
(
Block
block
=
from
->
read
())
queue
.
push
(
block
);
from
->
readSuffix
();
});
}
for
(
auto
&
thread
:
from_threads_
)
thread
.
join
();
for
(
size_t
i
=
0
;
i
<
tos
.
size
();
i
++
)
queue
.
push
({});
for
(
auto
&
thread
:
to_threads
)
thread
.
join
();
for
(
auto
&
exception
:
exceptions
)
if
(
exception
)
std
::
rethrow_exception
(
exception
);
}
ConcurrentBoundedQueue
<
IBlockOutputStream
*>
outputs
;
std
::
vector
<
std
::
exception_ptr
>
exceptions
;
CencellationHook
cancellation_hook
;
};
}
static
void
copyDataImpl
(
BlockInputStreams
&
froms
,
BlockOutputStreams
&
tos
)
{
for
(
auto
&
output
:
tos
)
output
->
writePrefix
();
using
Processor
=
ParallelInputsProcessor
<
ParallelInsertsHandler
>
;
Processor
*
processor_ptr
=
nullptr
;
ParallelInsertsHandler
handler
(
tos
,
[
&
processor_ptr
]()
{
processor_ptr
->
cancel
(
false
);
});
ParallelInputsProcessor
<
ParallelInsertsHandler
>
processor
(
froms
,
nullptr
,
froms
.
size
(),
handler
);
processor_ptr
=
&
processor
;
processor
.
process
();
processor
.
wait
();
handler
.
rethrowFirstException
();
/// readPrefix is called in ParallelInputsProcessor.
for
(
auto
&
input
:
froms
)
input
->
readSuffix
();
for
(
auto
&
output
:
tos
)
output
->
writeSuffix
();
}
void
copyData
(
IBlockInputStream
&
from
,
IBlockOutputStream
&
to
,
std
::
atomic
<
bool
>
*
is_cancelled
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录