Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
96df2e6b
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
96df2e6b
编写于
7月 02, 2020
作者:
A
alesapin
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Better shutdown and conversion
上级
64583ceb
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
23 addition
and
20 deletion
+23
-20
src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp
src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp
+1
-7
src/Storages/RabbitMQ/StorageRabbitMQ.cpp
src/Storages/RabbitMQ/StorageRabbitMQ.cpp
+21
-12
tests/integration/test_storage_rabbitmq/test.py
tests/integration/test_storage_rabbitmq/test.py
+1
-1
未找到文件。
src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp
浏览文件 @
96df2e6b
#include <DataStreams/ConvertingBlockInputStream.h>
#include <DataStreams/OneBlockInputStream.h>
#include <Formats/FormatFactory.h>
#include <Processors/Formats/InputStreamFromInputFormat.h>
#include <Storages/RabbitMQ/RabbitMQBlockInputStream.h>
...
...
@@ -152,11 +150,7 @@ Block RabbitMQBlockInputStream::readImpl()
result_block
.
insert
(
column
);
}
return
ConvertingBlockInputStream
(
std
::
make_shared
<
OneBlockInputStream
>
(
result_block
),
getHeader
(),
ConvertingBlockInputStream
::
MatchColumnsMode
::
Name
)
.
read
();
return
result_block
;
}
}
src/Storages/RabbitMQ/StorageRabbitMQ.cpp
浏览文件 @
96df2e6b
#include <Storages/RabbitMQ/StorageRabbitMQ.h>
#include <DataStreams/IBlockInputStream.h>
#include <DataStreams/ConvertingBlockInputStream.h>
#include <DataStreams/LimitBlockInputStream.h>
#include <DataStreams/UnionBlockInputStream.h>
#include <DataStreams/copyData.h>
...
...
@@ -153,11 +154,14 @@ Pipes StorageRabbitMQ::read(
Pipes
pipes
;
pipes
.
reserve
(
num_created_consumers
);
auto
sample_block
=
metadata_snapshot
->
getSampleBlockForColumns
(
column_names
,
getVirtuals
(),
getStorageID
());
for
(
size_t
i
=
0
;
i
<
num_created_consumers
;
++
i
)
{
pipes
.
emplace_back
(
std
::
make_shared
<
SourceFromInputStream
>
(
std
::
make_shared
<
RabbitMQBlockInputStream
>
(
*
this
,
metadata_snapshot
,
context
,
column_names
,
log
)));
auto
rabbit_stream
=
std
::
make_shared
<
RabbitMQBlockInputStream
>
(
*
this
,
metadata_snapshot
,
context
,
column_names
,
log
);
auto
converting_stream
=
std
::
make_shared
<
ConvertingBlockInputStream
>
(
rabbit_stream
,
sample_block
,
ConvertingBlockInputStream
::
MatchColumnsMode
::
Name
);
pipes
.
emplace_back
(
std
::
make_shared
<
SourceFromInputStream
>
(
converting_stream
));
}
if
(
!
loop_started
)
...
...
@@ -202,16 +206,16 @@ void StorageRabbitMQ::shutdown()
{
stream_cancelled
=
true
;
for
(
size_t
i
=
0
;
i
<
num_created_consumers
;
++
i
)
{
popReadBuffer
();
}
event_handler
->
stop
();
looping_task
->
deactivate
();
streaming_task
->
deactivate
();
heartbeat_task
->
deactivate
();
event_handler
->
stop
();
looping_task
->
deactivate
();
for
(
size_t
i
=
0
;
i
<
num_created_consumers
;
++
i
)
{
popReadBuffer
();
}
connection
->
close
();
}
...
...
@@ -355,17 +359,22 @@ bool StorageRabbitMQ::streamToViews()
BlockInputStreams
streams
;
streams
.
reserve
(
num_created_consumers
);
auto
metadata_snapshot
=
getInMemoryMetadataPtr
();
auto
column_names
=
block_io
.
out
->
getHeader
().
getNames
();
auto
sample_block
=
metadata_snapshot
->
getSampleBlockForColumns
(
column_names
,
getVirtuals
(),
getStorageID
());
for
(
size_t
i
=
0
;
i
<
num_created_consumers
;
++
i
)
{
auto
stream
=
std
::
make_shared
<
RabbitMQBlockInputStream
>
(
*
this
,
getInMemoryMetadataPtr
(),
rabbitmq_context
,
block_io
.
out
->
getHeader
().
getNames
(),
log
);
streams
.
emplace_back
(
stream
);
auto
rabbit_stream
=
std
::
make_shared
<
RabbitMQBlockInputStream
>
(
*
this
,
metadata_snapshot
,
rabbitmq_context
,
column_names
,
log
);
auto
converting_stream
=
std
::
make_shared
<
ConvertingBlockInputStream
>
(
rabbit_stream
,
sample_block
,
ConvertingBlockInputStream
::
MatchColumnsMode
::
Name
);
streams
.
emplace_back
(
converting_stream
);
// Limit read batch to maximum block size to allow DDL
IBlockInputStream
::
LocalLimits
limits
;
const
Settings
&
settings
=
global_context
.
getSettingsRef
();
limits
.
speed_limits
.
max_execution_time
=
settings
.
stream_flush_interval_ms
;
limits
.
timeout_overflow_mode
=
OverflowMode
::
BREAK
;
stream
->
setLimits
(
limits
);
rabbit_
stream
->
setLimits
(
limits
);
}
if
(
!
loop_started
)
...
...
tests/integration/test_storage_rabbitmq/test.py
浏览文件 @
96df2e6b
...
...
@@ -816,7 +816,7 @@ def test_rabbitmq_insert(rabbitmq_cluster):
channel
.
stop_consuming
()
consumer
.
basic_qos
(
prefetch_count
=
50
)
consumer
.
basic_consume
(
onReceived
,
queue_name
)
consumer
.
basic_consume
(
queue_name
,
onReceived
)
consumer
.
start_consuming
()
consumer_connection
.
close
()
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录