Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
68b94c5c
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
68b94c5c
编写于
6月 05, 2020
作者:
K
kssenii
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fixes
上级
972611e3
变更
8
显示空白变更内容
内联
并排
Showing
8 changed file
with
28 addition
and
24 deletion
+28
-24
programs/server/config.xml
programs/server/config.xml
+2
-1
src/Storages/RabbitMQ/RabbitMQHandler.cpp
src/Storages/RabbitMQ/RabbitMQHandler.cpp
+1
-13
src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp
src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp
+5
-2
src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h
src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h
+1
-0
src/Storages/RabbitMQ/StorageRabbitMQ.cpp
src/Storages/RabbitMQ/StorageRabbitMQ.cpp
+8
-4
src/Storages/RabbitMQ/StorageRabbitMQ.h
src/Storages/RabbitMQ/StorageRabbitMQ.h
+2
-1
src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.cpp
src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.cpp
+5
-2
src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h
src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h
+4
-1
未找到文件。
programs/server/config.xml
浏览文件 @
68b94c5c
...
...
@@ -51,7 +51,8 @@
<https_port>8443</https_port>
<tcp_port_secure>9440</tcp_port_secure>
-->
<rabbitmq_username>
root
</rabbitmq_username>
<rabbitmq_password>
clickhouse
</rabbitmq_password>
<!-- Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -->
<openSSL>
<server>
<!-- Used for https server AND secure tcp port -->
...
...
src/Storages/RabbitMQ/RabbitMQHandler.cpp
浏览文件 @
68b94c5c
...
...
@@ -6,8 +6,7 @@ namespace DB
enum
{
Lock_timeout
=
50
,
Max_threads_to_pass
=
10
Lock_timeout
=
50
};
...
...
@@ -50,17 +49,6 @@ void RabbitMQHandler::start(std::atomic<bool> & check_param)
mutex_before_event_loop
.
unlock
();
}
else
{
if
(
++
count_passed
==
Max_threads_to_pass
)
{
/* Event loop is blocking to the thread that started it and it is not good to block one single thread as it loops
* untill there are no active events, but there can be too many of them for one thread to be blocked for so long.
*/
stop
();
count_passed
=
0
;
}
}
}
void
RabbitMQHandler
::
stop
()
...
...
src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp
浏览文件 @
68b94c5c
...
...
@@ -44,6 +44,7 @@ ReadBufferFromRabbitMQConsumer::ReadBufferFromRabbitMQConsumer(
,
stopped
(
stopped_
)
,
exchange_declared
(
false
)
,
false_param
(
false
)
,
loop_attempt
(
false
)
{
messages
.
clear
();
current
=
messages
.
begin
();
...
...
@@ -236,7 +237,7 @@ void ReadBufferFromRabbitMQConsumer::subscribe(const String & queue_name)
/* As event loop is blocking to the thread that started it and a single thread should not be blocked while
* executing all callbacks on the connection (not only its own), then there should be some point to unblock
*/
if
(
received
.
size
()
>=
Received_max_to_stop_loop
)
if
(
!
loop_attempt
&&
received
.
size
()
%
Received_max_to_stop_loop
==
0
)
{
stop_loop
=
true
;
}
...
...
@@ -284,7 +285,9 @@ bool ReadBufferFromRabbitMQConsumer::nextImpl()
if
(
received
.
empty
())
{
/// Run the onReceived callbacks to save the messages that have been received by now
loop_attempt
=
true
;
startEventLoop
(
false_param
);
loop_attempt
=
false
;
}
if
(
received
.
empty
())
...
...
src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h
浏览文件 @
68b94c5c
...
...
@@ -66,6 +66,7 @@ private:
String
current_exchange_name
;
size_t
count_subscribed
=
0
;
size_t
count_bound_queues
=
0
;
std
::
atomic
<
bool
>
loop_attempt
;
Messages
received
;
Messages
messages
;
...
...
src/Storages/RabbitMQ/StorageRabbitMQ.cpp
浏览文件 @
68b94c5c
...
...
@@ -74,10 +74,14 @@ StorageRabbitMQ::StorageRabbitMQ(
,
hash_exchange
(
hash_exchange_
)
,
log
(
&
Poco
::
Logger
::
get
(
"StorageRabbitMQ ("
+
table_id_
.
table_name
+
")"
))
,
semaphore
(
0
,
num_consumers_
)
,
login_password
(
std
::
make_pair
(
rabbitmq_context
.
getConfigRef
().
getString
(
"rabbitmq_username"
,
"root"
),
rabbitmq_context
.
getConfigRef
().
getString
(
"rabbitmq_password"
,
"clickhouse"
)))
,
parsed_address
(
parseAddress
(
global_context
.
getMacros
()
->
expand
(
host_port_
),
5672
))
,
evbase
(
event_base_new
())
,
eventHandler
(
evbase
,
log
)
,
connection
(
&
eventHandler
,
AMQP
::
Address
(
parsed_address
.
first
,
parsed_address
.
second
,
AMQP
::
Login
(
"root"
,
"clickhouse"
),
"/"
))
,
connection
(
&
eventHandler
,
AMQP
::
Address
(
parsed_address
.
first
,
parsed_address
.
second
,
AMQP
::
Login
(
login_password
.
first
,
login_password
.
second
),
"/"
))
{
size_t
cnt_retries
=
0
;
while
(
!
connection
.
ready
()
&&
++
cnt_retries
!=
Connection_setup_retries_max
)
...
...
@@ -208,14 +212,14 @@ ConsumerBufferPtr StorageRabbitMQ::createReadBuffer()
ChannelPtr
consumer_channel
=
std
::
make_shared
<
AMQP
::
TcpChannel
>
(
&
connection
);
return
std
::
make_shared
<
ReadBufferFromRabbitMQConsumer
>
(
consumer_channel
,
eventHandler
,
exchange_name
,
routing_key
,
next_channel_id
,
log
,
row_delimiter
,
bind_by_id
,
hash_exchange
,
num_queues
,
stream_cancelled
);
return
std
::
make_shared
<
ReadBufferFromRabbitMQConsumer
>
(
consumer_channel
,
eventHandler
,
exchange_name
,
routing_key
,
next_channel_id
,
log
,
row_delimiter
,
bind_by_id
,
hash_exchange
,
num_queues
,
stream_cancelled
);
}
ProducerBufferPtr
StorageRabbitMQ
::
createWriteBuffer
()
{
return
std
::
make_shared
<
WriteBufferToRabbitMQProducer
>
(
parsed_address
,
routing_key
,
exchange_name
,
return
std
::
make_shared
<
WriteBufferToRabbitMQProducer
>
(
parsed_address
,
login_password
,
routing_key
,
exchange_name
,
log
,
num_consumers
*
num_queues
,
bind_by_id
,
hash_exchange
,
row_delimiter
?
std
::
optional
<
char
>
{
row_delimiter
}
:
std
::
nullopt
,
1
,
1024
);
}
...
...
src/Storages/RabbitMQ/StorageRabbitMQ.h
浏览文件 @
68b94c5c
...
...
@@ -86,7 +86,8 @@ private:
const
bool
hash_exchange
;
Poco
::
Logger
*
log
;
std
::
pair
<
std
::
string
,
UInt16
>
parsed_address
;
std
::
pair
<
String
,
UInt16
>
parsed_address
;
std
::
pair
<
String
,
String
>
login_password
;
event_base
*
evbase
;
RabbitMQHandler
eventHandler
;
...
...
src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.cpp
浏览文件 @
68b94c5c
...
...
@@ -20,7 +20,8 @@ enum
};
WriteBufferToRabbitMQProducer
::
WriteBufferToRabbitMQProducer
(
std
::
pair
<
std
::
string
,
UInt16
>
&
parsed_address
,
std
::
pair
<
String
,
UInt16
>
&
parsed_address
,
std
::
pair
<
String
,
String
>
&
login_password_
,
const
String
&
routing_key_
,
const
String
&
exchange_
,
Poco
::
Logger
*
log_
,
...
...
@@ -31,6 +32,7 @@ WriteBufferToRabbitMQProducer::WriteBufferToRabbitMQProducer(
size_t
rows_per_message
,
size_t
chunk_size_
)
:
WriteBuffer
(
nullptr
,
0
)
,
login_password
(
login_password_
)
,
routing_key
(
routing_key_
)
,
exchange_name
(
exchange_
)
,
log
(
log_
)
...
...
@@ -42,7 +44,8 @@ WriteBufferToRabbitMQProducer::WriteBufferToRabbitMQProducer(
,
chunk_size
(
chunk_size_
)
,
producerEvbase
(
event_base_new
())
,
eventHandler
(
producerEvbase
,
log
)
,
connection
(
&
eventHandler
,
AMQP
::
Address
(
parsed_address
.
first
,
parsed_address
.
second
,
AMQP
::
Login
(
"root"
,
"clickhouse"
),
"/"
))
,
connection
(
&
eventHandler
,
AMQP
::
Address
(
parsed_address
.
first
,
parsed_address
.
second
,
AMQP
::
Login
(
login_password
.
first
,
login_password
.
second
),
"/"
))
{
/* The reason behind making a separate connection for each concurrent producer is explained here:
* https://github.com/CopernicaMarketingSoftware/AMQP-CPP/issues/128#issuecomment-300780086 - publishing from
...
...
src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h
浏览文件 @
68b94c5c
...
...
@@ -7,6 +7,7 @@
#include <atomic>
#include <amqpcpp.h>
#include <Storages/RabbitMQ/RabbitMQHandler.h>
#include <Interpreters/Context.h>
namespace
DB
{
...
...
@@ -18,7 +19,8 @@ class WriteBufferToRabbitMQProducer : public WriteBuffer
{
public:
WriteBufferToRabbitMQProducer
(
std
::
pair
<
std
::
string
,
UInt16
>
&
parsed_address
,
std
::
pair
<
String
,
UInt16
>
&
parsed_address
,
std
::
pair
<
String
,
String
>
&
login_password_
,
const
String
&
routing_key_
,
const
String
&
exchange_
,
Poco
::
Logger
*
log_
,
...
...
@@ -40,6 +42,7 @@ private:
void
checkExchange
();
void
startEventLoop
(
std
::
atomic
<
bool
>
&
check_param
);
std
::
pair
<
String
,
String
>
&
login_password
;
const
String
routing_key
;
const
String
exchange_name
;
const
bool
bind_by_id
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录