Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
09e43b0b
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
09e43b0b
编写于
9月 24, 2020
作者:
S
shengjh
提交者:
yefu.chen
9月 24, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix resource dead lock
Signed-off-by:
N
shengjh
<
1572099106@qq.com
>
上级
f4d29c97
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
52 addition
and
16 deletion
+52
-16
proxy/src/message_client/ClientV2.cpp
proxy/src/message_client/ClientV2.cpp
+23
-8
proxy/src/server/MessageWrapper.cpp
proxy/src/server/MessageWrapper.cpp
+15
-0
proxy/src/server/MessageWrapper.h
proxy/src/server/MessageWrapper.h
+4
-0
proxy/src/server/MetaWrapper.cpp
proxy/src/server/MetaWrapper.cpp
+5
-1
proxy/src/server/MetaWrapper.h
proxy/src/server/MetaWrapper.h
+3
-0
proxy/src/server/Server.cpp
proxy/src/server/Server.cpp
+2
-7
未找到文件。
proxy/src/message_client/ClientV2.cpp
浏览文件 @
09e43b0b
...
...
@@ -7,6 +7,7 @@
#include <omp.h>
#include <numeric>
#include <algorithm>
#include "log/Log.h"
namespace
milvus
::
message_client
{
...
...
@@ -161,6 +162,8 @@ Status MsgClientV2::SendMutMessage(const milvus::grpc::InsertParam &request,
const
std
::
function
<
uint64_t
(
const
std
::
string
&
collection_name
,
uint64_t
channel_id
,
uint64_t
timestamp
)
>
&
segment_id
)
{
using
stdclock
=
std
::
chrono
::
high_resolution_clock
;
auto
start
=
stdclock
::
now
();
// may have retry policy?
auto
row_count
=
request
.
rows_data_size
();
auto
topic_num
=
config
.
pulsar
.
topicnum
();
...
...
@@ -184,7 +187,7 @@ Status MsgClientV2::SendMutMessage(const milvus::grpc::InsertParam &request,
mut_msg
.
mutable_rows_data
()
->
CopyFrom
(
request
.
rows_data
(
i
));
mut_msg
.
mutable_extra_params
()
->
CopyFrom
(
request
.
extra_params
());
auto
callback
=
[
&
stats
,
&
msg_sended
,
this_thread
](
Result
result
,
const
pulsar
::
MessageId
&
messageId
)
{
auto
callback
=
[
&
stats
,
&
msg_sended
,
this_thread
](
Result
result
,
const
pulsar
::
MessageId
&
messageId
)
{
msg_sended
+=
1
;
if
(
result
!=
pulsar
::
ResultOk
)
{
stats
[
this_thread
]
=
Status
(
DB_ERROR
,
pulsar
::
strResult
(
result
));
...
...
@@ -197,9 +200,15 @@ Status MsgClientV2::SendMutMessage(const milvus::grpc::InsertParam &request,
stats
[
this_thread
]
=
Status
(
DB_ERROR
,
e
.
what
());
}
}
while
(
msg_sended
<
row_count
){
while
(
msg_sended
<
row_count
)
{
}
auto
end
=
stdclock
::
now
();
auto
data_size
=
request
.
ByteSize
();
LOG_SERVER_INFO_
<<
"InsertReq Batch size:"
<<
data_size
/
1024.0
/
1024.0
<<
"M, "
<<
"throughput: "
<<
data_size
/
std
::
chrono
::
duration_cast
<
std
::
chrono
::
milliseconds
>
(
end
-
start
).
count
()
*
1000
/
1024.0
/
1024
<<
"M/s"
;
for
(
auto
&
stat
:
stats
)
{
if
(
!
stat
.
ok
())
{
return
stat
;
...
...
@@ -213,6 +222,9 @@ Status MsgClientV2::SendMutMessage(const milvus::grpc::DeleteByIDParam &request,
const
std
::
function
<
uint64_t
(
const
std
::
string
&
collection_name
,
uint64_t
channel_id
,
uint64_t
timestamp
)
>
&
segment_id
)
{
using
stdclock
=
std
::
chrono
::
high_resolution_clock
;
auto
start
=
stdclock
::
now
();
auto
row_count
=
request
.
id_array_size
();
auto
topicnum
=
config
.
pulsar
.
topicnum
();
auto
stats
=
std
::
vector
<
Status
>
(
topicnum
);
...
...
@@ -226,12 +238,9 @@ Status MsgClientV2::SendMutMessage(const milvus::grpc::DeleteByIDParam &request,
mut_msg
.
set_uid
(
request
.
id_array
(
i
));
mut_msg
.
set_collection_name
(
request
.
collection_name
());
mut_msg
.
set_timestamp
(
timestamp
);
uint64_t
uid
=
request
.
id_array
(
i
);
auto
channel_id
=
makeHash
(
&
uid
,
sizeof
(
uint64_t
))
%
topicnum
;
mut_msg
.
set_segment_id
(
segment_id
(
request
.
collection_name
(),
channel_id
,
timestamp
));
int
this_thread
=
omp_get_thread_num
();
auto
callback
=
[
&
stats
,
&
msg_sended
,
this_thread
](
Result
result
,
const
pulsar
::
MessageId
&
messageId
)
{
auto
callback
=
[
&
stats
,
&
msg_sended
,
this_thread
](
Result
result
,
const
pulsar
::
MessageId
&
messageId
)
{
msg_sended
+=
1
;
if
(
result
!=
pulsar
::
ResultOk
)
{
stats
[
this_thread
]
=
Status
(
DB_ERROR
,
pulsar
::
strResult
(
result
));
...
...
@@ -239,9 +248,15 @@ Status MsgClientV2::SendMutMessage(const milvus::grpc::DeleteByIDParam &request,
};
paralle_mut_producers_
[
this_thread
]
->
sendAsync
(
mut_msg
,
callback
);
}
while
(
msg_sended
<
row_count
){
while
(
msg_sended
<
row_count
)
{
}
auto
end
=
stdclock
::
now
();
auto
data_size
=
request
.
ByteSize
();
LOG_SERVER_INFO_
<<
"InsertReq Batch size:"
<<
data_size
/
1024.0
/
1024.0
<<
"M, "
<<
"throughput: "
<<
data_size
/
std
::
chrono
::
duration_cast
<
std
::
chrono
::
milliseconds
>
(
end
-
start
).
count
()
*
1000
/
1024.0
/
1024
<<
"M/s"
;
for
(
auto
&
stat
:
stats
)
{
if
(
!
stat
.
ok
())
{
return
stat
;
...
...
@@ -294,7 +309,7 @@ Status MsgClientV2::SendQueryMessage(const milvus::grpc::SearchParam &request, u
}
MsgClientV2
::~
MsgClientV2
()
{
//
insert_delete_producer_->close();
//
insert_delete_producer_->close();
for
(
auto
&
producer
:
paralle_mut_producers_
)
{
producer
->
close
();
}
...
...
proxy/src/server/MessageWrapper.cpp
浏览文件 @
09e43b0b
#include "server/delivery/ReqScheduler.h"
#include "MessageWrapper.h"
#include "config/ServerConfig.h"
...
...
@@ -15,11 +16,25 @@ Status MessageWrapper::Init() {
int
client_id
=
config
.
proxy_id
();
msg_client_
=
std
::
make_shared
<
message_client
::
MsgClientV2
>
(
client_id
,
pulsar_server_addr
,
config
.
pulsar
.
topicnum
());
auto
status
=
msg_client_
->
Init
(
"InsertOrDelete"
,
"Search"
,
"TimeSync"
,
"SearchById"
,
"SearchResult"
);
if
(
!
status
.
ok
()){
return
status
;
}
// timeSync
time_sync_
=
std
::
make_shared
<
timesync
::
TimeSync
>
(
client_id
,
GetMessageTimeSyncTime
,
config
.
timesync
.
interval
(),
pulsar_server_addr
,
"TimeSync"
);
return
status
;
}
const
std
::
shared_ptr
<
message_client
::
MsgClientV2
>
&
MessageWrapper
::
MessageClient
()
{
return
msg_client_
;
}
void
MessageWrapper
::
Stop
()
{
if
(
time_sync_
!=
nullptr
){
time_sync_
->
Stop
();
time_sync_
=
nullptr
;
}
msg_client_
=
nullptr
;
}
}
}
\ No newline at end of file
proxy/src/server/MessageWrapper.h
浏览文件 @
09e43b0b
#pragma once
#include "message_client/ClientV2.h"
#include "timesync/TimeSync.h"
namespace
milvus
{
namespace
server
{
...
...
@@ -14,11 +15,14 @@ class MessageWrapper {
const
std
::
shared_ptr
<
message_client
::
MsgClientV2
>&
MessageClient
();
void
Stop
();
private:
MessageWrapper
()
=
default
;
private:
std
::
shared_ptr
<
message_client
::
MsgClientV2
>
msg_client_
;
std
::
shared_ptr
<
milvus
::
timesync
::
TimeSync
>
time_sync_
;
};
}
...
...
proxy/src/server/MetaWrapper.cpp
浏览文件 @
09e43b0b
...
...
@@ -70,7 +70,7 @@ Status MetaWrapper::Init() {
return
SyncMeta
();
}
catch
(
const
std
::
exception
&
e
)
{
return
Status
(
DB_ERROR
,
"
Init meta erro
r"
);
return
Status
(
DB_ERROR
,
"
Can not connect to meta serve
r"
);
}
}
...
...
@@ -173,5 +173,9 @@ int64_t MetaWrapper::CountCollection(const std::string &collection_name) {
return
count
;
}
void
MetaWrapper
::
Stop
()
{
watcher_
->
Cancel
();
}
}
}
\ No newline at end of file
proxy/src/server/MetaWrapper.h
浏览文件 @
09e43b0b
...
...
@@ -34,6 +34,9 @@ class MetaWrapper {
int64_t
CountCollection
(
const
std
::
string
&
collection_name
);
void
Stop
();
private:
bool
IsCollectionMetaKey
(
const
std
::
string
&
key
);
...
...
proxy/src/server/Server.cpp
浏览文件 @
09e43b0b
...
...
@@ -236,13 +236,6 @@ Status
Server
::
StartService
()
{
Status
stat
;
// timeSync
// client id should same to MessageWrapper
int
client_id
=
0
;
std
::
string
pulsar_server_addr
(
std
::
string
{
"pulsar://"
}
+
config
.
pulsar
.
address
()
+
":"
+
std
::
to_string
(
config
.
pulsar
.
port
()));
static
timesync
::
TimeSync
syc
(
client_id
,
GetMessageTimeSyncTime
,
config
.
timesync
.
interval
(),
pulsar_server_addr
,
"TimeSync"
);
// stat = engine::KnowhereResource::Initialize();
if
(
!
stat
.
ok
())
{
LOG_SERVER_ERROR_
<<
"KnowhereResource initialize fail: "
<<
stat
.
message
();
...
...
@@ -274,6 +267,8 @@ void
Server
::
StopService
()
{
// storage::S3ClientWrapper::GetInstance().StopService();
grpc
::
GrpcServer
::
GetInstance
().
Stop
();
MetaWrapper
::
GetInstance
().
Stop
();
MessageWrapper
::
GetInstance
().
Stop
();
}
void
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录