Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
9cb66ac8
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
9cb66ac8
编写于
6月 11, 2019
作者:
Y
yu yunfeng
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix
Former-commit-id: 00af0b4a367be2dfb26fff09105482364fd56a8e
上级
c74d3686
变更
6
显示空白变更内容
内联
并排
Showing
6 changed file
with
94 addition
and
95 deletion
+94
-95
cpp/src/metrics/PrometheusMetrics.cpp
cpp/src/metrics/PrometheusMetrics.cpp
+26
-2
cpp/src/metrics/PrometheusMetrics.h
cpp/src/metrics/PrometheusMetrics.h
+5
-39
cpp/src/metrics/SystemInfo.cpp
cpp/src/metrics/SystemInfo.cpp
+47
-45
cpp/src/metrics/SystemInfo.h
cpp/src/metrics/SystemInfo.h
+4
-4
cpp/src/server/MegasearchThreadPoolServer.cpp
cpp/src/server/MegasearchThreadPoolServer.cpp
+11
-4
cpp/src/thrift/megasearch.thrift
cpp/src/thrift/megasearch.thrift
+1
-1
未找到文件。
cpp/src/metrics/PrometheusMetrics.cpp
浏览文件 @
9cb66ac8
...
...
@@ -37,6 +37,7 @@ PrometheusMetrics::Init() {
}
void
PrometheusMetrics
::
CPUUsagePercentSet
()
{
if
(
!
startup_
)
return
;
...
...
@@ -54,8 +55,11 @@ PrometheusMetrics::RAMUsagePercentSet() {
void
PrometheusMetrics
::
GPUPercentGaugeSet
()
{
if
(
!
startup_
)
return
;
int
numDevide
=
server
::
SystemInfo
::
GetInstance
().
NumD
evice
();
int
numDevide
=
server
::
SystemInfo
::
GetInstance
().
num_d
evice
();
std
::
vector
<
unsigned
int
>
values
=
server
::
SystemInfo
::
GetInstance
().
GPUPercent
();
// for (int i = 0; i < numDevide; ++i) {
// GPU_percent_gauges_[i].Set(static_cast<double>(values[i]));
// }
if
(
numDevide
>=
1
)
GPU0_percent_gauge_
.
Set
(
static_cast
<
double
>
(
values
[
0
]));
if
(
numDevide
>=
2
)
GPU1_percent_gauge_
.
Set
(
static_cast
<
double
>
(
values
[
1
]));
if
(
numDevide
>=
3
)
GPU2_percent_gauge_
.
Set
(
static_cast
<
double
>
(
values
[
2
]));
...
...
@@ -70,9 +74,13 @@ PrometheusMetrics::GPUPercentGaugeSet() {
void
PrometheusMetrics
::
GPUMemoryUsageGaugeSet
()
{
if
(
!
startup_
)
return
;
int
numDevide
=
server
::
SystemInfo
::
GetInstance
().
num_device
();
std
::
vector
<
unsigned
long
long
>
values
=
server
::
SystemInfo
::
GetInstance
().
GPUMemoryUsed
();
unsigned
long
long
MtoB
=
1024
*
1024
;
constexpr
unsigned
long
long
MtoB
=
1024
*
1024
;
int
numDevice
=
values
.
size
();
// for (int i = 0; i < numDevice; ++i) {
// GPU_memory_usage_gauges_[i].Set(values[i]/MtoB);
// }
if
(
numDevice
>=
1
)
GPU0_memory_usage_gauge_
.
Set
(
values
[
0
]
/
MtoB
);
if
(
numDevice
>=
2
)
GPU1_memory_usage_gauge_
.
Set
(
values
[
1
]
/
MtoB
);
if
(
numDevice
>=
3
)
GPU2_memory_usage_gauge_
.
Set
(
values
[
2
]
/
MtoB
);
...
...
@@ -111,6 +119,22 @@ void PrometheusMetrics::ConnectionGaugeDecrement() {
connection_gauge_
.
Decrement
();
}
//void PrometheusMetrics::GpuPercentInit() {
// int num_device = SystemInfo::GetInstance().num_device();
// constexpr char device_number[] = "DeviceNum";
// for(int i = 0; i < num_device; ++ i) {
// GPU_percent_gauges_.emplace_back(GPU_percent_.Add({{device_number,std::to_string(i)}}));
// }
//
//}
//void PrometheusMetrics::GpuMemoryInit() {
// int num_device = SystemInfo::GetInstance().num_device();
// constexpr char device_number[] = "DeviceNum";
// for(int i = 0; i < num_device; ++ i) {
// GPU_memory_usage_gauges_.emplace_back(GPU_memory_usage_.Add({{device_number,std::to_string(i)}}));
// }
//}
}
}
...
...
cpp/src/metrics/PrometheusMetrics.h
浏览文件 @
9cb66ac8
...
...
@@ -49,6 +49,8 @@ class PrometheusMetrics: public MetricsBase {
std
::
shared_ptr
<
prometheus
::
Exposer
>
exposer_ptr_
;
std
::
shared_ptr
<
prometheus
::
Registry
>
registry_
=
std
::
make_shared
<
prometheus
::
Registry
>
();
bool
startup_
=
false
;
// void GpuPercentInit();
// void GpuMemoryInit();
public:
void
AddGroupSuccessTotalIncrement
(
double
value
=
1.0
)
override
{
if
(
startup_
)
add_group_success_total_
.
Increment
(
value
);};
...
...
@@ -115,45 +117,7 @@ class PrometheusMetrics: public MetricsBase {
void
ConnectionGaugeDecrement
()
override
;
void
KeepingAliveCounterIncrement
(
double
value
=
1
)
override
{
if
(
startup_
)
keeping_alive_counter_
.
Increment
(
value
);};
// prometheus::Counter &connection_total() {return connection_total_; }
//
// prometheus::Counter &add_group_success_total() { return add_group_success_total_; }
// prometheus::Counter &add_group_fail_total() { return add_group_fail_total_; }
//
// prometheus::Counter &get_group_success_total() { return get_group_success_total_;}
// prometheus::Counter &get_group_fail_total() { return get_group_fail_total_;}
//
// prometheus::Counter &has_group_success_total() { return has_group_success_total_;}
// prometheus::Counter &has_group_fail_total() { return has_group_fail_total_;}
//
// prometheus::Counter &get_group_files_success_total() { return get_group_files_success_total_;};
// prometheus::Counter &get_group_files_fail_total() { return get_group_files_fail_total_;}
//
// prometheus::Counter &add_vectors_success_total() { return add_vectors_success_total_; }
// prometheus::Counter &add_vectors_fail_total() { return add_vectors_fail_total_; }
//
// prometheus::Histogram &add_vectors_duration_histogram() { return add_vectors_duration_histogram_;}
//
// prometheus::Counter &search_success_total() { return search_success_total_; }
// prometheus::Counter &search_fail_total() { return search_fail_total_; }
//
// prometheus::Histogram &search_duration_histogram() { return search_duration_histogram_; }
// prometheus::Histogram &raw_files_size_histogram() { return raw_files_size_histogram_; }
// prometheus::Histogram &index_files_size_histogram() { return index_files_size_histogram_; }
//
// prometheus::Histogram &build_index_duration_seconds_histogram() { return build_index_duration_seconds_histogram_; }
//
// prometheus::Histogram &all_build_index_duration_seconds_histogram() { return all_build_index_duration_seconds_histogram_; }
//
// prometheus::Gauge &cache_usage_gauge() { return cache_usage_gauge_; }
//
// prometheus::Counter &meta_visit_total() { return meta_visit_total_; }
//
// prometheus::Histogram &meta_visit_duration_seconds_histogram() { return meta_visit_duration_seconds_histogram_; }
//
// prometheus::Gauge &mem_usage_percent_gauge() { return mem_usage_percent_gauge_; }
//
// prometheus::Gauge &mem_usage_total_gauge() { return mem_usage_total_gauge_; }
...
...
@@ -477,6 +441,7 @@ class PrometheusMetrics: public MetricsBase {
prometheus
::
Gauge
&
GPU5_percent_gauge_
=
GPU_percent_
.
Add
({{
"DeviceNum"
,
"5"
}});
prometheus
::
Gauge
&
GPU6_percent_gauge_
=
GPU_percent_
.
Add
({{
"DeviceNum"
,
"6"
}});
prometheus
::
Gauge
&
GPU7_percent_gauge_
=
GPU_percent_
.
Add
({{
"DeviceNum"
,
"7"
}});
// std::vector<prometheus::Gauge> GPU_percent_gauges_;
...
...
@@ -494,6 +459,7 @@ class PrometheusMetrics: public MetricsBase {
prometheus
::
Gauge
&
GPU5_memory_usage_gauge_
=
GPU_memory_usage_
.
Add
({{
"DeviceNum"
,
"5"
}});
prometheus
::
Gauge
&
GPU6_memory_usage_gauge_
=
GPU_memory_usage_
.
Add
({{
"DeviceNum"
,
"6"
}});
prometheus
::
Gauge
&
GPU7_memory_usage_gauge_
=
GPU_memory_usage_
.
Add
({{
"DeviceNum"
,
"7"
}});
// std::vector<prometheus::Gauge> GPU_memory_usage_gauges_;
prometheus
::
Family
<
prometheus
::
Gauge
>
&
query_index_type_per_second_
=
prometheus
::
BuildGauge
()
.
Name
(
"query_index_throughtout_per_microsecond"
)
...
...
cpp/src/metrics/SystemInfo.cpp
浏览文件 @
9cb66ac8
...
...
@@ -56,7 +56,7 @@ void SystemInfo::Init() {
}
long
long
SystemInfo
::
p
arseLine
(
char
*
line
)
{
SystemInfo
::
P
arseLine
(
char
*
line
)
{
// This assumes that a digit will be found and the line ends in " Kb".
int
i
=
strlen
(
line
);
const
char
*
p
=
line
;
...
...
@@ -80,18 +80,20 @@ unsigned long
SystemInfo
::
GetProcessUsedMemory
()
{
//Note: this value is in KB!
FILE
*
file
=
fopen
(
"/proc/self/status"
,
"r"
);
constexpr
int64_t
line_length
=
128
;
long
long
result
=
-
1
;
char
line
[
128
];
constexpr
int64_t
KB_SIZE
=
1024
;
char
line
[
line_length
];
while
(
fgets
(
line
,
128
,
file
)
!=
NULL
){
while
(
fgets
(
line
,
line_length
,
file
)
!=
NULL
){
if
(
strncmp
(
line
,
"VmRSS:"
,
6
)
==
0
){
result
=
p
arseLine
(
line
);
result
=
P
arseLine
(
line
);
break
;
}
}
fclose
(
file
);
// return value in Byte
return
(
result
*
1024
);
return
(
result
*
KB_SIZE
);
}
...
...
@@ -128,46 +130,46 @@ SystemInfo::CPUPercent() {
return
percent
;
}
std
::
unordered_map
<
int
,
std
::
vector
<
double
>>
SystemInfo
::
GetGPUMemPercent
(){
// return GPUID: MEM%
//write GPU info to a file
system
(
"nvidia-smi pmon -c 1 > GPUInfo.txt"
);
int
pid
=
(
int
)
getpid
();
//parse line
std
::
ifstream
read_file
;
read_file
.
open
(
"GPUInfo.txt"
);
std
::
string
line
;
while
(
getline
(
read_file
,
line
)){
std
::
vector
<
std
::
string
>
words
=
split
(
line
);
// 0 1 2 3 4 5 6 7
//words stand for gpuindex, pid, type, sm, mem, enc, dec, command respectively
if
(
std
::
stoi
(
words
[
1
])
!=
pid
)
continue
;
int
GPUindex
=
std
::
stoi
(
words
[
0
]);
double
sm_percent
=
std
::
stod
(
words
[
3
]);
double
mem_percent
=
std
::
stod
(
words
[
4
]);
}
}
std
::
vector
<
std
::
string
>
SystemInfo
::
split
(
std
::
string
input
)
{
std
::
vector
<
std
::
string
>
words
;
input
+=
" "
;
int
word_start
=
0
;
for
(
int
i
=
0
;
i
<
input
.
size
();
++
i
)
{
if
(
input
[
i
]
!=
' '
)
continue
;
if
(
input
[
i
]
==
' '
)
{
word_start
=
i
+
1
;
continue
;
}
words
.
push_back
(
input
.
substr
(
word_start
,
i
-
word_start
));
}
return
words
;
}
//
std::unordered_map<int,std::vector<double>>
//
SystemInfo::GetGPUMemPercent(){
//
// return GPUID: MEM%
//
//
//write GPU info to a file
//
system("nvidia-smi pmon -c 1 > GPUInfo.txt");
//
int pid = (int)getpid();
//
//
//parse line
//
std::ifstream read_file;
//
read_file.open("GPUInfo.txt");
//
std::string line;
//
while(getline(read_file, line)){
//
std::vector<std::string> words = split(line);
//
// 0 1 2 3 4 5 6 7
//
//words stand for gpuindex, pid, type, sm, mem, enc, dec, command respectively
//
if(std::stoi(words[1]) != pid) continue;
//
int GPUindex = std::stoi(words[0]);
//
double sm_percent = std::stod(words[3]);
//
double mem_percent = std::stod(words[4]);
//
//
}
//
//
}
//
std::vector<std::string>
//
SystemInfo::split(std::string input) {
//
std::vector<std::string> words;
//
input += " ";
//
int word_start = 0;
//
for (int i = 0; i < input.size(); ++i) {
//
if(input[i] != ' ') continue;
//
if(input[i] == ' ') {
//
word_start = i + 1;
//
continue;
//
}
//
words.push_back(input.substr(word_start,i-word_start));
//
}
//
return words;
//
}
std
::
vector
<
unsigned
int
>
SystemInfo
::
GPUPercent
()
{
...
...
cpp/src/metrics/SystemInfo.h
浏览文件 @
9cb66ac8
...
...
@@ -42,14 +42,14 @@ class SystemInfo {
}
void
Init
();
int
NumDevice
()
{
return
num_device_
;};
long
long
p
arseLine
(
char
*
line
);
int
num_device
()
const
{
return
num_device_
;};
long
long
P
arseLine
(
char
*
line
);
unsigned
long
GetPhysicalMemory
();
unsigned
long
GetProcessUsedMemory
();
double
MemoryPercent
();
double
CPUPercent
();
std
::
unordered_map
<
int
,
std
::
vector
<
double
>>
GetGPUMemPercent
()
;
std
::
vector
<
std
::
string
>
split
(
std
::
string
input
)
;
// std::unordered_map<int,std::vector<double>> GetGPUMemPercent() {}
;
// std::vector<std::string> split(std::string input) {}
;
std
::
vector
<
unsigned
int
>
GPUPercent
();
std
::
vector
<
unsigned
long
long
>
GPUMemoryUsed
();
...
...
cpp/src/server/MegasearchThreadPoolServer.cpp
浏览文件 @
9cb66ac8
...
...
@@ -4,16 +4,20 @@
* Proprietary and confidential.
******************************************************************************/
#include "metrics/Metrics.h"
#include "MegasearchThreadPoolServer.h"
namespace
zilliz
{
namespace
vecwise
{
namespace
server
{
void
zilliz
::
vecwise
::
server
::
MegasearchThreadPoolServer
::
onClientConnected
(
const
std
::
shared_ptr
<
apache
::
thrift
::
server
::
TConnectedClient
>
&
pClient
)
{
void
MegasearchThreadPoolServer
::
onClientConnected
(
const
std
::
shared_ptr
<
apache
::
thrift
::
server
::
TConnectedClient
>
&
pClient
)
{
server
::
Metrics
::
GetInstance
().
ConnectionGaugeIncrement
();
TThreadPoolServer
::
onClientConnected
(
pClient
);
}
void
zilliz
::
vecwise
::
server
::
MegasearchThreadPoolServer
::
onClientDisconnected
(
apache
::
thrift
::
server
::
TConnectedClient
*
pClient
)
{
void
MegasearchThreadPoolServer
::
onClientDisconnected
(
apache
::
thrift
::
server
::
TConnectedClient
*
pClient
)
{
server
::
Metrics
::
GetInstance
().
ConnectionGaugeDecrement
();
TThreadPoolServer
::
onClientDisconnected
(
pClient
);
}
...
...
@@ -25,3 +29,6 @@ zilliz::vecwise::server::MegasearchThreadPoolServer::MegasearchThreadPoolServer(
:
TThreadPoolServer
(
processor
,
serverTransport
,
transportFactory
,
protocolFactory
,
threadManager
)
{
}
}
}
}
\ No newline at end of file
cpp/src/thrift/megasearch.thrift
浏览文件 @
9cb66ac8
...
...
@@ -29,7 +29,7 @@ exception Exception {
}
/**
* @brief Table Schema
*/
struct TableSchema {
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录