Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
2ab5e246
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
2ab5e246
编写于
5月 31, 2019
作者:
Y
yu yunfeng
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add metrics
Former-commit-id: d7acd398be8aeee11de3f972d9bcf6e84df39358
上级
d5712b79
变更
10
隐藏空白更改
内联
并排
Showing
10 changed file
with
74 addition
and
16 deletion
+74
-16
cpp/CHANGELOG.md
cpp/CHANGELOG.md
+1
-0
cpp/src/db/DBImpl.inl
cpp/src/db/DBImpl.inl
+17
-3
cpp/src/db/MemManager.inl
cpp/src/db/MemManager.inl
+7
-0
cpp/src/main.cpp
cpp/src/main.cpp
+1
-1
cpp/src/metrics/MetricBase.h
cpp/src/metrics/MetricBase.h
+5
-1
cpp/src/metrics/Metrics.cpp
cpp/src/metrics/Metrics.cpp
+0
-1
cpp/src/metrics/PrometheusMetrics.h
cpp/src/metrics/PrometheusMetrics.h
+38
-6
cpp/src/server/Server.cpp
cpp/src/server/Server.cpp
+2
-2
cpp/unittest/metrics/CMakeLists.txt
cpp/unittest/metrics/CMakeLists.txt
+2
-1
cpp/unittest/metrics/metrics_test.cpp
cpp/unittest/metrics/metrics_test.cpp
+1
-1
未找到文件。
cpp/CHANGELOG.md
浏览文件 @
2ab5e246
...
...
@@ -26,3 +26,4 @@ Please mark all change in change log and use the ticket from JIRA.
-
MS-30 - Use faiss v1.5.2
-
MS-32 - Fix thrift error
-
MS-34 - Fix prometheus-cpp thirdparty
-
MS-37 - Add query, cache usage, disk write speed and file data size metrics
cpp/src/db/DBImpl.inl
浏览文件 @
2ab5e246
...
...
@@ -71,17 +71,26 @@ Status DBImpl<EngineT>::InsertVectors(const std::string& table_id_,
// server::Metrics::GetInstance().add_vector_duration_seconds_quantiles().Observe((average_time));
if (!status.ok()) {
server::Metrics::GetInstance().AddVectorsFailTotalIncrement(n);
server::Metrics::GetInstance().AddVectorsFailGaugeSet(n);
return status;
}
server::Metrics::GetInstance().AddVectorsSuccessTotalIncrement(n);
server::Metrics::GetInstance().AddVectorsSuccessGaugeSet(n);
}
template<typename EngineT>
Status DBImpl<EngineT>::Query(const std::string &table_id, size_t k, size_t nq,
const float *vectors, QueryResults &results) {
auto start_time = METRICS_NOW_TIME;
meta::DatesT dates = {meta::Meta::GetDate()};
return Query(table_id, k, nq, vectors, dates, results);
Status result = Query(table_id, k, nq, vectors, dates, results);
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time,end_time);
auto average_time = total_time / nq;
for (int i = 0; i < nq; ++i) {
server::Metrics::GetInstance().QueryResponseSummaryObserve(average_time);
}
return result;
}
template<typename EngineT>
...
...
@@ -250,7 +259,12 @@ void DBImpl<EngineT>::BackgroundTimerTask(int interval) {
if (shutting_down_.load(std::memory_order_acquire)) break;
std::this_thread::sleep_for(std::chrono::seconds(interval));
int64_t cache_total = cache::CpuCacheMgr::GetInstance()->CacheUsage();
LOG(DEBUG) << "Cache usage " << cache_total;
server::Metrics::GetInstance().CacheUsageGaugeSet(static_cast<double>(cache_total));
long size;
Size(size);
server::Metrics::GetInstance().DataFileSizeGaugeSet(size);
TrySchedule();
}
}
...
...
cpp/src/db/MemManager.inl
浏览文件 @
2ab5e246
...
...
@@ -8,6 +8,7 @@
#include "MemManager.h"
#include "Meta.h"
#include "MetaConsts.h"
#include "metrics/Metrics.h"
#include <iostream>
#include <sstream>
...
...
@@ -48,8 +49,14 @@ template<typename EngineT>
Status MemVectors<EngineT>::Serialize(std::string& table_id) {
table_id = schema_.table_id;
auto size = ApproximateSize();
auto start_time = METRICS_NOW_TIME;
pEE_->Serialize();
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
schema_.size = size;
server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet(size/total_time);
schema_.file_type = (size >= options_.index_trigger_size) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
...
...
cpp/src/main.cpp
浏览文件 @
2ab5e246
...
...
@@ -11,6 +11,7 @@
#include <string>
#include <signal.h>
#include <easylogging++.h>
#include "metrics/Metrics.h"
#include "utils/SignalUtil.h"
#include "utils/CommonUtil.h"
...
...
@@ -25,7 +26,6 @@ using namespace zilliz::vecwise;
int
main
(
int
argc
,
char
*
argv
[])
{
printf
(
"Vecwise engine server start...
\n
"
);
// zilliz::lib::gpu::InitMemoryAllocator();
signal
(
SIGINT
,
server
::
SignalUtil
::
HandleSignal
);
...
...
cpp/src/metrics/MetricBase.h
浏览文件 @
2ab5e246
...
...
@@ -64,7 +64,11 @@ class MetricsBase{
virtual
void
IndexFileSizeGaugeSet
(
double
value
)
{};
virtual
void
RawFileSizeGaugeSet
(
double
value
)
{};
virtual
void
FaissDiskLoadIOSpeedGaugeSet
(
double
value
)
{};
virtual
void
QueryResponseSummaryObserve
(
double
value
)
{};
virtual
void
DiskStoreIOSpeedGaugeSet
(
double
value
)
{};
virtual
void
DataFileSizeGaugeSet
(
double
value
)
{};
virtual
void
AddVectorsSuccessGaugeSet
(
double
value
)
{};
virtual
void
AddVectorsFailGaugeSet
(
double
value
)
{};
};
...
...
cpp/src/metrics/Metrics.cpp
浏览文件 @
2ab5e246
...
...
@@ -4,7 +4,6 @@
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "Metrics.h"
#include "PrometheusMetrics.h"
...
...
cpp/src/metrics/PrometheusMetrics.h
浏览文件 @
2ab5e246
...
...
@@ -97,7 +97,11 @@ class PrometheusMetrics: public MetricsBase {
void
RawFileSizeTotalIncrement
(
double
value
=
1
)
{
if
(
startup_
)
raw_file_size_total_
.
Increment
(
value
);};
void
IndexFileSizeGaugeSet
(
double
value
)
{
if
(
startup_
)
index_file_size_gauge_
.
Set
(
value
);};
void
RawFileSizeGaugeSet
(
double
value
)
{
if
(
startup_
)
raw_file_size_gauge_
.
Set
(
value
);};
void
QueryResponseSummaryObserve
(
double
value
)
{
if
(
startup_
)
query_response_summary_
.
Observe
(
value
);};
void
DiskStoreIOSpeedGaugeSet
(
double
value
)
{
if
(
startup_
)
disk_store_IO_speed_gauge_
.
Set
(
value
);};
void
DataFileSizeGaugeSet
(
double
value
)
{
if
(
startup_
)
data_file_size_gauge_
.
Set
(
value
);};
void
AddVectorsSuccessGaugeSet
(
double
value
)
{
if
(
startup_
)
add_vectors_success_gauge_
.
Set
(
value
);};
void
AddVectorsFailGaugeSet
(
double
value
)
{
if
(
startup_
)
add_vectors_fail_gauge_
.
Set
(
value
);};
...
...
@@ -295,11 +299,6 @@ class PrometheusMetrics: public MetricsBase {
////all form Cache.cpp
//record cache usage, when insert/erase/clear/free
prometheus
::
Family
<
prometheus
::
Gauge
>
&
cache_usage_
=
prometheus
::
BuildGauge
()
.
Name
(
"cache_usage"
)
.
Help
(
"total bytes that cache used"
)
.
Register
(
*
registry_
);
prometheus
::
Gauge
&
cache_usage_gauge_
=
cache_usage_
.
Add
({});
////all from Meta.cpp
...
...
@@ -386,6 +385,39 @@ class PrometheusMetrics: public MetricsBase {
.
Register
(
*
registry_
);
prometheus
::
Counter
&
cache_access_total_
=
cache_access_
.
Add
({});
// record cache usage and %
prometheus
::
Family
<
prometheus
::
Gauge
>
&
cache_usage_
=
prometheus
::
BuildGauge
()
.
Name
(
"cache_usage_bytes"
)
.
Help
(
"current cache usage by bytes"
)
.
Register
(
*
registry_
);
prometheus
::
Gauge
&
cache_usage_gauge_
=
cache_usage_
.
Add
({});
// record query response
using
Quantiles
=
std
::
vector
<
prometheus
::
detail
::
CKMSQuantiles
::
Quantile
>
;
prometheus
::
Family
<
prometheus
::
Summary
>
&
query_response_
=
prometheus
::
BuildSummary
()
.
Name
(
"query_response_summary"
)
.
Help
(
"query response summary"
)
.
Register
(
*
registry_
);
prometheus
::
Summary
&
query_response_summary_
=
query_response_
.
Add
({},
Quantiles
{{
0.95
,
0.00
},{
0.9
,
0.05
},{
0.8
,
0.1
}});
prometheus
::
Family
<
prometheus
::
Gauge
>
&
disk_store_IO_speed_
=
prometheus
::
BuildGauge
()
.
Name
(
"disk_store_IO_speed_bytes_per_microseconds"
)
.
Help
(
"disk_store_IO_speed"
)
.
Register
(
*
registry_
);
prometheus
::
Gauge
&
disk_store_IO_speed_gauge_
=
disk_store_IO_speed_
.
Add
({});
prometheus
::
Family
<
prometheus
::
Gauge
>
&
data_file_size_
=
prometheus
::
BuildGauge
()
.
Name
(
"data_file_size_bytes"
)
.
Help
(
"data file size by bytes"
)
.
Register
(
*
registry_
);
prometheus
::
Gauge
&
data_file_size_gauge_
=
data_file_size_
.
Add
({});
prometheus
::
Family
<
prometheus
::
Gauge
>
&
add_vectors_
=
prometheus
::
BuildGauge
()
.
Name
(
"add_vectors"
)
.
Help
(
"current added vectors"
)
.
Register
(
*
registry_
);
prometheus
::
Gauge
&
add_vectors_success_gauge_
=
add_vectors_
.
Add
({{
"outcome"
,
"success"
}});
prometheus
::
Gauge
&
add_vectors_fail_gauge_
=
add_vectors_
.
Add
({{
"outcome"
,
"fail"
}});
};
...
...
cpp/src/server/Server.cpp
浏览文件 @
2ab5e246
...
...
@@ -138,7 +138,7 @@ int
Server
::
Start
()
{
// server::Metrics::GetInstance().Init();
// server::Metrics::GetInstance().exposer_ptr()->RegisterCollectable(server::Metrics::GetInstance().registry_ptr());
server
::
Metrics
::
GetInstance
().
Init
();
//
server::Metrics::GetInstance().Init();
if
(
daemonized_
)
{
Daemonize
();
...
...
@@ -177,7 +177,7 @@ Server::Start() {
signal
(
SIGINT
,
SignalUtil
::
HandleSignal
);
signal
(
SIGHUP
,
SignalUtil
::
HandleSignal
);
signal
(
SIGTERM
,
SignalUtil
::
HandleSignal
);
server
::
Metrics
::
GetInstance
().
Init
();
SERVER_LOG_INFO
<<
"Vecwise server is running..."
;
StartService
();
...
...
cpp/unittest/metrics/CMakeLists.txt
浏览文件 @
2ab5e246
...
...
@@ -31,7 +31,8 @@ set(require_files
../../src/metrics/Metrics.cpp
# ../../src/cache/CacheMgr.cpp
# ../../src/metrics/PrometheusMetrics.cpp
../../src/metrics/PrometheusMetrics.cpp
../../src/metrics/MetricBase.h
../../src/server/ServerConfig.cpp
../../src/utils/CommonUtil.cpp
../../src/utils/TimeRecorder.cpp
...
...
cpp/unittest/metrics/metrics_test.cpp
浏览文件 @
2ab5e246
...
...
@@ -32,7 +32,7 @@ TEST_F(DBTest, Metric_Tes) {
// server::Metrics::GetInstance().exposer_ptr()->RegisterCollectable(server::Metrics::GetInstance().registry_ptr());
server
::
Metrics
::
GetInstance
().
Init
();
// server::PrometheusMetrics::GetInstance().exposer_ptr()->RegisterCollectable(server::PrometheusMetrics::GetInstance().registry_ptr());
zilliz
::
vecwise
::
cache
::
CpuCacheMgr
::
GetInstance
()
->
SetCapacity
(
1
*
1024
*
1024
*
1024
);
zilliz
::
vecwise
::
cache
::
CpuCacheMgr
::
GetInstance
()
->
SetCapacity
(
4
*
1024
*
1024
*
1024
);
std
::
cout
<<
zilliz
::
vecwise
::
cache
::
CpuCacheMgr
::
GetInstance
()
->
CacheCapacity
()
<<
std
::
endl
;
static
const
std
::
string
group_name
=
"test_group"
;
static
const
int
group_dim
=
256
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录