diff --git a/cpp/src/metrics/MetricBase.h b/cpp/src/metrics/MetricBase.h index c18e857e4e3f2f5e48dd64463477e1ffb990e75d..fe9b246503b4095510910070b7faad5df2ee1ba0 100644 --- a/cpp/src/metrics/MetricBase.h +++ b/cpp/src/metrics/MetricBase.h @@ -22,40 +22,21 @@ class MetricsBase{ } virtual ServerError Init() {}; - virtual void AddGroupSuccessTotalIncrement(double value = 1) {}; - virtual void AddGroupFailTotalIncrement(double value = 1) {}; - virtual void HasGroupSuccessTotalIncrement(double value = 1) {}; - virtual void HasGroupFailTotalIncrement(double value = 1) {}; - virtual void GetGroupSuccessTotalIncrement(double value = 1) {}; - virtual void GetGroupFailTotalIncrement(double value = 1) {}; - virtual void GetGroupFilesSuccessTotalIncrement(double value = 1) {}; - virtual void GetGroupFilesFailTotalIncrement(double value = 1) {}; + virtual void AddVectorsSuccessTotalIncrement(double value = 1) {}; virtual void AddVectorsFailTotalIncrement(double value = 1) {}; virtual void AddVectorsDurationHistogramOberve(double value) {}; - virtual void SearchSuccessTotalIncrement(double value = 1) {}; - virtual void SearchFailTotalIncrement(double value = 1) {}; - virtual void SearchDurationHistogramObserve(double value) {}; + virtual void RawFileSizeHistogramObserve(double value) {}; virtual void IndexFileSizeHistogramObserve(double value) {}; virtual void BuildIndexDurationSecondsHistogramObserve(double value) {}; - virtual void AllBuildIndexDurationSecondsHistogramObserve(double value) {}; - virtual void CacheUsageGaugeIncrement(double value = 1) {}; - virtual void CacheUsageGaugeDecrement(double value = 1) {}; + virtual void CacheUsageGaugeSet(double value) {}; - virtual void MetaVisitTotalIncrement(double value = 1) {}; - virtual void MetaVisitDurationSecondsHistogramObserve(double value) {}; - virtual void MemUsagePercentGaugeSet(double value) {}; - virtual void MemUsagePercentGaugeIncrement(double value = 1) {}; - virtual void MemUsagePercentGaugeDecrement(double value = 1) {}; - virtual void MemUsageTotalGaugeSet(double value) {}; - virtual void MemUsageTotalGaugeIncrement(double value = 1) {}; - virtual void MemUsageTotalGaugeDecrement(double value = 1) {}; + virtual void MetaAccessTotalIncrement(double value = 1) {}; virtual void MetaAccessDurationSecondsHistogramObserve(double value) {}; virtual void FaissDiskLoadDurationSecondsHistogramObserve(double value) {}; virtual void FaissDiskLoadSizeBytesHistogramObserve(double value) {}; - virtual void FaissDiskLoadIOSpeedHistogramObserve(double value) {}; virtual void CacheAccessTotalIncrement(double value = 1) {}; virtual void MemTableMergeDurationSecondsHistogramObserve(double value) {}; virtual void SearchIndexDataDurationSecondsHistogramObserve(double value) {}; diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index d0d50800adb41eedbda6bc2cfdac70e210eedcaf..df0516344e75614527e12bc0c60cbad8d6d23a76 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -60,9 +60,6 @@ PrometheusMetrics::GPUPercentGaugeSet() { if(!startup_) return; int numDevide = server::SystemInfo::GetInstance().num_device(); std::vector values = server::SystemInfo::GetInstance().GPUPercent(); -// for (int i = 0; i < numDevide; ++i) { -// GPU_percent_gauges_[i].Set(static_cast(values[i])); -// } if(numDevide >= 1) GPU0_percent_gauge_.Set(static_cast(values[0])); if(numDevide >= 2) GPU1_percent_gauge_.Set(static_cast(values[1])); if(numDevide >= 3) GPU2_percent_gauge_.Set(static_cast(values[2])); @@ -77,13 +74,10 @@ PrometheusMetrics::GPUPercentGaugeSet() { void PrometheusMetrics::GPUMemoryUsageGaugeSet() { if(!startup_) return; - int numDevide = server::SystemInfo::GetInstance().num_device(); std::vector values = server::SystemInfo::GetInstance().GPUMemoryUsed(); constexpr unsigned long long MtoB = 1024*1024; int numDevice = values.size(); -// for (int i = 0; i < numDevice; ++i) { -// GPU_memory_usage_gauges_[i].Set(values[i]/MtoB); -// } + if(numDevice >=1) GPU0_memory_usage_gauge_.Set(values[0]/MtoB); if(numDevice >=2) GPU1_memory_usage_gauge_.Set(values[1]/MtoB); if(numDevice >=3) GPU2_memory_usage_gauge_.Set(values[2]/MtoB); @@ -146,21 +140,6 @@ void PrometheusMetrics::OctetsSet() { outoctets_gauge_.Set((in_and_out_octets.second-old_outoctets)/total_second); } -//void PrometheusMetrics::GpuPercentInit() { -// int num_device = SystemInfo::GetInstance().num_device(); -// constexpr char device_number[] = "DeviceNum"; -// for(int i = 0; i < num_device; ++ i) { -// GPU_percent_gauges_.emplace_back(GPU_percent_.Add({{device_number,std::to_string(i)}})); -// } -// -//} -//void PrometheusMetrics::GpuMemoryInit() { -// int num_device = SystemInfo::GetInstance().num_device(); -// constexpr char device_number[] = "DeviceNum"; -// for(int i = 0; i < num_device; ++ i) { -// GPU_memory_usage_gauges_.emplace_back(GPU_memory_usage_.Add({{device_number,std::to_string(i)}})); -// } -//} } diff --git a/cpp/src/metrics/PrometheusMetrics.h b/cpp/src/metrics/PrometheusMetrics.h index 902e79b5a5676147bd83d36e18e2bb0c32c004d0..b7e991e1c2a2a505638b60251c9957891bf068d7 100644 --- a/cpp/src/metrics/PrometheusMetrics.h +++ b/cpp/src/metrics/PrometheusMetrics.h @@ -34,10 +34,6 @@ class PrometheusMetrics: public MetricsBase { public: static PrometheusMetrics & GetInstance() { -// switch(MetricCollectorType) { -// case: prometheus:: -// static -// } static PrometheusMetrics instance; return instance; } @@ -49,46 +45,21 @@ class PrometheusMetrics: public MetricsBase { std::shared_ptr exposer_ptr_; std::shared_ptr registry_ = std::make_shared(); bool startup_ = false; -// void GpuPercentInit(); -// void GpuMemoryInit(); public: - void AddGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) add_group_success_total_.Increment(value);}; - void AddGroupFailTotalIncrement(double value = 1.0) override { if(startup_) add_group_fail_total_.Increment(value);}; - void HasGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) has_group_success_total_.Increment(value);}; - void HasGroupFailTotalIncrement(double value = 1.0) override { if(startup_) has_group_fail_total_.Increment(value);}; - void GetGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) get_group_success_total_.Increment(value);}; - void GetGroupFailTotalIncrement(double value = 1.0) override { if(startup_) get_group_fail_total_.Increment(value);}; - void GetGroupFilesSuccessTotalIncrement(double value = 1.0) override { if(startup_) get_group_files_success_total_.Increment(value);}; - void GetGroupFilesFailTotalIncrement(double value = 1.0) override { if(startup_) get_group_files_fail_total_.Increment(value);}; void AddVectorsSuccessTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_success_total_.Increment(value);}; void AddVectorsFailTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_fail_total_.Increment(value);}; void AddVectorsDurationHistogramOberve(double value) override { if(startup_) add_vectors_duration_histogram_.Observe(value);}; - void SearchSuccessTotalIncrement(double value = 1.0) override { if(startup_) search_success_total_.Increment(value);}; - void SearchFailTotalIncrement(double value = 1.0) override { if(startup_) search_fail_total_.Increment(value); }; - void SearchDurationHistogramObserve(double value) override { if(startup_) search_duration_histogram_.Observe(value);}; void RawFileSizeHistogramObserve(double value) override { if(startup_) raw_files_size_histogram_.Observe(value);}; void IndexFileSizeHistogramObserve(double value) override { if(startup_) index_files_size_histogram_.Observe(value);}; void BuildIndexDurationSecondsHistogramObserve(double value) override { if(startup_) build_index_duration_seconds_histogram_.Observe(value);}; - void AllBuildIndexDurationSecondsHistogramObserve(double value) override { if(startup_) all_build_index_duration_seconds_histogram_.Observe(value);}; - void CacheUsageGaugeIncrement(double value = 1.0) override { if(startup_) cache_usage_gauge_.Increment(value);}; - void CacheUsageGaugeDecrement(double value = 1.0) override { if(startup_) cache_usage_gauge_.Decrement(value);}; void CacheUsageGaugeSet(double value) override { if(startup_) cache_usage_gauge_.Set(value);}; -// void MetaVisitTotalIncrement(double value = 1.0) override { meta_visit_total_.Increment(value);}; -// void MetaVisitDurationSecondsHistogramObserve(double value) override { meta_visit_duration_seconds_histogram_.Observe(value);}; - void MemUsagePercentGaugeSet(double value) override { if(startup_) mem_usage_percent_gauge_.Set(value);}; - void MemUsagePercentGaugeIncrement(double value = 1.0) override { if(startup_) mem_usage_percent_gauge_.Increment(value);}; - void MemUsagePercentGaugeDecrement(double value = 1.0) override { if(startup_) mem_usage_percent_gauge_.Decrement(value);}; - void MemUsageTotalGaugeSet(double value) override { if(startup_) mem_usage_total_gauge_.Set(value);}; - void MemUsageTotalGaugeIncrement(double value = 1.0) override { if(startup_) mem_usage_total_gauge_.Increment(value);}; - void MemUsageTotalGaugeDecrement(double value = 1.0) override { if(startup_) mem_usage_total_gauge_.Decrement(value);}; void MetaAccessTotalIncrement(double value = 1) override { if(startup_) meta_access_total_.Increment(value);}; void MetaAccessDurationSecondsHistogramObserve(double value) override { if(startup_) meta_access_duration_seconds_histogram_.Observe(value);}; void FaissDiskLoadDurationSecondsHistogramObserve(double value) override { if(startup_) faiss_disk_load_duration_seconds_histogram_.Observe(value);}; void FaissDiskLoadSizeBytesHistogramObserve(double value) override { if(startup_) faiss_disk_load_size_bytes_histogram_.Observe(value);}; -// void FaissDiskLoadIOSpeedHistogramObserve(double value) { if(startup_) faiss_disk_load_IO_speed_histogram_.Observe(value);}; void FaissDiskLoadIOSpeedGaugeSet(double value) override { if(startup_) faiss_disk_load_IO_speed_gauge_.Set(value);}; void CacheAccessTotalIncrement(double value = 1) override { if(startup_) cache_access_total_.Increment(value);}; diff --git a/cpp/src/metrics/SystemInfo.cpp b/cpp/src/metrics/SystemInfo.cpp index d4ea2de575a0a0300a89b741880392a21e57b406..a64cbc49929168b6a64950ad72e5ef053c39e914 100644 --- a/cpp/src/metrics/SystemInfo.cpp +++ b/cpp/src/metrics/SystemInfo.cpp @@ -135,46 +135,6 @@ SystemInfo::CPUPercent() { return percent; } -//std::unordered_map> -//SystemInfo::GetGPUMemPercent(){ -// // return GPUID: MEM% -// -// //write GPU info to a file -// system("nvidia-smi pmon -c 1 > GPUInfo.txt"); -// int pid = (int)getpid(); -// -// //parse line -// std::ifstream read_file; -// read_file.open("GPUInfo.txt"); -// std::string line; -// while(getline(read_file, line)){ -// std::vector words = split(line); -// // 0 1 2 3 4 5 6 7 -// //words stand for gpuindex, pid, type, sm, mem, enc, dec, command respectively -// if(std::stoi(words[1]) != pid) continue; -// int GPUindex = std::stoi(words[0]); -// double sm_percent = std::stod(words[3]); -// double mem_percent = std::stod(words[4]); -// -// } -// -//} - -//std::vector -//SystemInfo::split(std::string input) { -// std::vector words; -// input += " "; -// int word_start = 0; -// for (int i = 0; i < input.size(); ++i) { -// if(input[i] != ' ') continue; -// if(input[i] == ' ') { -// word_start = i + 1; -// continue; -// } -// words.push_back(input.substr(word_start,i-word_start)); -// } -// return words; -//} std::vector SystemInfo::GPUPercent() { diff --git a/cpp/src/metrics/SystemInfo.h b/cpp/src/metrics/SystemInfo.h index eeff4b61f5cda9082b21b15b03734107b354cb05..2562e316e4362984417029ecfa29abb15b1c9062 100644 --- a/cpp/src/metrics/SystemInfo.h +++ b/cpp/src/metrics/SystemInfo.h @@ -59,8 +59,6 @@ class SystemInfo { double MemoryPercent(); double CPUPercent(); std::pair Octets(); -// std::unordered_map> GetGPUMemPercent() {}; -// std::vector split(std::string input) {}; std::vector GPUPercent(); std::vector GPUMemoryUsed(); diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 72e1b43867eb1291131f9d86cef08bb4a0cb002e..8c5b09638d429ea46213a535875f610ffdfaf909 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -55,6 +55,7 @@ set(count_test_src ${wrapper_src} ${metrics_src} metrics_test.cpp + prometheus_test.cpp ../db/utils.cpp ) diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index 72596dc79e98693240813492a97119fb84aa8efe..923c7b717baa26d48c5b2a7aa9fe7ccc5eb93229 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -24,7 +24,6 @@ using namespace zilliz::milvus; - TEST_F(DBTest, Metric_Tes) { server::SystemInfo::GetInstance().Init(); diff --git a/cpp/unittest/metrics/prometheus_test.cpp b/cpp/unittest/metrics/prometheus_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f1e239d73bf681e308fb707ac1242bb3f60d2787 --- /dev/null +++ b/cpp/unittest/metrics/prometheus_test.cpp @@ -0,0 +1,56 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ + +#include "metrics/PrometheusMetrics.h" + +#include +#include + +using namespace zilliz::milvus; + +TEST(PrometheusTest, Prometheus_Test){ + server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance(); + instance.Init(); + server::SystemInfo::GetInstance().Init(); + instance.AddVectorsSuccessTotalIncrement(); + instance.AddVectorsFailTotalIncrement(); + instance.AddVectorsDurationHistogramOberve(1.0); + instance.RawFileSizeHistogramObserve(1.0); + instance.IndexFileSizeHistogramObserve(1.0); + instance.BuildIndexDurationSecondsHistogramObserve(1.0); + instance.CacheUsageGaugeSet(1.0); + instance.MetaAccessTotalIncrement(); + instance.MetaAccessDurationSecondsHistogramObserve(1.0); + instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0); + instance.FaissDiskLoadSizeBytesHistogramObserve(1.0); + instance.FaissDiskLoadIOSpeedGaugeSet(1.0); + instance.CacheAccessTotalIncrement(); + instance.MemTableMergeDurationSecondsHistogramObserve(1.0); + instance.SearchIndexDataDurationSecondsHistogramObserve(1.0); + instance.SearchRawDataDurationSecondsHistogramObserve(1.0); + instance.IndexFileSizeTotalIncrement(); + instance.RawFileSizeTotalIncrement(); + instance.IndexFileSizeGaugeSet(1.0); + instance.RawFileSizeGaugeSet(1.0); + instance.QueryResponseSummaryObserve(1.0); + instance.DiskStoreIOSpeedGaugeSet(1.0); + instance.DataFileSizeGaugeSet(1.0); + instance.AddVectorsSuccessGaugeSet(1.0); + instance.AddVectorsFailGaugeSet(1.0); + instance.QueryVectorResponseSummaryObserve(1.0, 1); + instance.QueryVectorResponsePerSecondGaugeSet(1.0); + instance.CPUUsagePercentSet(); + instance.RAMUsagePercentSet(); + instance.QueryResponsePerSecondGaugeSet(1.0); + instance.GPUPercentGaugeSet(); + instance.GPUMemoryUsageGaugeSet(); + instance.AddVectorsPerSecondGaugeSet(1,1,1); + instance.QueryIndexTypePerSecondSet("IVF", 1.0); + instance.ConnectionGaugeIncrement(); + instance.ConnectionGaugeDecrement(); + instance.KeepingAliveCounterIncrement(); + instance.OctetsSet(); +} \ No newline at end of file