Merge branch 'branch-0.3.0' into 'branch-0.3.0'

MS - 130 Add prometheus_test

See merge request megasearch/vecwise_engine!131

Former-commit-id: ebb3c84b3a76f185658a293b91521cd98c0dbc56

cpp/CHANGELOG.md

@@ -46,7 +46,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-85 - add NetIO metric
 - MS-96 - add new query interface for specified files
 - MS-97 - Add S3 SDK for MinIO Storage
-
+- MS-130 - Add prometheus_test
 ## Task
 - MS-74 - Change README.md in cpp
 - MS-88 - Add support for arm architecture

cpp/conf/log_config.conf

 * GLOBAL:
     FORMAT                  =   "%datetime | %level | %logger | %msg"
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%H:%m}-global.log"
+    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-global.log"
     ENABLED                 =   true
     TO_FILE                 =   true
     TO_STANDARD_OUTPUT      =   false
@@ -8,12 +8,12 @@
     PERFORMANCE_TRACKING    =   false
     MAX_LOG_FILE_SIZE       =   2097152 ## Throw log files away after 2MB
 * DEBUG:
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%H:%m}-debug.log"
+    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-debug.log"
     ENABLED                 =   true
 * WARNING:
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%H:%m}-warning.log"
+    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-warning.log"
 * TRACE:
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%H:%m}-trace.log"
+    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-trace.log"
 * VERBOSE:
     FORMAT                  =   "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
     TO_FILE                 =   false
@@ -21,7 +21,7 @@
 ## Error logs
 * ERROR:
     ENABLED                 =   true
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log"
+    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-error.log"
 * FATAL:
     ENABLED                 =   true
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log"
+    FILENAME                =   "/opt/milvus/logs/milvus-%datetime{%H:%m}-fatal.log"

cpp/conf/server_config.yaml

@@ -5,7 +5,7 @@ server_config:
   mode: single                # milvus deployment type: single, cluster
 
 db_config:
-  db_path: /tmp/milvus             # milvus data storage path
+  db_path: /opt/milvus             # milvus data storage path
   db_backend_url: http://127.0.0.1 # meta database uri
   index_building_threshold: 1024   # index building trigger threshold, default: 1024, unit: MB
   archive_disk_threshold: 512      # triger archive action if storage size exceed this value, unit: GB
@@ -22,7 +22,7 @@ metric_config:
 
 
 license_config:         # license configure
-  license_path: "/tmp/milvus/system.license" # license file path
+  license_path: "/opt/milvus/system.license" # license file path
 
 cache_config:           # cache configure
   cpu_cache_capacity: 16   # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory

cpp/src/metrics/MetricBase.h

@@ -22,40 +22,21 @@ class MetricsBase{
     }
 
     virtual ServerError Init() {};
-    virtual void AddGroupSuccessTotalIncrement(double value = 1) {};
-    virtual void AddGroupFailTotalIncrement(double value = 1) {};
-    virtual void HasGroupSuccessTotalIncrement(double value = 1) {};
-    virtual void HasGroupFailTotalIncrement(double value = 1) {};
-    virtual void GetGroupSuccessTotalIncrement(double value = 1) {};
-    virtual void GetGroupFailTotalIncrement(double value = 1) {};
-    virtual void GetGroupFilesSuccessTotalIncrement(double value = 1) {};
-    virtual void GetGroupFilesFailTotalIncrement(double value = 1) {};
+
     virtual void AddVectorsSuccessTotalIncrement(double value = 1) {};
     virtual void AddVectorsFailTotalIncrement(double value = 1) {};
     virtual void AddVectorsDurationHistogramOberve(double value) {};
-    virtual void SearchSuccessTotalIncrement(double value = 1) {};
-    virtual void SearchFailTotalIncrement(double value = 1) {};
-    virtual void SearchDurationHistogramObserve(double value) {};
+
     virtual void RawFileSizeHistogramObserve(double value) {};
     virtual void IndexFileSizeHistogramObserve(double value) {};
     virtual void BuildIndexDurationSecondsHistogramObserve(double value) {};
-    virtual void AllBuildIndexDurationSecondsHistogramObserve(double value) {};
-    virtual void CacheUsageGaugeIncrement(double value = 1) {};
-    virtual void CacheUsageGaugeDecrement(double value = 1) {};
+
     virtual void CacheUsageGaugeSet(double value) {};
-    virtual void MetaVisitTotalIncrement(double value = 1) {};
-    virtual void MetaVisitDurationSecondsHistogramObserve(double value) {};
-    virtual void MemUsagePercentGaugeSet(double value) {};
-    virtual void MemUsagePercentGaugeIncrement(double value = 1) {};
-    virtual void MemUsagePercentGaugeDecrement(double value = 1) {};
-    virtual void MemUsageTotalGaugeSet(double value) {};
-    virtual void MemUsageTotalGaugeIncrement(double value = 1) {};
-    virtual void MemUsageTotalGaugeDecrement(double value = 1) {};
+
     virtual void MetaAccessTotalIncrement(double value = 1) {};
     virtual void MetaAccessDurationSecondsHistogramObserve(double value) {};
     virtual void FaissDiskLoadDurationSecondsHistogramObserve(double value) {};
     virtual void FaissDiskLoadSizeBytesHistogramObserve(double value) {};
-    virtual void FaissDiskLoadIOSpeedHistogramObserve(double value) {};
     virtual void CacheAccessTotalIncrement(double value = 1) {};
     virtual void MemTableMergeDurationSecondsHistogramObserve(double value) {};
     virtual void SearchIndexDataDurationSecondsHistogramObserve(double value) {};

cpp/src/metrics/PrometheusMetrics.cpp

@@ -60,9 +60,6 @@ PrometheusMetrics::GPUPercentGaugeSet() {
     if(!startup_) return;
     int numDevide = server::SystemInfo::GetInstance().num_device();
     std::vector<unsigned int> values = server::SystemInfo::GetInstance().GPUPercent();
-//    for (int i = 0; i < numDevide; ++i) {
-//        GPU_percent_gauges_[i].Set(static_cast<double>(values[i]));
-//    }
     if(numDevide >= 1) GPU0_percent_gauge_.Set(static_cast<double>(values[0]));
     if(numDevide >= 2) GPU1_percent_gauge_.Set(static_cast<double>(values[1]));
     if(numDevide >= 3) GPU2_percent_gauge_.Set(static_cast<double>(values[2]));
@@ -77,13 +74,10 @@ PrometheusMetrics::GPUPercentGaugeSet() {
 
 void PrometheusMetrics::GPUMemoryUsageGaugeSet() {
     if(!startup_) return;
-    int numDevide = server::SystemInfo::GetInstance().num_device();
     std::vector<unsigned long long> values = server::SystemInfo::GetInstance().GPUMemoryUsed();
     constexpr unsigned long long MtoB = 1024*1024;
     int numDevice = values.size();
-//    for (int i = 0; i < numDevice; ++i) {
-//        GPU_memory_usage_gauges_[i].Set(values[i]/MtoB);
-//    }
+
     if(numDevice >=1) GPU0_memory_usage_gauge_.Set(values[0]/MtoB);
     if(numDevice >=2) GPU1_memory_usage_gauge_.Set(values[1]/MtoB);
     if(numDevice >=3) GPU2_memory_usage_gauge_.Set(values[2]/MtoB);
@@ -146,21 +140,6 @@ void PrometheusMetrics::OctetsSet() {
     outoctets_gauge_.Set((in_and_out_octets.second-old_outoctets)/total_second);
 }
 
-//void PrometheusMetrics::GpuPercentInit() {
-//    int num_device = SystemInfo::GetInstance().num_device();
-//    constexpr char device_number[] = "DeviceNum";
-//    for(int i = 0; i < num_device; ++ i) {
-//        GPU_percent_gauges_.emplace_back(GPU_percent_.Add({{device_number,std::to_string(i)}}));
-//    }
-//
-//}
-//void PrometheusMetrics::GpuMemoryInit() {
-//    int num_device = SystemInfo::GetInstance().num_device();
-//    constexpr char device_number[] = "DeviceNum";
-//    for(int i = 0; i < num_device; ++ i) {
-//        GPU_memory_usage_gauges_.emplace_back(GPU_memory_usage_.Add({{device_number,std::to_string(i)}}));
-//    }
-//}
 
 
 }

cpp/src/metrics/PrometheusMetrics.h

@@ -34,10 +34,6 @@ class PrometheusMetrics: public MetricsBase {
  public:
     static PrometheusMetrics &
     GetInstance() {
-//        switch(MetricCollectorType) {
-//            case: prometheus::
-//                static
-//        }
         static PrometheusMetrics instance;
         return instance;
     }
@@ -49,46 +45,21 @@ class PrometheusMetrics: public MetricsBase {
     std::shared_ptr<prometheus::Exposer> exposer_ptr_;
     std::shared_ptr<prometheus::Registry> registry_ = std::make_shared<prometheus::Registry>();
     bool startup_ = false;
-//    void GpuPercentInit();
-//    void GpuMemoryInit();
  public:
-
-    void AddGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) add_group_success_total_.Increment(value);};
-    void AddGroupFailTotalIncrement(double value = 1.0) override { if(startup_) add_group_fail_total_.Increment(value);};
-    void HasGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) has_group_success_total_.Increment(value);};
-    void HasGroupFailTotalIncrement(double value = 1.0) override { if(startup_) has_group_fail_total_.Increment(value);};
-    void GetGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) get_group_success_total_.Increment(value);};
-    void GetGroupFailTotalIncrement(double value = 1.0) override { if(startup_) get_group_fail_total_.Increment(value);};
-    void GetGroupFilesSuccessTotalIncrement(double value = 1.0) override { if(startup_) get_group_files_success_total_.Increment(value);};
-    void GetGroupFilesFailTotalIncrement(double value = 1.0) override { if(startup_) get_group_files_fail_total_.Increment(value);};
+    void SetStartup(bool startup) {startup_ = startup;};
     void AddVectorsSuccessTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_success_total_.Increment(value);};
     void AddVectorsFailTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_fail_total_.Increment(value);};
     void AddVectorsDurationHistogramOberve(double value) override { if(startup_) add_vectors_duration_histogram_.Observe(value);};
-    void SearchSuccessTotalIncrement(double value = 1.0) override { if(startup_) search_success_total_.Increment(value);};
-    void SearchFailTotalIncrement(double value = 1.0) override { if(startup_) search_fail_total_.Increment(value); };
-    void SearchDurationHistogramObserve(double value) override { if(startup_) search_duration_histogram_.Observe(value);};
     void RawFileSizeHistogramObserve(double value) override { if(startup_) raw_files_size_histogram_.Observe(value);};
     void IndexFileSizeHistogramObserve(double value) override { if(startup_) index_files_size_histogram_.Observe(value);};
     void BuildIndexDurationSecondsHistogramObserve(double value) override { if(startup_) build_index_duration_seconds_histogram_.Observe(value);};
-    void AllBuildIndexDurationSecondsHistogramObserve(double value) override { if(startup_) all_build_index_duration_seconds_histogram_.Observe(value);};
-    void CacheUsageGaugeIncrement(double value = 1.0) override { if(startup_) cache_usage_gauge_.Increment(value);};
-    void CacheUsageGaugeDecrement(double value = 1.0) override { if(startup_) cache_usage_gauge_.Decrement(value);};
     void CacheUsageGaugeSet(double value) override { if(startup_) cache_usage_gauge_.Set(value);};
-//    void MetaVisitTotalIncrement(double value = 1.0) override { meta_visit_total_.Increment(value);};
-//    void MetaVisitDurationSecondsHistogramObserve(double value) override { meta_visit_duration_seconds_histogram_.Observe(value);};
-    void MemUsagePercentGaugeSet(double value) override { if(startup_) mem_usage_percent_gauge_.Set(value);};
-    void MemUsagePercentGaugeIncrement(double value = 1.0) override { if(startup_) mem_usage_percent_gauge_.Increment(value);};
-    void MemUsagePercentGaugeDecrement(double value = 1.0) override { if(startup_) mem_usage_percent_gauge_.Decrement(value);};
-    void MemUsageTotalGaugeSet(double value) override { if(startup_) mem_usage_total_gauge_.Set(value);};
-    void MemUsageTotalGaugeIncrement(double value = 1.0) override { if(startup_) mem_usage_total_gauge_.Increment(value);};
-    void MemUsageTotalGaugeDecrement(double value = 1.0) override { if(startup_) mem_usage_total_gauge_.Decrement(value);};
 
     void MetaAccessTotalIncrement(double value = 1) override { if(startup_) meta_access_total_.Increment(value);};
     void MetaAccessDurationSecondsHistogramObserve(double value) override { if(startup_) meta_access_duration_seconds_histogram_.Observe(value);};
 
     void FaissDiskLoadDurationSecondsHistogramObserve(double value) override { if(startup_) faiss_disk_load_duration_seconds_histogram_.Observe(value);};
     void FaissDiskLoadSizeBytesHistogramObserve(double value) override { if(startup_) faiss_disk_load_size_bytes_histogram_.Observe(value);};
-//    void FaissDiskLoadIOSpeedHistogramObserve(double value) { if(startup_) faiss_disk_load_IO_speed_histogram_.Observe(value);};
     void FaissDiskLoadIOSpeedGaugeSet(double value) override { if(startup_) faiss_disk_load_IO_speed_gauge_.Set(value);};
 
     void CacheAccessTotalIncrement(double value = 1) override { if(startup_) cache_access_total_.Increment(value);};

cpp/src/metrics/SystemInfo.cpp

@@ -135,46 +135,6 @@ SystemInfo::CPUPercent() {
     return percent;
 }
 
-//std::unordered_map<int,std::vector<double>>
-//SystemInfo::GetGPUMemPercent(){
-//    // return GPUID: MEM%
-//
-//    //write GPU info to a file
-//    system("nvidia-smi pmon -c 1 > GPUInfo.txt");
-//    int pid = (int)getpid();
-//
-//    //parse line
-//    std::ifstream read_file;
-//    read_file.open("GPUInfo.txt");
-//    std::string line;
-//    while(getline(read_file, line)){
-//        std::vector<std::string> words = split(line);
-//        //                    0      1     2    3   4    5    6      7
-//        //words stand for gpuindex, pid, type, sm, mem, enc, dec, command respectively
-//        if(std::stoi(words[1]) != pid) continue;
-//        int GPUindex = std::stoi(words[0]);
-//        double sm_percent = std::stod(words[3]);
-//        double mem_percent = std::stod(words[4]);
-//
-//    }
-//
-//}
-
-//std::vector<std::string>
-//SystemInfo::split(std::string input) {
-//    std::vector<std::string> words;
-//    input += " ";
-//    int word_start = 0;
-//    for (int i = 0; i < input.size(); ++i) {
-//        if(input[i] != ' ') continue;
-//        if(input[i] == ' ') {
-//            word_start = i + 1;
-//            continue;
-//        }
-//        words.push_back(input.substr(word_start,i-word_start));
-//    }
-//    return words;
-//}
 
 std::vector<unsigned int>
 SystemInfo::GPUPercent() {

cpp/src/metrics/SystemInfo.h

@@ -59,8 +59,6 @@ class SystemInfo {
     double MemoryPercent();
     double CPUPercent();
     std::pair<unsigned long long , unsigned long long > Octets();
-//    std::unordered_map<int,std::vector<double>> GetGPUMemPercent() {};
-//    std::vector<std::string> split(std::string input) {};
     std::vector<unsigned int> GPUPercent();
     std::vector<unsigned long long> GPUMemoryUsed();
 

cpp/unittest/metrics/CMakeLists.txt

@@ -55,8 +55,9 @@ set(count_test_src
         ${wrapper_src}
         ${metrics_src}
         metrics_test.cpp
+        prometheus_test.cpp
         ../db/utils.cpp
-        )
+        metricbase_test.cpp)
 
 
 add_executable(metrics_test ${count_test_src} ${require_files} )

cpp/unittest/metrics/metricbase_test.cpp

+/*******************************************************************************
+ * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+ * Unauthorized copying of this file, via any medium is strictly prohibited.
+ * Proprietary and confidential.
+ ******************************************************************************/
+
+#include "metrics/Metrics.h"
+
+#include <gtest/gtest.h>
+#include <iostream>
+
+using namespace zilliz::milvus;
+
+TEST(MetricbaseTest, Metricbase_Test){
+    server::MetricsBase instance = server::MetricsBase::GetInstance();
+    instance.Init();
+    server::SystemInfo::GetInstance().Init();
+    instance.AddVectorsSuccessTotalIncrement();
+    instance.AddVectorsFailTotalIncrement();
+    instance.AddVectorsDurationHistogramOberve(1.0);
+    instance.RawFileSizeHistogramObserve(1.0);
+    instance.IndexFileSizeHistogramObserve(1.0);
+    instance.BuildIndexDurationSecondsHistogramObserve(1.0);
+    instance.CacheUsageGaugeSet(1.0);
+    instance.MetaAccessTotalIncrement();
+    instance.MetaAccessDurationSecondsHistogramObserve(1.0);
+    instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0);
+    instance.FaissDiskLoadSizeBytesHistogramObserve(1.0);
+    instance.FaissDiskLoadIOSpeedGaugeSet(1.0);
+    instance.CacheAccessTotalIncrement();
+    instance.MemTableMergeDurationSecondsHistogramObserve(1.0);
+    instance.SearchIndexDataDurationSecondsHistogramObserve(1.0);
+    instance.SearchRawDataDurationSecondsHistogramObserve(1.0);
+    instance.IndexFileSizeTotalIncrement();
+    instance.RawFileSizeTotalIncrement();
+    instance.IndexFileSizeGaugeSet(1.0);
+    instance.RawFileSizeGaugeSet(1.0);
+    instance.QueryResponseSummaryObserve(1.0);
+    instance.DiskStoreIOSpeedGaugeSet(1.0);
+    instance.DataFileSizeGaugeSet(1.0);
+    instance.AddVectorsSuccessGaugeSet(1.0);
+    instance.AddVectorsFailGaugeSet(1.0);
+    instance.QueryVectorResponseSummaryObserve(1.0, 1);
+    instance.QueryVectorResponsePerSecondGaugeSet(1.0);
+    instance.CPUUsagePercentSet();
+    instance.RAMUsagePercentSet();
+    instance.QueryResponsePerSecondGaugeSet(1.0);
+    instance.GPUPercentGaugeSet();
+    instance.GPUMemoryUsageGaugeSet();
+    instance.AddVectorsPerSecondGaugeSet(1,1,1);
+    instance.QueryIndexTypePerSecondSet("IVF", 1.0);
+    instance.ConnectionGaugeIncrement();
+    instance.ConnectionGaugeDecrement();
+    instance.KeepingAliveCounterIncrement();
+    instance.OctetsSet();
+}
\ No newline at end of file

cpp/unittest/metrics/metrics_test.cpp

@@ -24,7 +24,6 @@
 using namespace zilliz::milvus;
 
 
-
 TEST_F(DBTest, Metric_Tes) {
 
     server::SystemInfo::GetInstance().Init();

cpp/unittest/metrics/prometheus_test.cpp

+/*******************************************************************************
+ * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+ * Unauthorized copying of this file, via any medium is strictly prohibited.
+ * Proprietary and confidential.
+ ******************************************************************************/
+
+#include "metrics/PrometheusMetrics.h"
+
+#include <gtest/gtest.h>
+#include <iostream>
+
+using namespace zilliz::milvus;
+
+TEST(PrometheusTest, Prometheus_Test){
+    server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance();
+    instance.Init();
+    instance.SetStartup(true);
+    server::SystemInfo::GetInstance().Init();
+    instance.AddVectorsSuccessTotalIncrement();
+    instance.AddVectorsFailTotalIncrement();
+    instance.AddVectorsDurationHistogramOberve(1.0);
+    instance.RawFileSizeHistogramObserve(1.0);
+    instance.IndexFileSizeHistogramObserve(1.0);
+    instance.BuildIndexDurationSecondsHistogramObserve(1.0);
+    instance.CacheUsageGaugeSet(1.0);
+    instance.MetaAccessTotalIncrement();
+    instance.MetaAccessDurationSecondsHistogramObserve(1.0);
+    instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0);
+    instance.FaissDiskLoadSizeBytesHistogramObserve(1.0);
+    instance.FaissDiskLoadIOSpeedGaugeSet(1.0);
+    instance.CacheAccessTotalIncrement();
+    instance.MemTableMergeDurationSecondsHistogramObserve(1.0);
+    instance.SearchIndexDataDurationSecondsHistogramObserve(1.0);
+    instance.SearchRawDataDurationSecondsHistogramObserve(1.0);
+    instance.IndexFileSizeTotalIncrement();
+    instance.RawFileSizeTotalIncrement();
+    instance.IndexFileSizeGaugeSet(1.0);
+    instance.RawFileSizeGaugeSet(1.0);
+    instance.QueryResponseSummaryObserve(1.0);
+    instance.DiskStoreIOSpeedGaugeSet(1.0);
+    instance.DataFileSizeGaugeSet(1.0);
+    instance.AddVectorsSuccessGaugeSet(1.0);
+    instance.AddVectorsFailGaugeSet(1.0);
+    instance.QueryVectorResponseSummaryObserve(1.0, 1);
+    instance.QueryVectorResponsePerSecondGaugeSet(1.0);
+    instance.CPUUsagePercentSet();
+    instance.RAMUsagePercentSet();
+    instance.QueryResponsePerSecondGaugeSet(1.0);
+    instance.GPUPercentGaugeSet();
+    instance.GPUMemoryUsageGaugeSet();
+    instance.AddVectorsPerSecondGaugeSet(1,1,1);
+    instance.QueryIndexTypePerSecondSet("IVF", 1.0);
+    instance.ConnectionGaugeIncrement();
+    instance.ConnectionGaugeDecrement();
+    instance.KeepingAliveCounterIncrement();
+    instance.OctetsSet();
+}
\ No newline at end of file