add metrics

Former-commit-id: 3e811d555cf69eb2c8f515ada44c8df43e460de0

cpp/src/metrics/MetricBase.h

+/*******************************************************************************
+ * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+ * Unauthorized copying of this file, via any medium is strictly prohibited.
+ * Proprietary and confidential.
+ ******************************************************************************/
+
+#pragma once
+
+#include "utils/Error.h"
+#include "server/ServerConfig.h"
+
+namespace zilliz {
+namespace vecwise {
+namespace server {
+class MetricsBase{
+ public:
+    static MetricsBase&
+    GetInstance(){
+        static MetricsBase instance;
+        return instance;
+    }
+
+    virtual ServerError Init() {};
+    virtual void AddGroupSuccessTotalIncrement(double value = 1) {};
+    virtual void AddGroupFailTotalIncrement(double value = 1) {};
+    virtual void HasGroupSuccessTotalIncrement(double value = 1) {};
+    virtual void HasGroupFailTotalIncrement(double value = 1) {};
+    virtual void GetGroupSuccessTotalIncrement(double value = 1) {};
+    virtual void GetGroupFailTotalIncrement(double value = 1) {};
+    virtual void GetGroupFilesSuccessTotalIncrement(double value = 1) {};
+    virtual void GetGroupFilesFailTotalIncrement(double value = 1) {};
+    virtual void AddVectorsSuccessTotalIncrement(double value = 1) {};
+    virtual void AddVectorsFailTotalIncrement(double value = 1) {};
+    virtual void AddVectorsDurationHistogramOberve(double value) {};
+    virtual void SearchSuccessTotalIncrement(double value = 1) {};
+    virtual void SearchFailTotalIncrement(double value = 1) {};
+    virtual void SearchDurationHistogramObserve(double value) {};
+    virtual void RawFileSizeHistogramObserve(double value) {};
+    virtual void IndexFileSizeHistogramObserve(double value) {};
+    virtual void BuildIndexDurationSecondsHistogramObserve(double value) {};
+    virtual void AllBuildIndexDurationSecondsHistogramObserve(double value) {};
+    virtual void CacheUsageGaugeIncrement(double value = 1) {};
+    virtual void CacheUsageGaugeDecrement(double value = 1) {};
+    virtual void CacheUsageGaugeSet(double value) {};
+    virtual void MetaVisitTotalIncrement(double value = 1) {};
+    virtual void MetaVisitDurationSecondsHistogramObserve(double value) {};
+    virtual void MemUsagePercentGaugeSet(double value) {};
+    virtual void MemUsagePercentGaugeIncrement(double value = 1) {};
+    virtual void MemUsagePercentGaugeDecrement(double value = 1) {};
+    virtual void MemUsageTotalGaugeSet(double value) {};
+    virtual void MemUsageTotalGaugeIncrement(double value = 1) {};
+    virtual void MemUsageTotalGaugeDecrement(double value = 1) {};
+    virtual void MetaAccessTotalIncrement(double value = 1) {};
+    virtual void MetaAccessDurationSecondsHistogramObserve(double value) {};
+    virtual void FaissDiskLoadDurationSecondsHistogramObserve(double value) {};
+    virtual void FaissDiskLoadSizeBytesHistogramObserve(double value) {};
+    virtual void FaissDiskLoadIOSpeedHistogramObserve(double value) {};
+    virtual void CacheAccessTotalIncrement(double value = 1) {};
+    virtual void MemTableMergeDurationSecondsHistogramObserve(double value) {};
+    virtual void SearchIndexDataDurationSecondsHistogramObserve(double value) {};
+    virtual void SearchRawDataDurationSecondsHistogramObserve(double value) {};
+    virtual void IndexFileSizeTotalIncrement(double value = 1) {};
+    virtual void RawFileSizeTotalIncrement(double value = 1) {};
+    virtual void IndexFileSizeGaugeSet(double value) {};
+    virtual void RawFileSizeGaugeSet(double value) {};
+
+};
+
+
+
+
+
+
+}
+}
+}
\ No newline at end of file

cpp/src/metrics/PrometheusMetrics.cpp

+/*******************************************************************************
+ * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+ * Unauthorized copying of this file, via any medium is strictly prohibited.
+ * Proprietary and confidential.
+ ******************************************************************************/
+
+#include "PrometheusMetrics.h"
+
+namespace zilliz {
+namespace vecwise {
+namespace server {
+
+ServerError
+PrometheusMetrics::Init() {
+    ConfigNode& configNode = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC);
+    startup_ = configNode.GetValue(CONFIG_METRIC_IS_STARTUP) == "true" ? true:false;
+    // Following should be read from config file.
+    const std::string bind_address = "8080";
+    const std::string uri = std::string("/metrics");
+    const std::size_t num_threads = 2;
+
+    // Init Exposer
+    exposer_ptr_ = std::make_shared<prometheus::Exposer>(bind_address, uri, num_threads);
+
+    // Exposer Registry
+    exposer_ptr_->RegisterCollectable(registry_);
+
+    return SERVER_SUCCESS;
+}
+
+}
+}
+}

cpp/unittest/metrics/CMakeLists.txt

+
+
+#set(CURL_LIBRARY "-lcurl")
+#find_package(CURL REQUIRED)
+#INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR})
+#
+#INCLUDE_DIRECTORIES(~/development/lib/usr/local/include)
+#LINK_DIRECTORIES(~/development/lib/usr/local/lib)
+
+
+include_directories(../../src)
+
+
+
+aux_source_directory(../../src/db db_srcs)
+aux_source_directory(../../src/config config_files)
+aux_source_directory(../../src/cache cache_srcs)
+aux_source_directory(../../src/wrapper wrapper_src)
+aux_source_directory(../../src/metrics metrics_files)
+
+include_directories(/usr/include)
+include_directories(../../third_party/build/include)
+link_directories(../../third_party/build/lib)
+include_directories(/usr/local/cuda/include)
+link_directories("/usr/local/cuda/lib64")
+include_directories(../db/utils.h)
+
+
+set(require_files
+        ../../src/metrics/PrometheusMetrics.cpp
+        ../../src/server/ServerConfig.cpp
+        ../../src/utils/CommonUtil.cpp
+        ../../src/utils/TimeRecorder.cpp
+        ../../src/metrics/PrometheusMetrics.cpp
+        )
+
+set(count_test_src
+        #        ${unittest_srcs}
+        #        ${config_files}
+        #        ${require_files}
+        metrics_test.cpp
+        ${unittest_srcs}
+        ${config_files}
+        ${cache_srcs}
+        ${db_srcs}
+        ${wrapper_src}
+        ${require_files}
+        metrics_test.cpp
+        ../db/utils.cpp
+        ../../src/metrics/Metrics.h ../../src/metrics/Metrics.cpp)
+
+
+add_executable(metrics_test ${count_test_src} ${require_files} )
+
+target_link_libraries(metrics_test
+        ${unittest_libs}
+        faiss
+        cudart
+        cublas
+        sqlite3
+        boost_system
+        boost_filesystem
+        lz4
+        metrics
+        gtest
+        prometheus-cpp-pull
+        prometheus-cpp-push
+        prometheus-cpp-core
+        pthread
+        z
+
+        )
\ No newline at end of file

cpp/unittest/metrics/metrics_test.cpp

+/*******************************************************************************
+ * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+ * Unauthorized copying of this file, via any medium is strictly prohibited.
+ * Proprietary and confidential.
+ ******************************************************************************/
+#include <chrono>
+#include <chrono>
+#include <map>
+#include <memory>
+#include <string>
+#include <thread>
+#include <gtest/gtest.h>
+#include "prometheus/registry.h"
+#include "prometheus/exposer.h"
+#include <cache/CpuCacheMgr.h>
+
+#include "metrics/MetricBase.h"
+#include "../db/utils.h"
+#include "db/DB.h"
+#include "db/DBMetaImpl.h"
+#include "db/Factories.h"
+
+
+using namespace zilliz::vecwise;
+
+TEST_F(DBTest, Metric_Test) {
+
+    using namespace zilliz::vecwise;
+//    server::Metrics::GetInstance().Init();
+    server::PrometheusMetrics::GetInstance().Init();
+//    server::PrometheusMetrics::GetInstance().exposer_ptr()->RegisterCollectable(server::PrometheusMetrics::GetInstance().registry_ptr());
+//    server::Metrics::GetInstance().exposer_ptr()->RegisterCollectable(server::Metrics::GetInstance().registry_ptr());
+
+    static const std::string group_name = "test_group";
+    static const int group_dim = 256;
+
+    engine::meta::GroupSchema group_info;
+    group_info.dimension = group_dim;
+    group_info.group_id = group_name;
+    engine::Status stat = db_->add_group(group_info);
+
+    engine::meta::GroupSchema group_info_get;
+    group_info_get.group_id = group_name;
+
+//    int iter = 600000;
+//    for (int i = 0; i < iter; ++i) {
+//        db_->get_group(group_info);
+//        bool b = i % 2;
+//        db_->has_group(std::to_string(i),b);
+//        db_->add_group(group_info);
+//
+//        std::this_thread::sleep_for(std::chrono::microseconds(1));
+//    }
+    stat = db_->get_group(group_info_get);
+    ASSERT_STATS(stat);
+    ASSERT_EQ(group_info_get.dimension, group_dim);
+
+    engine::IDNumbers vector_ids;
+    engine::IDNumbers target_ids;
+
+    int d = 256;
+    int nb = 50;
+    float *xb = new float[d * nb];
+    for(int i = 0; i < nb; i++) {
+        for(int j = 0; j < d; j++) xb[d * i + j] = drand48();
+        xb[d * i] += i / 2000.;
+    }
+
+    int qb = 5;
+    float *qxb = new float[d * qb];
+    for(int i = 0; i < qb; i++) {
+        for(int j = 0; j < d; j++) qxb[d * i + j] = drand48();
+        qxb[d * i] += i / 2000.;
+    }
+
+    int loop =100000;
+
+    for (auto i=0; i<loop; ++i) {
+        if (i==40) {
+            db_->add_vectors(group_name, qb, qxb, target_ids);
+            ASSERT_EQ(target_ids.size(), qb);
+        } else {
+            db_->add_vectors(group_name, nb, xb, vector_ids);
+        }
+        std::this_thread::sleep_for(std::chrono::microseconds(5));
+    }
+
+//    search.join();
+
+    delete [] xb;
+    delete [] qxb;
+};
+
+
+TEST_F(DBTest, Metric_Tes) {
+
+
+//    server::Metrics::GetInstance().Init();
+//    server::Metrics::GetInstance().exposer_ptr()->RegisterCollectable(server::Metrics::GetInstance().registry_ptr());
+    server::PrometheusMetrics::GetInstance().Init();
+//    server::PrometheusMetrics::GetInstance().exposer_ptr()->RegisterCollectable(server::PrometheusMetrics::GetInstance().registry_ptr());
+    zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->SetCapacity(1*1024*1024*1024);
+    std::cout<<zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->CacheCapacity()<<std::endl;
+    static const std::string group_name = "test_group";
+    static const int group_dim = 256;
+
+    engine::meta::GroupSchema group_info;
+    group_info.dimension = group_dim;
+    group_info.group_id = group_name;
+    engine::Status stat = db_->add_group(group_info);
+
+    engine::meta::GroupSchema group_info_get;
+    group_info_get.group_id = group_name;
+    stat = db_->get_group(group_info_get);
+    ASSERT_STATS(stat);
+    ASSERT_EQ(group_info_get.dimension, group_dim);
+
+    engine::IDNumbers vector_ids;
+    engine::IDNumbers target_ids;
+
+    int d = 256;
+    int nb = 50;
+    float *xb = new float[d * nb];
+    for(int i = 0; i < nb; i++) {
+        for(int j = 0; j < d; j++) xb[d * i + j] = drand48();
+        xb[d * i] += i / 2000.;
+    }
+
+    int qb = 5;
+    float *qxb = new float[d * qb];
+    for(int i = 0; i < qb; i++) {
+        for(int j = 0; j < d; j++) qxb[d * i + j] = drand48();
+        qxb[d * i] += i / 2000.;
+    }
+
+    std::thread search([&]() {
+        engine::QueryResults results;
+        int k = 10;
+        std::this_thread::sleep_for(std::chrono::seconds(2));
+
+        INIT_TIMER;
+        std::stringstream ss;
+        long count = 0;
+        long prev_count = -1;
+
+        for (auto j=0; j<10; ++j) {
+            ss.str("");
+            db_->count(group_name, count);
+            prev_count = count;
+
+            START_TIMER;
+            stat = db_->search(group_name, k, qb, qxb, results);
+            ss << "Search " << j << " With Size " << (float)(count*group_dim*sizeof(float))/(1024*1024) << " M";
+//            STOP_TIMER(ss.str());
+
+            ASSERT_STATS(stat);
+            for (auto k=0; k<qb; ++k) {
+                ASSERT_EQ(results[k][0], target_ids[k]);
+                ss.str("");
+                ss << "Result [" << k << "]:";
+                for (auto result : results[k]) {
+                    ss << result << " ";
+                }
+                /* LOG(DEBUG) << ss.str(); */
+            }
+            ASSERT_TRUE(count >= prev_count);
+            std::this_thread::sleep_for(std::chrono::seconds(1));
+        }
+    });
+
+    int loop = 100000;
+
+    for (auto i=0; i<loop; ++i) {
+        if (i==40) {
+            db_->add_vectors(group_name, qb, qxb, target_ids);
+            ASSERT_EQ(target_ids.size(), qb);
+        } else {
+            db_->add_vectors(group_name, nb, xb, vector_ids);
+        }
+        std::this_thread::sleep_for(std::chrono::microseconds(1));
+    }
+
+    search.join();
+
+    delete [] xb;
+    delete [] qxb;
+};
+
+