Merge pull request #332 from youny626/0.6.0

Add option to enable / disable prometheus

Merge pull request #332 from youny626/0.6.0
Add option to enable / disable prometheus
2e62a052 · Jin Hai · GitHub · 651f5e91 · 8ae3a839 · 2e62a052
10 changed file
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#260 - C++ SDK README
 - \#314 - add Find FAISS in CMake
 - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue
+- \#322 - Add option to enable / disable prometheus 

 ## Task


--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -33,7 +33,7 @@ message(STATUS "Build time = ${BUILD_TIME}")

 MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
    execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'"
-                    OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
+            OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
 ENDMACRO(GET_GIT_BRANCH_NAME)

 GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
@@ -117,17 +117,17 @@ include(DefineOptions)
 include(BuildUtils)
 include(ThirdPartyPackages)

-if(MILVUS_USE_CCACHE)
-  find_program(CCACHE_FOUND ccache)
-  if(CCACHE_FOUND)
-    message(STATUS "Using ccache: ${CCACHE_FOUND}")
-    set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
-    set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
-    # let ccache preserve C++ comments, because some of them may be
-    # meaningful to the compiler
-    set(ENV{CCACHE_COMMENTS} "1")
-  endif(CCACHE_FOUND)
-endif()
+if (MILVUS_USE_CCACHE)
+    find_program(CCACHE_FOUND ccache)
+    if (CCACHE_FOUND)
+        message(STATUS "Using ccache: ${CCACHE_FOUND}")
+        set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
+        set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
+        # let ccache preserve C++ comments, because some of them may be
+        # meaningful to the compiler
+        set(ENV{CCACHE_COMMENTS} "1")
+    endif (CCACHE_FOUND)
+endif ()

 set(MILVUS_CPU_VERSION false)
 if (MILVUS_GPU_VERSION)
@@ -142,6 +142,10 @@ else ()
    add_compile_definitions("MILVUS_CPU_VERSION")
 endif ()

+if (MILVUS_WITH_PROMETHEUS)
+    add_compile_definitions("MILVUS_WITH_PROMETHEUS")
+endif ()
+
 if (CMAKE_BUILD_TYPE STREQUAL "Release")
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
    if (MILVUS_GPU_VERSION)
@@ -176,9 +180,9 @@ endif ()

 if (MILVUS_GPU_VERSION)
    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_gpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
-else()
+else ()
    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_cpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
-endif()
+endif ()

 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf)


--- a/core/build.sh
+++ b/core/build.sh
@@ -14,10 +14,11 @@ CUSTOMIZATION="OFF" # default use ori faiss
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
 GPU_VERSION="OFF" #defaults to CPU version
 WITH_MKL="OFF"
-FAISS_ROOT=""
+FAISS_ROOT="" #FAISS root path
 FAISS_SOURCE="BUNDLED"
+WITH_PROMETHEUS="ON"

-while getopts "p:d:t:f:ulrcgjhxzm" arg
+while getopts "p:d:t:f:ulrcgjhxzme" arg
 do
        case $arg in
             p)
@@ -63,7 +64,10 @@ do
                ;;
             m)
                WITH_MKL="ON"
-                ;;   
+                ;;
+             e)
+               WITH_PROMETHEUS="OFF"
+                ;;
             h) # help
                echo "

@@ -80,10 +84,11 @@ parameter:
 -j: use jfrog cache build directory(default: OFF)
 -g: build GPU version(default: OFF)
 -m: build with MKL(default: OFF)
+-e: build without prometheus
 -h: help

 usage:
-./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h]
+./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-e] [-h]
                "
                exit 0
                ;;
@@ -118,6 +123,7 @@ CMAKE_CMD="cmake \
 -DCUSTOMIZATION=${CUSTOMIZATION} \
 -DMILVUS_GPU_VERSION=${GPU_VERSION} \
 -DFAISS_WITH_MKL=${WITH_MKL} \
+-DMILVUS_WITH_PROMETHEUS=${WITH_PROMETHEUS} \
 ../"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD}

--- a/core/src/CMakeLists.txt
+++ b/core/src/CMakeLists.txt
@@ -37,6 +37,7 @@ endforeach ()
 aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files)
+aux_source_directory(${MILVUS_ENGINE_SRC}/metrics/prometheus metrics_prometheus_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
@@ -91,6 +92,11 @@ set(engine_files
        ${wrapper_files}
        )

+if (MILVUS_WITH_PROMETHEUS)
+    set(engine_files ${engine_files}
+            ${metrics_prometheus_files})
+endif ()
+
 set(client_grpc_lib
        grpcpp_channelz
        grpc++
@@ -115,7 +121,6 @@ set(third_party_libs
        sqlite
        ${client_grpc_lib}
        yaml-cpp
-        ${prometheus_lib}
        mysqlpp
        zlib
        ${boost_lib}
@@ -138,13 +143,19 @@ if (MILVUS_GPU_VERSION)
            )
 endif ()

-if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
+if (MILVUS_ENABLE_PROFILING)
    set(third_party_libs ${third_party_libs}
            gperftools
            libunwind
            )
 endif ()

+if (MILVUS_WITH_PROMETHEUS)
+    set(third_party_libs ${third_party_libs}
+            ${prometheus_lib}
+            )
+endif ()
+
 set(engine_libs
        pthread
        libgomp.a
@@ -166,13 +177,22 @@ target_link_libraries(milvus_engine
        ${engine_libs}
        )

-add_library(metrics STATIC ${metrics_files})
+if (MILVUS_WITH_PROMETHEUS)
+    add_library(metrics STATIC ${metrics_files} ${metrics_prometheus_files})
+else ()
+    add_library(metrics STATIC ${metrics_files})
+endif ()

 set(metrics_lib
        yaml-cpp
-        ${prometheus_lib}
        )

+if (MILVUS_WITH_PROMETHEUS)
+    set(metrics_lib ${metrics_lib}
+            ${prometheus_lib}
+            )
+endif ()
+
 target_link_libraries(metrics ${metrics_lib})

 set(server_libs

--- a/core/src/metrics/Metrics.cpp
+++ b/core/src/metrics/Metrics.cpp
@@ -16,8 +16,10 @@
 // under the License.

 #include "metrics/Metrics.h"
-#include "PrometheusMetrics.h"
 #include "server/Config.h"
+#ifdef MILVUS_WITH_PROMETHEUS
+#include "metrics/prometheus/PrometheusMetrics.h"
+#endif

 #include <string>

@@ -37,11 +39,15 @@ Metrics::CreateMetricsCollector() {

    config.GetMetricConfigCollector(collector_type_str);

+#ifdef MILVUS_WITH_PROMETHEUS
    if (collector_type_str == "prometheus") {
        return PrometheusMetrics::GetInstance();
    } else {
        return MetricsBase::GetInstance();
    }
+#else
+    return MetricsBase::GetInstance();
+#endif
 }

 }  // namespace server

--- a/core/src/metrics/PrometheusMetrics.cpp
+++ b/core/src/metrics/PrometheusMetrics.cpp
@@ -15,9 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.

-#include "metrics/PrometheusMetrics.h"
-#include "SystemInfo.h"
+#include "metrics/prometheus/PrometheusMetrics.h"
 #include "cache/GpuCacheMgr.h"
+#include "metrics/SystemInfo.h"
 #include "server/Config.h"
 #include "utils/Log.h"


--- a/core/src/metrics/PrometheusMetrics.h
+++ b/core/src/metrics/PrometheusMetrics.h
@@ -24,7 +24,7 @@
 #include <string>
 #include <vector>

-#include "MetricBase.h"
+#include "metrics/MetricBase.h"
 #include "utils/Error.h"

 #define METRICS_NOW_TIME std::chrono::system_clock::now()

--- a/core/unittest/CMakeLists.txt
+++ b/core/unittest/CMakeLists.txt
@@ -110,12 +110,18 @@ set(unittest_libs
        pthread
        metrics
        gfortran
-        prometheus-cpp-pull
-        prometheus-cpp-push
-        prometheus-cpp-core
-        dl
-        z
        )
+if (MILVUS_WITH_PROMETHEUS)
+    set(unittest_libs ${unittest_libs}
+            prometheus-cpp-push
+            prometheus-cpp-pull
+            prometheus-cpp-core
+            )
+endif ()
+set(unittest_libs ${unittest_libs}
+    dl
+    z
+    )

 if (MILVUS_GPU_VERSION)
    include_directories("${CUDA_INCLUDE_DIRS}")
@@ -135,4 +141,4 @@ add_subdirectory(db)
 add_subdirectory(wrapper)
 add_subdirectory(metrics)
 add_subdirectory(scheduler)
-add_subdirectory(server)
\ No newline at end of file
+add_subdirectory(server)
--- a/core/unittest/metrics/CMakeLists.txt
+++ b/core/unittest/metrics/CMakeLists.txt
@@ -18,10 +18,15 @@
 #-------------------------------------------------------------------------------

 set(test_files
-        ${CMAKE_CURRENT_SOURCE_DIR}/test_metricbase.cpp
-        ${CMAKE_CURRENT_SOURCE_DIR}/test_metrics.cpp
-        ${CMAKE_CURRENT_SOURCE_DIR}/test_prometheus.cpp
-        ${CMAKE_CURRENT_SOURCE_DIR}/utils.cpp)
+        test_metricbase.cpp
+        test_metrics.cpp
+        utils.cpp
+        )
+
+if (MILVUS_WITH_PROMETHEUS)
+    set(test_files ${test_files}
+            test_prometheus.cpp)
+endif ()

 add_executable(test_metrics
        ${common_files}

--- a/core/unittest/metrics/test_prometheus.cpp
+++ b/core/unittest/metrics/test_prometheus.cpp
@@ -15,8 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.

-#include "metrics/PrometheusMetrics.h"
 #include "server/Config.h"
+#include "metrics/prometheus/PrometheusMetrics.h"

 #include <gtest/gtest.h>
 #include <iostream>