Merge pull request #332 from youny626/0.6.0

Add option  to enable / disable prometheus

CHANGELOG.md

@@ -18,6 +18,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#260 - C++ SDK README
 - \#314 - add Find FAISS in CMake
 - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue
+- \#322 - Add option to enable / disable prometheus 
 
 ## Task
 

core/CMakeLists.txt

@@ -33,7 +33,7 @@ message(STATUS "Build time = ${BUILD_TIME}")
 
 MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
     execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'"
-                    OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
+            OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
 ENDMACRO(GET_GIT_BRANCH_NAME)
 
 GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
@@ -117,17 +117,17 @@ include(DefineOptions)
 include(BuildUtils)
 include(ThirdPartyPackages)
 
-if(MILVUS_USE_CCACHE)
-  find_program(CCACHE_FOUND ccache)
-  if(CCACHE_FOUND)
-    message(STATUS "Using ccache: ${CCACHE_FOUND}")
-    set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
-    set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
-    # let ccache preserve C++ comments, because some of them may be
-    # meaningful to the compiler
-    set(ENV{CCACHE_COMMENTS} "1")
-  endif(CCACHE_FOUND)
-endif()
+if (MILVUS_USE_CCACHE)
+    find_program(CCACHE_FOUND ccache)
+    if (CCACHE_FOUND)
+        message(STATUS "Using ccache: ${CCACHE_FOUND}")
+        set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
+        set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
+        # let ccache preserve C++ comments, because some of them may be
+        # meaningful to the compiler
+        set(ENV{CCACHE_COMMENTS} "1")
+    endif (CCACHE_FOUND)
+endif ()
 
 set(MILVUS_CPU_VERSION false)
 if (MILVUS_GPU_VERSION)
@@ -142,6 +142,10 @@ else ()
     add_compile_definitions("MILVUS_CPU_VERSION")
 endif ()
 
+if (MILVUS_WITH_PROMETHEUS)
+    add_compile_definitions("MILVUS_WITH_PROMETHEUS")
+endif ()
+
 if (CMAKE_BUILD_TYPE STREQUAL "Release")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
     if (MILVUS_GPU_VERSION)
@@ -176,9 +180,9 @@ endif ()
 
 if (MILVUS_GPU_VERSION)
     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_gpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
-else()
+else ()
     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_cpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
-endif()
+endif ()
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf)
 

core/build.sh

@@ -14,10 +14,11 @@ CUSTOMIZATION="OFF" # default use ori faiss
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
 GPU_VERSION="OFF" #defaults to CPU version
 WITH_MKL="OFF"
-FAISS_ROOT=""
+FAISS_ROOT="" #FAISS root path
 FAISS_SOURCE="BUNDLED"
+WITH_PROMETHEUS="ON"
 
-while getopts "p:d:t:f:ulrcgjhxzm" arg
+while getopts "p:d:t:f:ulrcgjhxzme" arg
 do
         case $arg in
              p)
@@ -63,7 +64,10 @@ do
                 ;;
              m)
                 WITH_MKL="ON"
-                ;;   
+                ;;
+             e)
+               WITH_PROMETHEUS="OFF"
+                ;;
              h) # help
                 echo "
 
@@ -80,10 +84,11 @@ parameter:
 -j: use jfrog cache build directory(default: OFF)
 -g: build GPU version(default: OFF)
 -m: build with MKL(default: OFF)
+-e: build without prometheus
 -h: help
 
 usage:
-./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h]
+./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-e] [-h]
                 "
                 exit 0
                 ;;
@@ -118,6 +123,7 @@ CMAKE_CMD="cmake \
 -DCUSTOMIZATION=${CUSTOMIZATION} \
 -DMILVUS_GPU_VERSION=${GPU_VERSION} \
 -DFAISS_WITH_MKL=${WITH_MKL} \
+-DMILVUS_WITH_PROMETHEUS=${WITH_PROMETHEUS} \
 ../"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD}

core/src/CMakeLists.txt

@@ -37,6 +37,7 @@ endforeach ()
 aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files)
+aux_source_directory(${MILVUS_ENGINE_SRC}/metrics/prometheus metrics_prometheus_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
@@ -91,6 +92,11 @@ set(engine_files
         ${wrapper_files}
         )
 
+if (MILVUS_WITH_PROMETHEUS)
+    set(engine_files ${engine_files}
+            ${metrics_prometheus_files})
+endif ()
+
 set(client_grpc_lib
         grpcpp_channelz
         grpc++
@@ -115,7 +121,6 @@ set(third_party_libs
         sqlite
         ${client_grpc_lib}
         yaml-cpp
-        ${prometheus_lib}
         mysqlpp
         zlib
         ${boost_lib}
@@ -138,13 +143,19 @@ if (MILVUS_GPU_VERSION)
             )
 endif ()
 
-if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
+if (MILVUS_ENABLE_PROFILING)
     set(third_party_libs ${third_party_libs}
             gperftools
             libunwind
             )
 endif ()
 
+if (MILVUS_WITH_PROMETHEUS)
+    set(third_party_libs ${third_party_libs}
+            ${prometheus_lib}
+            )
+endif ()
+
 set(engine_libs
         pthread
         libgomp.a
@@ -166,13 +177,22 @@ target_link_libraries(milvus_engine
         ${engine_libs}
         )
 
-add_library(metrics STATIC ${metrics_files})
+if (MILVUS_WITH_PROMETHEUS)
+    add_library(metrics STATIC ${metrics_files} ${metrics_prometheus_files})
+else ()
+    add_library(metrics STATIC ${metrics_files})
+endif ()
 
 set(metrics_lib
         yaml-cpp
-        ${prometheus_lib}
         )
 
+if (MILVUS_WITH_PROMETHEUS)
+    set(metrics_lib ${metrics_lib}
+            ${prometheus_lib}
+            )
+endif ()
+
 target_link_libraries(metrics ${metrics_lib})
 
 set(server_libs

core/src/metrics/Metrics.cpp

@@ -16,8 +16,10 @@
 // under the License.
 
 #include "metrics/Metrics.h"
-#include "PrometheusMetrics.h"
 #include "server/Config.h"
+#ifdef MILVUS_WITH_PROMETHEUS
+#include "metrics/prometheus/PrometheusMetrics.h"
+#endif
 
 #include <string>
 
@@ -37,11 +39,15 @@ Metrics::CreateMetricsCollector() {
 
     config.GetMetricConfigCollector(collector_type_str);
 
+#ifdef MILVUS_WITH_PROMETHEUS
     if (collector_type_str == "prometheus") {
         return PrometheusMetrics::GetInstance();
     } else {
         return MetricsBase::GetInstance();
     }
+#else
+    return MetricsBase::GetInstance();
+#endif
 }
 
 }  // namespace server

core/src/metrics/PrometheusMetrics.cpp → core/src/metrics/prometheus/PrometheusMetrics.cpp

@@ -15,9 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "metrics/PrometheusMetrics.h"
-#include "SystemInfo.h"
+#include "metrics/prometheus/PrometheusMetrics.h"
 #include "cache/GpuCacheMgr.h"
+#include "metrics/SystemInfo.h"
 #include "server/Config.h"
 #include "utils/Log.h"
 

core/src/metrics/PrometheusMetrics.h → core/src/metrics/prometheus/PrometheusMetrics.h

@@ -24,7 +24,7 @@
 #include <string>
 #include <vector>
 
-#include "MetricBase.h"
+#include "metrics/MetricBase.h"
 #include "utils/Error.h"
 
 #define METRICS_NOW_TIME std::chrono::system_clock::now()

core/unittest/CMakeLists.txt

@@ -110,12 +110,18 @@ set(unittest_libs
         pthread
         metrics
         gfortran
-        prometheus-cpp-pull
-        prometheus-cpp-push
-        prometheus-cpp-core
-        dl
-        z
         )
+if (MILVUS_WITH_PROMETHEUS)
+    set(unittest_libs ${unittest_libs}
+            prometheus-cpp-push
+            prometheus-cpp-pull
+            prometheus-cpp-core
+            )
+endif ()
+set(unittest_libs ${unittest_libs}
+    dl
+    z
+    )
 
 if (MILVUS_GPU_VERSION)
     include_directories("${CUDA_INCLUDE_DIRS}")
@@ -135,4 +141,4 @@ add_subdirectory(db)
 add_subdirectory(wrapper)
 add_subdirectory(metrics)
 add_subdirectory(scheduler)
-add_subdirectory(server)
+add_subdirectory(server)
\ No newline at end of file

core/unittest/metrics/CMakeLists.txt

@@ -18,10 +18,15 @@
 #-------------------------------------------------------------------------------
 
 set(test_files
-        ${CMAKE_CURRENT_SOURCE_DIR}/test_metricbase.cpp
-        ${CMAKE_CURRENT_SOURCE_DIR}/test_metrics.cpp
-        ${CMAKE_CURRENT_SOURCE_DIR}/test_prometheus.cpp
-        ${CMAKE_CURRENT_SOURCE_DIR}/utils.cpp)
+        test_metricbase.cpp
+        test_metrics.cpp
+        utils.cpp
+        )
+
+if (MILVUS_WITH_PROMETHEUS)
+    set(test_files ${test_files}
+            test_prometheus.cpp)
+endif ()
 
 add_executable(test_metrics
         ${common_files}

core/unittest/metrics/test_prometheus.cpp

@@ -15,8 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "metrics/PrometheusMetrics.h"
 #include "server/Config.h"
+#include "metrics/prometheus/PrometheusMetrics.h"
 
 #include <gtest/gtest.h>
 #include <iostream>