MS-248 support Milvus profiling

Former-commit-id: 1507ffd4cafbd0005a8edb161169f127f1532c04

cpp/CMakeLists.txt

@@ -141,6 +141,11 @@ endif()
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf)
 
+message("MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}")
+if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
+    ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING)
+endif()
+
 #install
 install(DIRECTORY scripts/
         DESTINATION scripts

cpp/build.sh

@@ -7,8 +7,9 @@ INSTALL_PREFIX=$(pwd)/milvus
 MAKE_CLEAN="OFF"
 BUILD_COVERAGE="OFF"
 DB_PATH="/opt/milvus"
+PROFILING="OFF"
 
-while getopts "p:d:t:uhlrc" arg
+while getopts "p:d:t:uhlrcg" arg
 do
         case $arg in
              t)
@@ -36,6 +37,9 @@ do
              c)
                 BUILD_COVERAGE="ON"
                 ;;
+             g)
+                PROFILING="ON"
+                ;;
              h) # help
                 echo "
 
@@ -47,6 +51,7 @@ parameter:
 -l: build license version(default: OFF)
 -r: remove previous build directory(default: OFF)
 -c: code coverage(default: OFF)
+-g: profiling(default: OFF)
 
 usage:
 ./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c]
@@ -77,6 +82,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
     -DCMAKE_LICENSE_CHECK=${LICENSE_CHECK} \
     -DBUILD_COVERAGE=${BUILD_COVERAGE} \
     -DMILVUS_DB_PATH=${DB_PATH} \
+    -DMILVUS_ENABLE_PROFILING=${PROFILING} \
     $@ ../"
     echo ${CMAKE_CMD}
 

cpp/cmake/DefineOptions.cmake

@@ -111,6 +111,11 @@ define_option(MILVUS_WITH_ZSTD "Build with zstd compression" ${MILVUS_WITH_ZSTD_
 
 define_option(MILVUS_WITH_AWS "Build with AWS SDK" ON)
 
+if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
+    define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON)
+    define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON)
+endif()
+
 #----------------------------------------------------------------------
 if(MSVC)
     set_option_category("MSVC")

cpp/cmake/ThirdPartyPackages.cmake

@@ -37,7 +37,9 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES
         yaml-cpp
         ZLIB
         ZSTD
-        AWS)
+        AWS
+        libunwind
+        gperftools)
 
 message(STATUS "Using ${MILVUS_DEPENDENCY_SOURCE} approach to find dependencies")
 
@@ -89,6 +91,10 @@ macro(build_dependency DEPENDENCY_NAME)
         build_zstd()
     elseif("${DEPENDENCY_NAME}" STREQUAL "AWS")
         build_aws()
+    elseif("${DEPENDENCY_NAME}" STREQUAL "libunwind")
+        build_libunwind()
+    elseif("${DEPENDENCY_NAME}" STREQUAL "gperftools")
+        build_gperftools()
     else()
         message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}")
     endif ()
@@ -347,6 +353,21 @@ if(DEFINED ENV{MILVUS_AWS_URL})
 else()
     set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz")
 endif()
+
+if(DEFINED ENV{MILVUS_LIBUNWIND_URL})
+    set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}")
+else()
+    set(LIBUNWIND_SOURCE_URL
+            "https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz")
+endif()
+
+if(DEFINED ENV{MILVUS_GPERFTOOLS_URL})
+    set(GPERFTOOLS_SOURCE_URL "$ENV{MILVUS_GPERFTOOLS_URL}")
+else()
+    set(GPERFTOOLS_SOURCE_URL
+            "https://github.com/gperftools/gperftools/releases/download/gperftools-${GPERFTOOLS_VERSION}/gperftools-${GPERFTOOLS_VERSION}.tar.gz")
+endif()
+
 # ----------------------------------------------------------------------
 # ARROW
 
@@ -1896,3 +1917,91 @@ if(MILVUS_WITH_AWS)
     include_directories(SYSTEM ${AWS_CPP_SDK_CORE_INCLUDE_DIR})
 
 endif()
+
+# ----------------------------------------------------------------------
+# libunwind
+
+macro(build_libunwind)
+    message(STATUS "Building libunwind-${LIBUNWIND_VERSION} from source")
+    set(LIBUNWIND_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/libunwind_ep-prefix/src/libunwind_ep/install")
+    set(LIBUNWIND_INCLUDE_DIR "${LIBUNWIND_PREFIX}/include")
+    set(LIBUNWIND_SHARED_LIB "${LIBUNWIND_PREFIX}/lib/libunwind${CMAKE_SHARED_LIBRARY_SUFFIX}")
+    set(LIBUNWIND_CONFIGURE_ARGS "--prefix=${LIBUNWIND_PREFIX}")
+
+    externalproject_add(libunwind_ep
+            URL
+            ${LIBUNWIND_SOURCE_URL}
+            ${EP_LOG_OPTIONS}
+            CONFIGURE_COMMAND
+            "./configure"
+            ${LIBUNWIND_CONFIGURE_ARGS}
+            BUILD_COMMAND
+            ${MAKE} ${MAKE_BUILD_ARGS}
+            BUILD_IN_SOURCE
+            1
+            INSTALL_COMMAND
+            ${MAKE} install
+            BUILD_BYPRODUCTS
+            ${LIBUNWIND_SHARED_LIB})
+
+    file(MAKE_DIRECTORY "${LIBUNWIND_INCLUDE_DIR}")
+
+    add_library(libunwind SHARED IMPORTED)
+    set_target_properties(libunwind
+            PROPERTIES IMPORTED_LOCATION "${LIBUNWIND_SHARED_LIB}"
+            INTERFACE_INCLUDE_DIRECTORIES "${LIBUNWIND_INCLUDE_DIR}")
+
+    add_dependencies(libunwind libunwind_ep)
+endmacro()
+
+if(MILVUS_WITH_LIBUNWIND)
+    resolve_dependency(libunwind)
+
+    # TODO: Don't use global includes but rather target_include_directories
+    get_target_property(LIBUNWIND_INCLUDE_DIR libunwind INTERFACE_INCLUDE_DIRECTORIES)
+    include_directories(SYSTEM ${LIBUNWIND_INCLUDE_DIR})
+endif()
+
+# ----------------------------------------------------------------------
+# gperftools
+
+macro(build_gperftools)
+    message(STATUS "Building gperftools-${GPERFTOOLS_VERSION} from source")
+    set(GPERFTOOLS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gperftools_ep-prefix/src/gperftools_ep")
+    set(GPERFTOOLS_INCLUDE_DIR "${GPERFTOOLS_PREFIX}/include")
+    set(GPERFTOOLS_STATIC_LIB "${GPERFTOOLS_PREFIX}/lib/libprofiler${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(GPERFTOOLS_CONFIGURE_ARGS "--prefix=${GPERFTOOLS_PREFIX}")
+
+    externalproject_add(gperftools_ep
+            URL
+            ${GPERFTOOLS_SOURCE_URL}
+            ${EP_LOG_OPTIONS}
+            CONFIGURE_COMMAND
+            "./configure"
+            ${GPERFTOOLS_CONFIGURE_ARGS}
+            BUILD_COMMAND
+            ${MAKE} ${MAKE_BUILD_ARGS}
+            BUILD_IN_SOURCE
+            1
+            INSTALL_COMMAND
+            ${MAKE} install
+            BUILD_BYPRODUCTS
+            ${GPERFTOOLS_STATIC_LIB})
+
+    file(MAKE_DIRECTORY "${GPERFTOOLS_INCLUDE_DIR}")
+
+    add_library(gperftools SHARED IMPORTED)
+    set_target_properties(gperftools
+            PROPERTIES IMPORTED_LOCATION "${GPERFTOOLS_STATIC_LIB}"
+            INTERFACE_INCLUDE_DIRECTORIES "${GPERFTOOLS_INCLUDE_DIR}")
+
+    add_dependencies(gperftools gperftools_ep)
+endmacro()
+
+if(MILVUS_WITH_GPERFTOOLS)
+    resolve_dependency(gperftools)
+
+    # TODO: Don't use global includes but rather target_include_directories
+    get_target_property(GPERFTOOLS_INCLUDE_DIR gperftools INTERFACE_INCLUDE_DIRECTORIES)
+    include_directories(SYSTEM ${GPERFTOOLS_INCLUDE_DIR})
+endif()

cpp/src/CMakeLists.txt

@@ -85,6 +85,7 @@ set(third_party_libs
         zlib
         zstd
         mysqlpp
+        profiler
         ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
         )
 if (MEGASEARCH_WITH_ARROW STREQUAL "ON")

cpp/src/sdk/examples/simple/src/ClientTest.cpp

@@ -148,7 +148,7 @@ namespace {
                 std::cout << "The top 1 result is wrong: " << result_id
                     << " vs. " << search_id << std::endl;
             } else {
-                std::cout << "Check result sucessfully" << std::endl;
+                std::cout << "No." << index-1 << " Check result successfully" << std::endl;
             }
         }
         BLOCK_SPLITER
@@ -234,6 +234,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
     std::vector<std::pair<int64_t, RowRecord>> search_record_array;
     {//add vectors
         for (int i = 0; i < ADD_VECTOR_LOOP; i++) {//add vectors
+            TimeRecorder recorder("Add vector No." + std::to_string(i));
             std::vector<RowRecord> record_array;
             int64_t begin_index = i * BATCH_ROW_COUNT;
             BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array);
@@ -255,6 +256,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
     }
 
     {//wait unit build index finish
+        TimeRecorder recorder("Build index");
         std::cout << "Wait until build all index done" << std::endl;
         Status stat = conn->BuildIndex(TABLE_NAME);
         std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl;

cpp/src/server/RequestTask.cpp

@@ -11,6 +11,10 @@
 #include "DBWrapper.h"
 #include "version.h"
 
+#ifdef MILVUS_ENABLE_PROFILING
+#include "gperftools/profiler.h"
+#endif
+
 namespace zilliz {
 namespace milvus {
 namespace server {
@@ -120,6 +124,18 @@ namespace {
             }
         }
     }
+
+    std::string
+    GetCurrTimeStr() {
+        char tm_buf[20] = {0};
+        time_t tt;
+        time(&tt);
+        tt = tt + 8 * 60 * 60;
+        tm* t = gmtime(&tt);
+        sprintf(tm_buf, "%4d%02d%02d_%02d%02d%02d", (t->tm_year+1900), (t->tm_mon+1), (t->tm_mday),
+                                                    (t->tm_hour), (t->tm_min), (t->tm_sec));
+        return tm_buf;
+    }
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -403,6 +419,14 @@ ServerError AddVectorTask::OnExecute() {
 
         rc.Record("check validation");
 
+#ifdef MILVUS_ENABLE_PROFILING
+        std::string fname = "/tmp/insert_" + std::to_string(this->record_array_.size()) + "_" +
+                            GetCurrTimeStr() + ".profiling";
+        ProfilerStart(fname.c_str());
+#else
+        std::cout << "CYD - MILVUS_ENABLE_PROFILING is OFF!" << std::endl;
+#endif
+
         //step 3: prepare float data
         std::vector<float> vec_f;
         ServerError error_code = SERVER_SUCCESS;
@@ -428,6 +452,10 @@ ServerError AddVectorTask::OnExecute() {
             return SetError(SERVER_ILLEGAL_VECTOR_ID, msg);
         }
 
+#ifdef MILVUS_ENABLE_PROFILING
+        ProfilerStop();
+#endif
+
         rc.Record("do insert");
         rc.Elapse("totally cost");
 
@@ -504,6 +532,13 @@ ServerError SearchVectorTask::OnExecute() {
 
         rc.Record("check validation");
 
+#ifdef MILVUS_ENABLE_PROFILING
+        std::string fname = "/tmp/search_nq_" + std::to_string(this->record_array_.size()) + "_" +
+                            "top_" + std::to_string(this->top_k_) + "_" +
+                            GetCurrTimeStr() + ".profiling";
+        ProfilerStart(fname.c_str());
+#endif
+
         //step 3: prepare float data
         std::vector<float> vec_f;
         ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f, error_code, error_msg);
@@ -556,6 +591,11 @@ ServerError SearchVectorTask::OnExecute() {
 
             result_array_.emplace_back(thrift_topk_result);
         }
+
+#ifdef MILVUS_ENABLE_PROFILING
+        ProfilerStop();
+#endif
+
         rc.Record("construct result");
         rc.Elapse("totally cost");
 

cpp/thirdparty/versions.txt

@@ -19,5 +19,7 @@ YAMLCPP_VERSION=0.6.2
 ZLIB_VERSION=v1.2.11
 ZSTD_VERSION=v1.4.0
 AWS_VERSION=1.7.125
+LIBUNWIND_VERSION=1.3.1
+GPERFTOOLS_VERSION=2.7
 
 # vim: set filetype=sh: