Merge branch 'branch-0.5.0-improve_log' into 'branch-0.5.0'

MS-552 Add and change the easylogging library

See merge request megasearch/milvus!567

Former-commit-id: e8f604b5a6ff74d012ee712bb1be22b812def760

cpp/CHANGELOG.md

@@ -7,6 +7,7 @@ Please mark all change in change log and use the ticket from JIRA.
 ## Bug
 
 ## Improvement
+- MS-552 - Add and change the easylogging library
 
 ## New Feature
 

cpp/cmake/ThirdPartyPackages.cmake

@@ -19,7 +19,6 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES
         ARROW
         BOOST
         BZip2
-        Easylogging++
         FAISS
         GTest
         Knowhere
@@ -56,8 +55,6 @@ macro(build_dependency DEPENDENCY_NAME)
         build_arrow()
     elseif("${DEPENDENCY_NAME}" STREQUAL "BZip2")
         build_bzip2()
-    elseif("${DEPENDENCY_NAME}" STREQUAL "Easylogging++")
-        build_easyloggingpp()
     elseif("${DEPENDENCY_NAME}" STREQUAL "FAISS")
         build_faiss()
     elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest")
@@ -295,13 +292,6 @@ else()
 endif()
 set(BZIP2_MD5 "00b516f4704d4a7cb50a1d97e6e8e15b")
 
-if(DEFINED ENV{MILVUS_EASYLOGGINGPP_URL})
-    set(EASYLOGGINGPP_SOURCE_URL "$ENV{MILVUS_EASYLOGGINGPP_URL}")
-else()
-    set(EASYLOGGINGPP_SOURCE_URL "https://github.com/zuhd-org/easyloggingpp/archive/${EASYLOGGINGPP_VERSION}.tar.gz")
-endif()
-set(EASYLOGGINGPP_MD5 "b78cd319db4be9b639927657b8aa7732")
-
 if(DEFINED ENV{MILVUS_FAISS_URL})
     set(FAISS_SOURCE_URL "$ENV{MILVUS_FAISS_URL}")
 else()
@@ -814,84 +804,6 @@ if(MILVUS_WITH_KNOWHERE)
     include_directories(SYSTEM "${KNOWHERE_INCLUDE_DIR}/SPTAG/AnnService")
 endif()
 
-# ----------------------------------------------------------------------
-# Easylogging++
-
-macro(build_easyloggingpp)
-    message(STATUS "Building Easylogging++-${EASYLOGGINGPP_VERSION} from source")
-    set(EASYLOGGINGPP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix/src/easyloggingpp_ep")
-    set(EASYLOGGINGPP_INCLUDE_DIR "${EASYLOGGINGPP_PREFIX}/include")
-    set(EASYLOGGINGPP_STATIC_LIB
-            "${EASYLOGGINGPP_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}easyloggingpp${CMAKE_STATIC_LIBRARY_SUFFIX}")
-
-    set(EASYLOGGINGPP_CMAKE_ARGS
-            ${EP_COMMON_CMAKE_ARGS}
-            "-DCMAKE_INSTALL_PREFIX=${EASYLOGGINGPP_PREFIX}"
-            -DCMAKE_INSTALL_LIBDIR=lib
-            -Dtest=OFF
-            -Dbuild_static_lib=ON)
-
-    if(USE_JFROG_CACHE STREQUAL "ON")
-        set(EASYLOGGINGPP_CACHE_PACKAGE_NAME "easyloggingpp_${EASYLOGGINGPP_MD5}.tar.gz")
-        set(EASYLOGGINGPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}")
-        set(EASYLOGGINGPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}")
-
-        execute_process(COMMAND wget -q --method HEAD ${EASYLOGGINGPP_CACHE_URL} RESULT_VARIABLE return_code)
-        message(STATUS "Check the remote cache file ${EASYLOGGINGPP_CACHE_URL}. return code = ${return_code}")
-        if (NOT return_code EQUAL 0)
-            externalproject_add(easyloggingpp_ep
-                URL
-                ${EASYLOGGINGPP_SOURCE_URL}
-                ${EP_LOG_OPTIONS}
-                CMAKE_ARGS
-                ${EASYLOGGINGPP_CMAKE_ARGS}
-                BUILD_COMMAND
-                ${MAKE}
-                ${MAKE_BUILD_ARGS}
-                BUILD_BYPRODUCTS
-                ${EASYLOGGINGPP_STATIC_LIB})
-
-            ExternalProject_Create_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${EASYLOGGINGPP_CACHE_URL})
-        else()
-            file(DOWNLOAD ${EASYLOGGINGPP_CACHE_URL} ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} STATUS status)
-            list(GET status 0 status_code)
-            message(STATUS "DOWNLOADING FROM ${EASYLOGGINGPP_CACHE_URL} TO ${EASYLOGGINGPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
-            if (status_code EQUAL 0)
-                ExternalProject_Use_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
-            endif()
-        endif()
-    else()
-        externalproject_add(easyloggingpp_ep
-                URL
-                ${EASYLOGGINGPP_SOURCE_URL}
-                ${EP_LOG_OPTIONS}
-                CMAKE_ARGS
-                ${EASYLOGGINGPP_CMAKE_ARGS}
-                BUILD_COMMAND
-                ${MAKE}
-                ${MAKE_BUILD_ARGS}
-                BUILD_BYPRODUCTS
-                ${EASYLOGGINGPP_STATIC_LIB})
-    endif()
-
-    file(MAKE_DIRECTORY "${EASYLOGGINGPP_INCLUDE_DIR}")
-    add_library(easyloggingpp STATIC IMPORTED)
-    set_target_properties(
-            easyloggingpp
-            PROPERTIES IMPORTED_LOCATION "${EASYLOGGINGPP_STATIC_LIB}"
-            INTERFACE_INCLUDE_DIRECTORIES "${EASYLOGGINGPP_INCLUDE_DIR}")
-
-    add_dependencies(easyloggingpp easyloggingpp_ep)
-endmacro()
-
-if(MILVUS_WITH_EASYLOGGINGPP)
-    resolve_dependency(Easylogging++)
-
-    get_target_property(EASYLOGGINGPP_INCLUDE_DIR easyloggingpp INTERFACE_INCLUDE_DIRECTORIES)
-    link_directories(SYSTEM "${EASYLOGGINGPP_PREFIX}/lib")
-    include_directories(SYSTEM "${EASYLOGGINGPP_INCLUDE_DIR}")
-endif()
-
 # ----------------------------------------------------------------------
 # OpenBLAS
 

cpp/src/CMakeLists.txt

@@ -73,6 +73,7 @@ set(db_files
         ${db_scheduler_files}
         ${metrics_files}
         ${knowhere_files}
+        ${utils_files}
         )
 
 set(s3_client_files
@@ -94,7 +95,6 @@ set(client_grpc_lib
 
 set(third_party_libs
         knowhere
-        easyloggingpp
         sqlite
         ${client_grpc_lib}
         yaml-cpp
@@ -145,7 +145,6 @@ target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs})
 add_library(metrics STATIC ${metrics_files})
 
 set(metrics_lib
-        easyloggingpp
         yaml-cpp
         prometheus-cpp-push
         prometheus-cpp-pull

cpp/src/db/Log.h

@@ -5,7 +5,7 @@
  ******************************************************************************/
 #pragma once
 
-#include <easylogging++.h>
+#include "utils/easylogging++.h"
 
 namespace zilliz {
 namespace milvus {

cpp/src/db/Options.cpp

@@ -5,7 +5,7 @@
  ******************************************************************************/
 #include <stdlib.h>
 #include <assert.h>
-#include <easylogging++.h>
+#include "utils/easylogging++.h"
 #include <boost/algorithm/string.hpp>
 
 #include "Options.h"

cpp/src/main.cpp

@@ -11,7 +11,7 @@
 #include <cstring>
 #include <string>
 #include <signal.h>
-#include <easylogging++.h>
+#include "utils/easylogging++.h"
 #include "metrics/Metrics.h"
 
 #include "utils/SignalUtil.h"

cpp/src/utils/Log.h

@@ -6,7 +6,7 @@
 #pragma once
 
 #include "Error.h"
-#include <easylogging++.h>
+#include "easylogging++.h"
 
 namespace zilliz {
 namespace milvus {

cpp/src/utils/LogUtil.cpp

@@ -5,8 +5,8 @@
 ////////////////////////////////////////////////////////////////////////////////
 #include "LogUtil.h"
 #include "server/ServerConfig.h"
+#include "easylogging++.h"
 
-#include <easylogging++.h>
 #include <ctype.h>
 
 #include <string>
@@ -27,7 +27,7 @@ static int fatal_idx = 0;
 }
 
 // TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename
-void RolloutHandler(const char *filename, std::size_t size) {
+void RolloutHandler(const char *filename, std::size_t size, el::Level level) {
     char *dirc = strdup(filename);
     char *basec = strdup(filename);
     char *dir = dirname(dirc);
@@ -48,22 +48,22 @@ void RolloutHandler(const char *filename, std::size_t size) {
     int ret;
     std::string m(std::string(dir) + "/" + s);
     s = m;
-    if ((position = s.find("global")) != std::string::npos) {
+    if (level == el::Level::Global) {
         s.append("." + std::to_string(++global_idx));
         ret = rename(m.c_str(), s.c_str());
-    } else if ((position = s.find("debug")) != std::string::npos) {
+    } else if (level == el::Level::Debug) {
         s.append("." + std::to_string(++debug_idx));
         ret = rename(m.c_str(), s.c_str());
-    } else if ((position = s.find("warning")) != std::string::npos) {
+    } else if (level == el::Level::Warning) {
         s.append("." + std::to_string(++warning_idx));
         ret = rename(m.c_str(), s.c_str());
-    } else if ((position = s.find("trace")) != std::string::npos) {
+    } else if (level == el::Level::Trace) {
         s.append("." + std::to_string(++trace_idx));
         ret = rename(m.c_str(), s.c_str());
-    } else if ((position = s.find("error")) != std::string::npos) {
+    } else if (level == el::Level::Error) {
         s.append("." + std::to_string(++error_idx));
         ret = rename(m.c_str(), s.c_str());
-    } else if ((position = s.find("fatal")) != std::string::npos) {
+    } else if (level == el::Level::Fatal) {
         s.append("." + std::to_string(++fatal_idx));
         ret = rename(m.c_str(), s.c_str());
     } else {
@@ -113,6 +113,7 @@ int32_t InitLog(const std::string &log_config_file) {
 
     el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck);
     el::Helpers::installPreRollOutCallback(RolloutHandler);
+    el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog);
     return 0;
 }
 

cpp/src/utils/LogUtil.h

@@ -7,6 +7,7 @@
 
 #include <string>
 #include <sstream>
+#include "easylogging++.h"
 
 namespace zilliz {
 namespace milvus {
@@ -16,7 +17,7 @@ inline std::string GetFileName(std::string filename) {
     int pos = filename.find_last_of('/');
     return filename.substr(pos + 1);
 }
-void RolloutHandler(const char *filename, std::size_t size);
+void RolloutHandler(const char *filename, std::size_t size, el::Level level);
 
 #define SHOW_LOCATION
 #ifdef SHOW_LOCATION

cpp/src/wrapper/knowhere/wrapper_log.h

@@ -6,7 +6,7 @@
 
 #pragma once
 
-#include <easylogging++.h>
+#include "utils/easylogging++.h"
 
 namespace zilliz {
 namespace milvus {

cpp/unittest/CMakeLists.txt

@@ -25,7 +25,6 @@ set(unittest_libs
         gmock
         gtest_main
         gmock_main
-        easyloggingpp
         pthread
         metrics
         gfortran

cpp/unittest/db/CMakeLists.txt

@@ -19,7 +19,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs)
 aux_source_directory(./ test_srcs)
 
 set(util_files
-        ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
+        ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
+        ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
+        ${MILVUS_ENGINE_SRC}/utils/easylogging++.h)
 
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)

cpp/unittest/db/db_tests.cpp

@@ -12,7 +12,7 @@
 #include "utils/CommonUtil.h"
 
 #include <gtest/gtest.h>
-#include <easylogging++.h>
+#include "utils/easylogging++.h"
 
 #include <boost/filesystem.hpp>
 

cpp/unittest/db/meta_tests.cpp

@@ -5,7 +5,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 #include <gtest/gtest.h>
 #include <thread>
-#include <easylogging++.h>
+#include "utils/easylogging++.h"
 #include <stdlib.h>
 #include <time.h>
 

cpp/unittest/db/misc_test.cpp

@@ -5,7 +5,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 #include <gtest/gtest.h>
 #include <thread>
-#include <easylogging++.h>
+#include "utils/easylogging++.h"
 #include <boost/filesystem.hpp>
 
 #include "db/Exception.h"

cpp/unittest/db/mysql_db_test.cpp

@@ -9,7 +9,7 @@
 #include "db/meta/MetaConsts.h"
 
 #include <gtest/gtest.h>
-#include <easylogging++.h>
+#include "utils/easylogging++.h"
 #include <boost/filesystem.hpp>
 
 #include <thread>

cpp/unittest/db/mysql_meta_test.cpp

@@ -5,7 +5,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 #include <gtest/gtest.h>
 #include <thread>
-#include <easylogging++.h>
+#include "utils/easylogging++.h"
 #include <stdlib.h>
 #include <time.h>
 

cpp/unittest/db/scheduler_test.cpp

@@ -5,7 +5,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 #include <gtest/gtest.h>
 #include <thread>
-#include <easylogging++.h>
+#include "utils/easylogging++.h"
 #include <boost/filesystem.hpp>
 
 #include "db/scheduler/TaskScheduler.h"

cpp/unittest/knowhere/CMakeLists.txt

@@ -7,7 +7,9 @@ set(knowhere_src
         ${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_index.cpp)
 
 set(helper
-        utils.cpp)
+        utils.cpp
+        ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
+        ${MILVUS_ENGINE_SRC}/utils/easylogging++.h)
 
 set(knowhere_libs
         knowhere

cpp/unittest/knowhere/knowhere_test.cpp

@@ -5,7 +5,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include <gtest/gtest.h>
-#include <easylogging++.h>
+#include "utils/easylogging++.h"
 
 #include <wrapper/knowhere/vec_index.h>
 #include "knowhere/index/vector_index/gpu_ivf.h"

cpp/unittest/main.cpp

@@ -5,7 +5,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 #include <gtest/gtest.h>
 #include <gmock/gmock.h>
-#include <easylogging++.h>
+#include "utils/easylogging++.h"
 
 #include "server/ServerConfig.h"
 #include "utils/CommonUtil.h"

cpp/unittest/metrics/CMakeLists.txt

@@ -18,7 +18,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/src/metrics metrics_src)
 aux_source_directory(./ test_srcs)
 
 set(util_files
-        ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
+        ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
+        ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
+        ${MILVUS_ENGINE_SRC}/utils/easylogging++.h)
 
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)

cpp/unittest/scheduler/CMakeLists.txt

@@ -21,7 +21,10 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs)
 aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs)
 aux_source_directory(./ test_srcs)
 
-set(util_files ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
+set(util_files
+        ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
+        ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
+        ${MILVUS_ENGINE_SRC}/utils/easylogging++.h)
 
 set(db_scheduler_srcs
         ${scheduler_files}

cpp/unittest/server/CMakeLists.txt

@@ -39,6 +39,11 @@ set(db_scheduler_srcs
         ${scheduler_task_files}
         )
 
+set(util_files
+        ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
+        ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
+        ${MILVUS_ENGINE_SRC}/utils/easylogging++.h)
+
 set(db_src
         ${config_files}
         ${cache_srcs}

cpp/unittest/server/util_test.cpp

@@ -5,7 +5,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 #include <gtest/gtest.h>
 #include <thread>
-#include <easylogging++.h>
+#include "utils/easylogging++.h"
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <boost/filesystem.hpp>
@@ -290,6 +290,7 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST){
     std::string dir1 = "/tmp/milvus_test";
     std::string dir2 = "/tmp/milvus_test/log_test";
     std::string filename[6] = {"log_global.log", "log_debug.log", "log_warning.log", "log_trace.log", "log_error.log", "log_fatal.log"};
+    el::Level list[6] = {el::Level::Global, el::Level::Debug, el::Level::Warning, el::Level::Trace, el::Level::Error, el::Level::Fatal};
 
     mkdir(dir1.c_str(), S_IRWXU);
     mkdir(dir2.c_str(), S_IRWXU);
@@ -300,7 +301,7 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST){
         file.open(tmp.c_str());
         file << "zilliz" << std::endl;
 
-        server::RolloutHandler(tmp.c_str(), 0);
+        server::RolloutHandler(tmp.c_str(), 0, list[i]);
 
         tmp.append(".1");
         std::ifstream file2;