merge Log.h

Former-commit-id: 1204cfbfddee0b863abf10874ef97f0581c36a9a

cpp/src/db/DBFactory.cpp

@@ -6,7 +6,7 @@
 
 #include "DBFactory.h"
 #include "DBImpl.h"
-#include "Exception.h"
+#include "utils/Exception.h"
 #include "meta/MetaFactory.h"
 #include "meta/SqliteMetaImpl.h"
 #include "meta/MySQLMetaImpl.h"

cpp/src/db/DBImpl.cpp

@@ -4,28 +4,27 @@
  * Proprietary and confidential.
  ******************************************************************************/
 #include "DBImpl.h"
-#include "src/db/meta/SqliteMetaImpl.h"
-#include "Log.h"
-#include "Utils.h"
+#include "cache/CpuCacheMgr.h"
+#include "cache/GpuCacheMgr.h"
 #include "engine/EngineFactory.h"
 #include "insert/MemMenagerFactory.h"
+#include "meta/SqliteMetaImpl.h"
 #include "meta/MetaFactory.h"
+#include "meta/MetaConsts.h"
 #include "metrics/Metrics.h"
 #include "scheduler/TaskScheduler.h"
-
 #include "scheduler/context/DeleteContext.h"
+#include "scheduler/SchedInst.h"
 #include "utils/TimeRecorder.h"
-#include "meta/MetaConsts.h"
+#include "utils/Log.h"
+#include "Utils.h"
 
 #include <assert.h>
 #include <chrono>
 #include <thread>
 #include <iostream>
 #include <cstring>
-#include <cache/CpuCacheMgr.h>
 #include <boost/filesystem.hpp>
-#include "scheduler/SchedInst.h"
-#include <src/cache/GpuCacheMgr.h>
 
 namespace zilliz {
 namespace milvus {
@@ -409,7 +408,7 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch
                           const meta::DatesT& dates, QueryResults& results) {
     server::CollectQueryMetrics metrics(nq);
 
-    server::TimeRecorder rc("");
+    TimeRecorder rc("");
 
     //step 1: get files to search
     ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: " << dates.size();
@@ -432,9 +431,9 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch
     double load_cost = context->LoadCost();
     double search_cost = context->SearchCost();
     double reduce_cost = context->ReduceCost();
-    std::string load_info = server::TimeRecorder::GetTimeSpanStr(load_cost);
-    std::string search_info = server::TimeRecorder::GetTimeSpanStr(search_cost);
-    std::string reduce_info = server::TimeRecorder::GetTimeSpanStr(reduce_cost);
+    std::string load_info = TimeRecorder::GetTimeSpanStr(load_cost);
+    std::string search_info = TimeRecorder::GetTimeSpanStr(search_cost);
+    std::string reduce_info = TimeRecorder::GetTimeSpanStr(reduce_cost);
     if(search_cost > 0.0 || reduce_cost > 0.0) {
         double total_cost = load_cost + search_cost + reduce_cost;
         double load_percent = load_cost/total_cost;

cpp/src/db/DBImpl.h

@@ -128,11 +128,11 @@ class DBImpl : public DB {
     MemManagerPtr mem_mgr_;
     std::mutex mem_serialize_mutex_;
 
-    server::ThreadPool compact_thread_pool_;
+    ThreadPool compact_thread_pool_;
     std::list<std::future<void>> compact_thread_results_;
     std::set<std::string> compact_table_ids_;
 
-    server::ThreadPool index_thread_pool_;
+    ThreadPool index_thread_pool_;
     std::list<std::future<void>> index_thread_results_;
 
     std::mutex build_index_mutex_;

cpp/src/db/Log.h

-/*******************************************************************************
- * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
- * Unauthorized copying of this file, via any medium is strictly prohibited.
- * Proprietary and confidential.
- ******************************************************************************/
-#pragma once
-
-#include "utils/easylogging++.h"
-
-namespace zilliz {
-namespace milvus {
-namespace engine {
-
-#define ENGINE_DOMAIN_NAME "[ENGINE] "
-#define ENGINE_ERROR_TEXT "ENGINE Error:"
-
-#define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME
-#define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME
-#define ENGINE_LOG_INFO LOG(INFO) << ENGINE_DOMAIN_NAME
-#define ENGINE_LOG_WARNING LOG(WARNING) << ENGINE_DOMAIN_NAME
-#define ENGINE_LOG_ERROR LOG(ERROR) << ENGINE_DOMAIN_NAME
-#define ENGINE_LOG_FATAL LOG(FATAL) << ENGINE_DOMAIN_NAME
-
-} // namespace sql
-} // namespace zilliz
-} // namespace server

cpp/src/db/Options.cpp

@@ -3,15 +3,14 @@
  * Unauthorized copying of this file, via any medium is strictly prohibited.
  * Proprietary and confidential.
  ******************************************************************************/
+#include "Options.h"
+#include "utils/Exception.h"
+#include "utils/easylogging++.h"
+
 #include <stdlib.h>
 #include <assert.h>
-#include "utils/easylogging++.h"
 #include <boost/algorithm/string.hpp>
 
-#include "Options.h"
-#include "db/meta/SqliteMetaImpl.h"
-#include "Exception.h"
-
 namespace zilliz {
 namespace milvus {
 namespace engine {

cpp/src/db/Utils.cpp

@@ -5,7 +5,7 @@
  ******************************************************************************/
 #include "Utils.h"
 #include "utils/CommonUtil.h"
-#include "Log.h"
+#include "utils/Log.h"
 
 #include <mutex>
 #include <chrono>

cpp/src/db/engine/EngineFactory.cpp

@@ -5,7 +5,7 @@
  ******************************************************************************/
 #include "EngineFactory.h"
 #include "ExecutionEngineImpl.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 
 namespace zilliz {
 namespace milvus {

cpp/src/db/engine/ExecutionEngineImpl.cpp

@@ -3,20 +3,19 @@
  * Unauthorized copying of this file, via any medium is strictly prohibited.
  * Proprietary and confidential.
  ******************************************************************************/
-#include <stdexcept>
-#include "src/cache/GpuCacheMgr.h"
-
-#include "src/metrics/Metrics.h"
-#include "db/Log.h"
+#include "ExecutionEngineImpl.h"
+#include "cache/GpuCacheMgr.h"
+#include "cache/CpuCacheMgr.h"
+#include "metrics/Metrics.h"
+#include "utils/Log.h"
 #include "utils/CommonUtil.h"
+#include "utils/Exception.h"
 
-#include "src/cache/CpuCacheMgr.h"
-#include "ExecutionEngineImpl.h"
 #include "wrapper/knowhere/vec_index.h"
 #include "wrapper/knowhere/vec_impl.h"
 #include "knowhere/common/exception.h"
-#include "db/Exception.h"
 
+#include <stdexcept>
 
 namespace zilliz {
 namespace milvus {

cpp/src/db/insert/MemManagerImpl.cpp

 #include "MemManagerImpl.h"
 #include "VectorSource.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 #include "db/Constants.h"
 
 #include <thread>

cpp/src/db/insert/MemMenagerFactory.cpp

@@ -3,11 +3,10 @@
 // Unauthorized copying of this file, via any medium is strictly prohibited.
 // Proprietary and confidential.
 ////////////////////////////////////////////////////////////////////////////////
-
 #include "MemMenagerFactory.h"
 #include "MemManagerImpl.h"
-#include "db/Log.h"
-#include "db/Exception.h"
+#include "utils/Log.h"
+#include "utils/Exception.h"
 
 #include <stdlib.h>
 #include <time.h>

cpp/src/db/insert/MemTable.cpp

 #include "MemTable.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 
 
 namespace zilliz {

cpp/src/db/insert/MemTableFile.cpp

 #include "MemTableFile.h"
 #include "db/Constants.h"
-#include "db/Log.h"
 #include "db/engine/EngineFactory.h"
 #include "metrics/Metrics.h"
+#include "utils/Log.h"
 
 #include <cmath>
 

cpp/src/db/insert/VectorSource.cpp

 #include "VectorSource.h"
 #include "db/engine/ExecutionEngine.h"
 #include "db/engine/EngineFactory.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 #include "metrics/Metrics.h"
 
 

cpp/src/db/meta/MetaFactory.cpp

@@ -3,12 +3,11 @@
 // Unauthorized copying of this file, via any medium is strictly prohibited.
 // Proprietary and confidential.
 ////////////////////////////////////////////////////////////////////////////////
-
 #include "MetaFactory.h"
 #include "SqliteMetaImpl.h"
 #include "MySQLMetaImpl.h"
-#include "db/Log.h"
-#include "db/Exception.h"
+#include "utils/Log.h"
+#include "utils/Exception.h"
 
 #include <stdlib.h>
 #include <time.h>

cpp/src/db/meta/MySQLConnectionPool.h

@@ -4,7 +4,7 @@
 #include <unistd.h>
 #include <atomic>
 
-#include "db/Log.h"
+#include "utils/Log.h"
 
 namespace zilliz {
 namespace milvus {

cpp/src/db/meta/MySQLMetaImpl.cpp

@@ -6,7 +6,7 @@
 #include "MySQLMetaImpl.h"
 #include "db/IDGenerator.h"
 #include "db/Utils.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 #include "MetaConsts.h"
 #include "metrics/Metrics.h"
 

cpp/src/db/meta/SqliteMetaImpl.cpp

@@ -6,7 +6,7 @@
 #include "SqliteMetaImpl.h"
 #include "db/IDGenerator.h"
 #include "db/Utils.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 #include "MetaConsts.h"
 #include "metrics/Metrics.h"
 

cpp/src/db/scheduler/TaskDispatchStrategy.cpp

@@ -10,7 +10,7 @@
 #include "task/DeleteTask.h"
 #include "cache/CpuCacheMgr.h"
 #include "utils/Error.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 
 namespace zilliz {
 namespace milvus {

cpp/src/db/scheduler/task/IndexLoadTask.cpp

@@ -6,8 +6,8 @@
 
 #include "IndexLoadTask.h"
 #include "SearchTask.h"
-#include "db/Log.h"
 #include "db/engine/EngineFactory.h"
+#include "utils/Log.h"
 #include "utils/TimeRecorder.h"
 #include "metrics/Metrics.h"
 

cpp/src/db/scheduler/task/SearchTask.cpp

@@ -5,7 +5,7 @@
  ******************************************************************************/
 #include "SearchTask.h"
 #include "metrics/Metrics.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 #include "utils/TimeRecorder.h"
 
 #include <thread>

cpp/src/scheduler/ResourceMgr.cpp

@@ -5,7 +5,7 @@
  * Proprietary and confidential.
  ******************************************************************************/
 #include "ResourceMgr.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 
 
 namespace zilliz {

cpp/src/scheduler/task/SearchTask.cpp

@@ -3,12 +3,11 @@
  * Unauthorized copying of this file, via any medium is strictly prohibited.
  * Proprietary and confidential.
  ******************************************************************************/
-
-#include "src/metrics/Metrics.h"
-#include "src/utils/TimeRecorder.h"
-#include "src/db/engine/EngineFactory.h"
-#include "src/db/Log.h"
 #include "SearchTask.h"
+#include "metrics/Metrics.h"
+#include "db/engine/EngineFactory.h"
+#include "utils/TimeRecorder.h"
+#include "utils/Log.h"
 
 #include <thread>
 
@@ -97,7 +96,7 @@ XSearchTask::XSearchTask(TableFileSchemaPtr file)
 
 void
 XSearchTask::Load(LoadType type, uint8_t device_id) {
-    server::TimeRecorder rc("");
+    TimeRecorder rc("");
     Status stat = Status::OK();
     std::string error_msg;
     std::string type_str;
@@ -166,7 +165,7 @@ XSearchTask::Execute() {
     ENGINE_LOG_DEBUG << "Searching in file id:" << index_id_ << " with "
                      << search_contexts_.size() << " tasks";
 
-    server::TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_));
+    TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_));
 
     server::CollectDurationMetrics metrics(index_type_);
 

cpp/src/db/Exception.h → cpp/src/utils/Exception.h

@@ -10,7 +10,6 @@
 
 namespace zilliz {
 namespace milvus {
-namespace engine {
 
 class Exception : public std::exception {
 public:
@@ -49,6 +48,5 @@ public:
     OutOfRangeException(const std::string& message) : Exception(message) {};
 };
 
-} // namespace engine
 } // namespace milvus
 } // namespace zilliz

cpp/src/utils/Log.h

@@ -5,13 +5,12 @@
  ******************************************************************************/
 #pragma once
 
-#include "Error.h"
 #include "easylogging++.h"
 
 namespace zilliz {
 namespace milvus {
-namespace server {
 
+/////////////////////////////////////////////////////////////////////////////////////////////////
 #define SERVER_DOMAIN_NAME "[SERVER] "
 #define SERVER_ERROR_TEXT "SERVER Error:"
 
@@ -22,20 +21,27 @@ namespace server {
 #define SERVER_LOG_ERROR LOG(ERROR) << SERVER_DOMAIN_NAME
 #define SERVER_LOG_FATAL LOG(FATAL) << SERVER_DOMAIN_NAME
 
-#define SERVER_ERROR(error)                             \
-    ({                                                  \
-        SERVER_LOG_ERROR << SERVER_ERROR_TEXT << error; \
-        (error);                                        \
-    })
-
-#define SERVER_CHECK(func)                                  \
-    {                                                       \
-        zilliz::milvus::server::ServerError error = func;  \
-        if (error != zilliz::milvus::server::SERVER_SUCCESS) { \
-            return SERVER_ERROR(error);                     \
-        }                                                   \
-    }                                                       \
-
-} // namespace sql
+/////////////////////////////////////////////////////////////////////////////////////////////////
+#define ENGINE_DOMAIN_NAME "[ENGINE] "
+#define ENGINE_ERROR_TEXT "ENGINE Error:"
+
+#define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME
+#define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME
+#define ENGINE_LOG_INFO LOG(INFO) << ENGINE_DOMAIN_NAME
+#define ENGINE_LOG_WARNING LOG(WARNING) << ENGINE_DOMAIN_NAME
+#define ENGINE_LOG_ERROR LOG(ERROR) << ENGINE_DOMAIN_NAME
+#define ENGINE_LOG_FATAL LOG(FATAL) << ENGINE_DOMAIN_NAME
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+#define WRAPPER_DOMAIN_NAME "[WRAPPER] "
+#define WRAPPER_ERROR_TEXT "WRAPPER Error:"
+
+#define WRAPPER_LOG_TRACE LOG(TRACE) << WRAPPER_DOMAIN_NAME
+#define WRAPPER_LOG_DEBUG LOG(DEBUG) << WRAPPER_DOMAIN_NAME
+#define WRAPPER_LOG_INFO LOG(INFO) << WRAPPER_DOMAIN_NAME
+#define WRAPPER_LOG_WARNING LOG(WARNING) << WRAPPER_DOMAIN_NAME
+#define WRAPPER_LOG_ERROR LOG(ERROR) << WRAPPER_DOMAIN_NAME
+#define WRAPPER_LOG_FATAL LOG(FATAL) << WRAPPER_DOMAIN_NAME
+
+} // namespace milvus
 } // namespace zilliz
-} // namespace server

cpp/src/utils/ThreadPool.h

@@ -20,7 +20,6 @@
 
 namespace zilliz {
 namespace milvus {
-namespace server {
 
 class ThreadPool {
 public:
@@ -114,5 +113,4 @@ inline ThreadPool::~ThreadPool() {
 
 }
 }
-}
 

cpp/src/utils/TimeRecorder.cpp

@@ -9,7 +9,6 @@
 
 namespace zilliz {
 namespace milvus {
-namespace server {
 
 TimeRecorder::TimeRecorder(const std::string &header,
                            int64_t log_level) :
@@ -91,4 +90,3 @@ TimeRecorder::ElapseFromBegin(const std::string &msg) {
 
 }
 }
-}

cpp/src/utils/TimeRecorder.h

@@ -11,7 +11,6 @@
 
 namespace zilliz {
 namespace milvus {
-namespace server {
 
 class TimeRecorder {
     using stdclock = std::chrono::high_resolution_clock;
@@ -40,4 +39,3 @@ private:
 
 }
 }
-}

cpp/src/wrapper/knowhere/vec_impl.cpp

@@ -4,7 +4,7 @@
 // Proprietary and confidential.
 ////////////////////////////////////////////////////////////////////////////////
 
-#include <src/utils/Log.h>
+#include "utils/Log.h"
 #include "knowhere/index/vector_index/idmap.h"
 #include "knowhere/index/vector_index/gpu_ivf.h"
 #include "knowhere/common/exception.h"
@@ -12,7 +12,6 @@
 
 #include "vec_impl.h"
 #include "data_transfer.h"
-#include "wrapper_log.h"
 
 
 namespace zilliz {

cpp/src/wrapper/knowhere/vec_index.cpp

@@ -12,7 +12,7 @@
 
 #include "vec_index.h"
 #include "vec_impl.h"
-#include "wrapper_log.h"
+#include "utils/Log.h"
 
 #include <cuda.h>
 

cpp/src/wrapper/knowhere/wrapper_log.h

-////////////////////////////////////////////////////////////////////////////////
-// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
-// Unauthorized copying of this file, via any medium is strictly prohibited.
-// Proprietary and confidential.
-////////////////////////////////////////////////////////////////////////////////
-
-#pragma once
-
-#include "utils/easylogging++.h"
-
-namespace zilliz {
-namespace milvus {
-namespace engine {
-
-#define WRAPPER_DOMAIN_NAME "[WRAPPER] "
-#define WRAPPER_ERROR_TEXT "WRAPPER Error:"
-
-#define WRAPPER_LOG_TRACE LOG(TRACE) << WRAPPER_DOMAIN_NAME
-#define WRAPPER_LOG_DEBUG LOG(DEBUG) << WRAPPER_DOMAIN_NAME
-#define WRAPPER_LOG_INFO LOG(INFO) << WRAPPER_DOMAIN_NAME
-#define WRAPPER_LOG_WARNING LOG(WARNING) << WRAPPER_DOMAIN_NAME
-#define WRAPPER_LOG_ERROR LOG(ERROR) << WRAPPER_DOMAIN_NAME
-#define WRAPPER_LOG_FATAL LOG(FATAL) << WRAPPER_DOMAIN_NAME
-
-}
-}
-}
-

cpp/unittest/db/misc_test.cpp

@@ -3,18 +3,17 @@
 // Unauthorized copying of this file, via any medium is strictly prohibited.
 // Proprietary and confidential.
 ////////////////////////////////////////////////////////////////////////////////
-#include <gtest/gtest.h>
-#include <thread>
-#include "utils/easylogging++.h"
-#include <boost/filesystem.hpp>
-
-#include "db/Exception.h"
 #include "db/Status.h"
 #include "db/Options.h"
 #include "db/meta/SqliteMetaImpl.h"
 #include "db/engine/EngineFactory.h"
 #include "db/Utils.h"
+#include "utils/Exception.h"
+#include "utils/easylogging++.h"
 
+#include <gtest/gtest.h>
+#include <thread>
+#include <boost/filesystem.hpp>
 #include <vector>
 
 using namespace zilliz::milvus;
@@ -27,11 +26,11 @@ namespace {
 }
 
 TEST(DBMiscTest, EXCEPTION_TEST) {
-    engine::Exception ex1("");
+    Exception ex1("");
     std::string what = ex1.what();
     ASSERT_FALSE(what.empty());
 
-    engine::OutOfRangeException ex2;
+    OutOfRangeException ex2;
     what = ex2.what();
     ASSERT_FALSE(what.empty());
 }

cpp/unittest/db/search_test.cpp

@@ -226,7 +226,7 @@ TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) {
     engine::SearchContext::ResultSet src_result;
 
     auto DoCluster = [&](int64_t nq, int64_t topk) {
-        server::TimeRecorder rc("DoCluster");
+        TimeRecorder rc("DoCluster");
         src_result.clear();
         BuildResult(nq, topk, ascending, target_ids, target_distence);
         rc.RecordSection("build id/dietance map");
@@ -268,7 +268,7 @@ TEST(DBSearchTest, PARALLEL_TOPK_TEST) {
         src_result.clear();
         insufficient_result.clear();
 
-        server::TimeRecorder rc("DoCluster");
+        TimeRecorder rc("DoCluster");
 
         BuildResult(nq, topk, ascending, target_ids, target_distence);
         auto status = engine::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result);

cpp/unittest/server/util_test.cpp

@@ -166,7 +166,7 @@ TEST(UtilTest, TIMERECORDER_TEST) {
         if(log_level == 5) {
             continue; //skip fatal
         }
-        server::TimeRecorder rc("time", log_level);
+        TimeRecorder rc("time", log_level);
         rc.RecordSection("end");
     }
 }