MS-273 Add some logs and reformat code

Former-commit-id: 4c906c96c1342ccd8aa674c8b4175916cd53a8d0

cpp/src/cache/Cache.cpp

@@ -89,7 +89,7 @@ void Cache::erase(const std::string& key) {
     const DataObjPtr& data_ptr = obj_ptr->data_;
     usage_ -= data_ptr->size();
 
-    SERVER_LOG_DEBUG << "Erase " << key << " from cache";
+    SERVER_LOG_DEBUG << "Erase " << key << " size: " << data_ptr->size();
 
     lru_.erase(key);
 }

cpp/src/cache/CacheMgr.cpp

@@ -4,6 +4,7 @@
 // Proprietary and confidential.
 ////////////////////////////////////////////////////////////////////////////////
 
+#include "utils/Log.h"
 #include "CacheMgr.h"
 #include "metrics/Metrics.h"
 
@@ -20,6 +21,7 @@ CacheMgr::~CacheMgr() {
 
 uint64_t CacheMgr::ItemCount() const {
     if(cache_ == nullptr) {
+        SERVER_LOG_ERROR << "Cache doesn't exist";
         return 0;
     }
 
@@ -28,6 +30,7 @@ uint64_t CacheMgr::ItemCount() const {
 
 bool CacheMgr::ItemExists(const std::string& key) {
     if(cache_ == nullptr) {
+        SERVER_LOG_ERROR << "Cache doesn't exist";
         return false;
     }
 
@@ -36,6 +39,7 @@ bool CacheMgr::ItemExists(const std::string& key) {
 
 DataObjPtr CacheMgr::GetItem(const std::string& key) {
     if(cache_ == nullptr) {
+        SERVER_LOG_ERROR << "Cache doesn't exist";
         return nullptr;
     }
     server::Metrics::GetInstance().CacheAccessTotalIncrement();
@@ -45,6 +49,7 @@ DataObjPtr CacheMgr::GetItem(const std::string& key) {
 engine::Index_ptr CacheMgr::GetIndex(const std::string& key) {
     DataObjPtr obj = GetItem(key);
     if(obj != nullptr) {
+        SERVER_LOG_ERROR << "Can't get object from key: " << key;
         return obj->data();
     }
 
@@ -53,6 +58,7 @@ engine::Index_ptr CacheMgr::GetIndex(const std::string& key) {
 
 void CacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) {
     if(cache_ == nullptr) {
+        SERVER_LOG_ERROR << "Cache doesn't exist";
         return;
     }
 
@@ -62,6 +68,7 @@ void CacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) {
 
 void CacheMgr::InsertItem(const std::string& key, const engine::Index_ptr& index) {
     if(cache_ == nullptr) {
+        SERVER_LOG_ERROR << "Cache doesn't exist";
         return;
     }
 
@@ -72,6 +79,7 @@ void CacheMgr::InsertItem(const std::string& key, const engine::Index_ptr& index
 
 void CacheMgr::EraseItem(const std::string& key) {
     if(cache_ == nullptr) {
+        SERVER_LOG_ERROR << "Cache doesn't exist";
         return;
     }
 
@@ -81,6 +89,7 @@ void CacheMgr::EraseItem(const std::string& key) {
 
 void CacheMgr::PrintInfo() {
     if(cache_ == nullptr) {
+        SERVER_LOG_ERROR << "Cache doesn't exist";
         return;
     }
 
@@ -89,6 +98,7 @@ void CacheMgr::PrintInfo() {
 
 void CacheMgr::ClearCache() {
     if(cache_ == nullptr) {
+        SERVER_LOG_ERROR << "Cache doesn't exist";
         return;
     }
 
@@ -97,6 +107,7 @@ void CacheMgr::ClearCache() {
 
 int64_t CacheMgr::CacheUsage() const {
     if(cache_ == nullptr) {
+        SERVER_LOG_ERROR << "Cache doesn't exist";
         return 0;
     }
 
@@ -105,6 +116,7 @@ int64_t CacheMgr::CacheUsage() const {
 
 int64_t CacheMgr::CacheCapacity() const {
     if(cache_ == nullptr) {
+        SERVER_LOG_ERROR << "Cache doesn't exist";
         return 0;
     }
 
@@ -113,6 +125,7 @@ int64_t CacheMgr::CacheCapacity() const {
 
 void CacheMgr::SetCapacity(int64_t capacity) {
     if(cache_ == nullptr) {
+        SERVER_LOG_ERROR << "Cache doesn't exist";
         return;
     }
     cache_->set_capacity(capacity);

cpp/src/cache/CpuCacheMgr.cpp

@@ -12,10 +12,14 @@ namespace zilliz {
 namespace milvus {
 namespace cache {
 
+namespace {
+    constexpr int64_t unit = 1024 * 1024 * 1024;
+}
+
 CpuCacheMgr::CpuCacheMgr() {
     server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
     int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16);
-    cap *= 1024*1024*1024;
+    cap *= unit;
     cache_ = std::make_shared<Cache>(cap, 1UL<<32);
 
     double free_percent = config.GetDoubleValue(server::CACHE_FREE_PERCENT, 0.85);

cpp/src/cache/GpuCacheMgr.cpp

@@ -11,10 +11,14 @@ namespace zilliz {
 namespace milvus {
 namespace cache {
 
+namespace {
+    constexpr int64_t unit = 1024 * 1024 * 1024;
+}
+
 GpuCacheMgr::GpuCacheMgr() {
     server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
     int64_t cap = config.GetInt64Value(server::CONFIG_GPU_CACHE_CAPACITY, 1);
-    cap *= 1024*1024*1024;
+    cap *= unit;
     cache_ = std::make_shared<Cache>(cap, 1UL<<32);
 }
 

cpp/src/config/ConfigNode.cpp

@@ -94,7 +94,7 @@ double
 ConfigNode::GetDoubleValue(const std::string &param_key, double default_val) const {
     std::string val = GetValue(param_key);
     if (!val.empty()) {
-        return std::strtold(val.c_str(), nullptr);
+        return std::strtod(val.c_str(), nullptr);
     } else {
         return default_val;
     }

cpp/src/db/Constants.h

@@ -9,14 +9,14 @@ namespace zilliz {
 namespace milvus {
 namespace engine {
 
-const size_t K = 1024UL;
-const size_t M = K * K;
-const size_t G = K * M;
-const size_t T = K * G;
+constexpr size_t K = 1024UL;
+constexpr size_t M = K * K;
+constexpr size_t G = K * M;
+constexpr size_t T = K * G;
 
-const size_t MAX_TABLE_FILE_MEM = 128 * M;
+constexpr size_t MAX_TABLE_FILE_MEM = 128 * M;
 
-const int VECTOR_TYPE_SIZE = sizeof(float);
+constexpr int VECTOR_TYPE_SIZE = sizeof(float);
 
 } // namespace engine
 } // namespace milvus

cpp/src/db/DB.cpp

@@ -12,11 +12,8 @@ namespace zilliz {
 namespace milvus {
 namespace engine {
 
-DB::~DB() {}
-
 void DB::Open(const Options& options, DB** dbptr) {
     *dbptr = DBFactory::Build(options);
-    return;
 }
 
 } // namespace engine

cpp/src/db/DB.h

@@ -52,7 +52,7 @@ public:
     DB(const DB&) = delete;
     DB& operator=(const DB&) = delete;
 
-    virtual ~DB();
+    virtual ~DB() = 0;
 }; // DB
 
 } // namespace engine

cpp/src/db/DBMetaImpl.h

@@ -8,67 +8,88 @@
 #include "Meta.h"
 #include "Options.h"
 
+
 namespace zilliz {
 namespace milvus {
 namespace engine {
 namespace meta {
 
-auto StoragePrototype(const std::string& path);
+auto StoragePrototype(const std::string &path);
 
 class DBMetaImpl : public Meta {
-public:
-    DBMetaImpl(const DBMetaOptions& options_);
+ public:
+    explicit DBMetaImpl(const DBMetaOptions &options_);
+
+    Status
+    CreateTable(TableSchema &table_schema) override;
+
+    Status
+    DescribeTable(TableSchema &group_info_) override;
+
+    Status
+    HasTable(const std::string &table_id, bool &has_or_not) override;
+
+    Status
+    AllTables(std::vector<TableSchema> &table_schema_array) override;
+
+    Status
+    DeleteTable(const std::string &table_id) override;
 
-    virtual Status CreateTable(TableSchema& table_schema) override;
-    virtual Status DescribeTable(TableSchema& group_info_) override;
-    virtual Status HasTable(const std::string& table_id, bool& has_or_not) override;
-    virtual Status AllTables(std::vector<TableSchema>& table_schema_array) override;
+    Status
+    DeleteTableFiles(const std::string &table_id) override;
 
-    virtual Status DeleteTable(const std::string& table_id) override;
-    virtual Status DeleteTableFiles(const std::string& table_id) override;
+    Status
+    CreateTableFile(TableFileSchema &file_schema) override;
 
-    virtual Status CreateTableFile(TableFileSchema& file_schema) override;
-    virtual Status DropPartitionsByDates(const std::string& table_id,
-            const DatesT& dates) override;
+    Status
+    DropPartitionsByDates(const std::string &table_id, const DatesT &dates) override;
 
-    virtual Status GetTableFiles(const std::string& table_id,
-                                 const std::vector<size_t>& ids,
-                                 TableFilesSchema& table_files) override;
+    Status
+    GetTableFiles(const std::string &table_id, const std::vector<size_t> &ids, TableFilesSchema &table_files) override;
 
-    virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override;
+    Status
+    HasNonIndexFiles(const std::string &table_id, bool &has) override;
 
-    virtual Status UpdateTableFilesToIndex(const std::string& table_id) override;
+    Status
+    UpdateTableFilesToIndex(const std::string &table_id) override;
 
-    virtual Status UpdateTableFile(TableFileSchema& file_schema) override;
+    Status
+    UpdateTableFile(TableFileSchema &file_schema) override;
 
-    virtual Status UpdateTableFiles(TableFilesSchema& files) override;
+    Status
+    UpdateTableFiles(TableFilesSchema &files) override;
 
-    virtual Status FilesToSearch(const std::string& table_id,
-                                  const DatesT& partition,
-                                  DatePartionedTableFilesSchema& files) override;
+    Status
+    FilesToSearch(const std::string &table_id, const DatesT &partition, DatePartionedTableFilesSchema &files) override;
 
-    virtual Status FilesToMerge(const std::string& table_id,
-            DatePartionedTableFilesSchema& files) override;
+    Status
+    FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) override;
 
-    virtual Status FilesToIndex(TableFilesSchema&) override;
+    Status
+    FilesToIndex(TableFilesSchema &) override;
 
-    virtual Status Archive() override;
+    Status
+    Archive() override;
 
-    virtual Status Size(uint64_t& result) override;
+    Status
+    Size(uint64_t &result) override;
 
-    virtual Status CleanUp() override;
+    Status
+    CleanUp() override;
 
-    virtual Status CleanUpFilesWithTTL(uint16_t seconds) override;
+    Status
+    CleanUpFilesWithTTL(uint16_t seconds) override;
 
-    virtual Status DropAll() override;
+    Status
+    DropAll() override;
 
-    virtual Status Count(const std::string& table_id, uint64_t& result) override;
+    Status Count(const std::string &table_id, uint64_t &result) override;
 
-    virtual ~DBMetaImpl();
+    ~DBMetaImpl() override;
 
-private:
-    Status NextFileId(std::string& file_id);
-    Status NextTableId(std::string& table_id);
+ private:
+    Status NextFileId(std::string &file_id);
+    Status NextTableId(std::string &table_id);
     Status DiscardFiles(long to_discard_size);
     Status Initialize();
 

cpp/src/db/IDGenerator.cpp

@@ -13,7 +13,7 @@ namespace zilliz {
 namespace milvus {
 namespace engine {
 
-IDGenerator::~IDGenerator() {}
+constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO;
 
 IDNumber SimpleIDGenerator::GetNextIDNumber() {
     auto now = std::chrono::system_clock::now();

cpp/src/db/IDGenerator.h

@@ -10,28 +10,34 @@
 #include <cstddef>
 #include <vector>
 
+
 namespace zilliz {
 namespace milvus {
 namespace engine {
 
 class IDGenerator {
-public:
+ public:
     virtual IDNumber GetNextIDNumber() = 0;
-    virtual void GetNextIDNumbers(size_t n, IDNumbers& ids) = 0;
-
-    virtual ~IDGenerator();
-
+    virtual void GetNextIDNumbers(size_t n, IDNumbers &ids) = 0;
+    virtual ~IDGenerator() = 0;
 }; // IDGenerator
 
 
 class SimpleIDGenerator : public IDGenerator {
-public:
-    virtual IDNumber GetNextIDNumber() override;
-    virtual void GetNextIDNumbers(size_t n, IDNumbers& ids) override;
+ public:
+    ~SimpleIDGenerator() override = default;
+
+    IDNumber
+    GetNextIDNumber() override;
+
+    void
+    GetNextIDNumbers(size_t n, IDNumbers &ids) override;
+
+ private:
+    void
+    NextIDNumbers(size_t n, IDNumbers &ids);
 
-private:
-    void NextIDNumbers(size_t n, IDNumbers& ids);
-    const size_t MAX_IDS_PER_MICRO = 1000;
+    static constexpr size_t MAX_IDS_PER_MICRO = 1000;
 
 }; // SimpleIDGenerator
 

cpp/src/db/Meta.h

@@ -23,6 +23,7 @@ class Meta {
 public:
     using Ptr = std::shared_ptr<Meta>;
 
+    virtual ~Meta() = 0;
     virtual Status CreateTable(TableSchema& table_schema) = 0;
     virtual Status DescribeTable(TableSchema& table_schema) = 0;
     virtual Status HasTable(const std::string& table_id, bool& has_or_not) = 0;

cpp/src/db/MySQLMetaImpl.h

@@ -12,79 +12,80 @@
 #include "mysql++/mysql++.h"
 #include <mutex>
 
+
 namespace zilliz {
 namespace milvus {
 namespace engine {
 namespace meta {
 
 //    auto StoragePrototype(const std::string& path);
-    using namespace mysqlpp;
+using namespace mysqlpp;
 
-    class MySQLMetaImpl : public Meta {
-    public:
-        MySQLMetaImpl(const DBMetaOptions& options_, const int& mode);
+class MySQLMetaImpl : public Meta {
+ public:
+    MySQLMetaImpl(const DBMetaOptions &options_, const int &mode);
 
-        virtual Status CreateTable(TableSchema& table_schema) override;
-        virtual Status DescribeTable(TableSchema& group_info_) override;
-        virtual Status HasTable(const std::string& table_id, bool& has_or_not) override;
-        virtual Status AllTables(std::vector<TableSchema>& table_schema_array) override;
+    Status CreateTable(TableSchema &table_schema) override;
+    Status DescribeTable(TableSchema &group_info_) override;
+    Status HasTable(const std::string &table_id, bool &has_or_not) override;
+    Status AllTables(std::vector<TableSchema> &table_schema_array) override;
 
-        virtual Status DeleteTable(const std::string& table_id) override;
-        virtual Status DeleteTableFiles(const std::string& table_id) override;
+    Status DeleteTable(const std::string &table_id) override;
+    Status DeleteTableFiles(const std::string &table_id) override;
 
-        virtual Status CreateTableFile(TableFileSchema& file_schema) override;
-        virtual Status DropPartitionsByDates(const std::string& table_id,
-                                             const DatesT& dates) override;
+    Status CreateTableFile(TableFileSchema &file_schema) override;
+    Status DropPartitionsByDates(const std::string &table_id,
+                                 const DatesT &dates) override;
 
-        virtual Status GetTableFiles(const std::string& table_id,
-                                     const std::vector<size_t>& ids,
-                                     TableFilesSchema& table_files) override;
+    Status GetTableFiles(const std::string &table_id,
+                         const std::vector<size_t> &ids,
+                         TableFilesSchema &table_files) override;
 
-        virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override;
+    Status HasNonIndexFiles(const std::string &table_id, bool &has) override;
 
-        virtual Status UpdateTableFile(TableFileSchema& file_schema) override;
+    Status UpdateTableFile(TableFileSchema &file_schema) override;
 
-        virtual Status UpdateTableFilesToIndex(const std::string& table_id) override;
+    Status UpdateTableFilesToIndex(const std::string &table_id) override;
 
-        virtual Status UpdateTableFiles(TableFilesSchema& files) override;
+    Status UpdateTableFiles(TableFilesSchema &files) override;
 
-        virtual Status FilesToSearch(const std::string& table_id,
-                                     const DatesT& partition,
-                                     DatePartionedTableFilesSchema& files) override;
+    Status FilesToSearch(const std::string &table_id,
+                         const DatesT &partition,
+                         DatePartionedTableFilesSchema &files) override;
 
-        virtual Status FilesToMerge(const std::string& table_id,
-                                    DatePartionedTableFilesSchema& files) override;
+    Status FilesToMerge(const std::string &table_id,
+                        DatePartionedTableFilesSchema &files) override;
 
-        virtual Status FilesToIndex(TableFilesSchema&) override;
+    Status FilesToIndex(TableFilesSchema &) override;
 
-        virtual Status Archive() override;
+    Status Archive() override;
 
-        virtual Status Size(uint64_t& result) override;
+    Status Size(uint64_t &result) override;
 
-        virtual Status CleanUp() override;
+    Status CleanUp() override;
 
-        virtual Status CleanUpFilesWithTTL(uint16_t seconds) override;
+    Status CleanUpFilesWithTTL(uint16_t seconds) override;
 
-        virtual Status DropAll() override;
+    Status DropAll() override;
 
-        virtual Status Count(const std::string& table_id, uint64_t& result) override;
+    Status Count(const std::string &table_id, uint64_t &result) override;
 
-        virtual ~MySQLMetaImpl();
+    virtual ~MySQLMetaImpl();
 
-    private:
-        Status NextFileId(std::string& file_id);
-        Status NextTableId(std::string& table_id);
-        Status DiscardFiles(long long to_discard_size);
-        Status Initialize();
+ private:
+    Status NextFileId(std::string &file_id);
+    Status NextTableId(std::string &table_id);
+    Status DiscardFiles(long long to_discard_size);
+    Status Initialize();
 
-        const DBMetaOptions options_;
-        const int mode_;
+    const DBMetaOptions options_;
+    const int mode_;
 
-        std::shared_ptr<MySQLConnectionPool> mysql_connection_pool_;
-        bool safe_grab = false;
+    std::shared_ptr<MySQLConnectionPool> mysql_connection_pool_;
+    bool safe_grab = false;
 
 //        std::mutex connectionMutex_;
-    }; // DBMetaImpl
+}; // DBMetaImpl
 
 } // namespace meta
 } // namespace engine

cpp/src/db/scheduler/TaskDispatchStrategy.cpp

@@ -20,6 +20,7 @@ class ReuseCacheIndexStrategy {
 public:
     bool Schedule(const SearchContextPtr &context, std::list<ScheduleTaskPtr>& task_list) {
         if(context == nullptr) {
+            ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist";
             return false;
         }
 
@@ -64,6 +65,7 @@ class DeleteTableStrategy {
 public:
     bool Schedule(const DeleteContextPtr &context, std::list<ScheduleTaskPtr> &task_list) {
         if (context == nullptr) {
+            ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist";
             return false;
         }
 
@@ -103,6 +105,7 @@ public:
 bool TaskDispatchStrategy::Schedule(const ScheduleContextPtr &context_ptr,
                                     std::list<zilliz::milvus::engine::ScheduleTaskPtr> &task_list) {
     if(context_ptr == nullptr) {
+        ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist";
         return false;
     }
 

cpp/src/db/scheduler/TaskScheduler.cpp

@@ -31,6 +31,7 @@ TaskScheduler& TaskScheduler::GetInstance() {
 bool
 TaskScheduler::Start() {
     if(!stopped_) {
+        SERVER_LOG_INFO << "Task Scheduler isn't started";
         return true;
     }
 
@@ -47,6 +48,7 @@ TaskScheduler::Start() {
 bool
 TaskScheduler::Stop() {
     if(stopped_) {
+        SERVER_LOG_INFO << "Task Scheduler already stopped";
         return true;
     }
 
@@ -80,7 +82,7 @@ TaskScheduler::TaskDispatchWorker() {
         ScheduleTaskPtr task_ptr = task_dispatch_queue_.Take();
         if(task_ptr == nullptr) {
             SERVER_LOG_INFO << "Stop db task dispatch thread";
-            break;//exit
+            return true;
         }
 
         //execute task
@@ -98,8 +100,8 @@ TaskScheduler::TaskWorker() {
     while(true) {
         ScheduleTaskPtr task_ptr = task_queue_.Take();
         if(task_ptr == nullptr) {
-            SERVER_LOG_INFO << "Stop db task thread";
-            break;//exit
+            SERVER_LOG_INFO << "Stop db task worker thread";
+            return true;
         }
 
         //execute task