From 988382277838d0cb5f178a7bda96f55164b00596 Mon Sep 17 00:00:00 2001 From: jinhai Date: Wed, 24 Jul 2019 11:02:22 +0800 Subject: [PATCH] MS-273 Add some logs and reformat code Former-commit-id: 4c906c96c1342ccd8aa674c8b4175916cd53a8d0 --- cpp/src/cache/Cache.cpp | 2 +- cpp/src/cache/CacheMgr.cpp | 13 +++ cpp/src/cache/CpuCacheMgr.cpp | 6 +- cpp/src/cache/GpuCacheMgr.cpp | 6 +- cpp/src/config/ConfigNode.cpp | 2 +- cpp/src/db/Constants.h | 12 +-- cpp/src/db/DB.cpp | 3 - cpp/src/db/DB.h | 2 +- cpp/src/db/DBMetaImpl.h | 91 ++++++++++++------- cpp/src/db/IDGenerator.cpp | 2 +- cpp/src/db/IDGenerator.h | 28 +++--- cpp/src/db/Meta.h | 1 + cpp/src/db/MySQLMetaImpl.h | 87 +++++++++--------- cpp/src/db/scheduler/TaskDispatchStrategy.cpp | 3 + cpp/src/db/scheduler/TaskScheduler.cpp | 8 +- 15 files changed, 159 insertions(+), 107 deletions(-) diff --git a/cpp/src/cache/Cache.cpp b/cpp/src/cache/Cache.cpp index a1f25203..336bf4e8 100644 --- a/cpp/src/cache/Cache.cpp +++ b/cpp/src/cache/Cache.cpp @@ -89,7 +89,7 @@ void Cache::erase(const std::string& key) { const DataObjPtr& data_ptr = obj_ptr->data_; usage_ -= data_ptr->size(); - SERVER_LOG_DEBUG << "Erase " << key << " from cache"; + SERVER_LOG_DEBUG << "Erase " << key << " size: " << data_ptr->size(); lru_.erase(key); } diff --git a/cpp/src/cache/CacheMgr.cpp b/cpp/src/cache/CacheMgr.cpp index 14efb3a0..95deb4c8 100644 --- a/cpp/src/cache/CacheMgr.cpp +++ b/cpp/src/cache/CacheMgr.cpp @@ -4,6 +4,7 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// +#include "utils/Log.h" #include "CacheMgr.h" #include "metrics/Metrics.h" @@ -20,6 +21,7 @@ CacheMgr::~CacheMgr() { uint64_t CacheMgr::ItemCount() const { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } @@ -28,6 +30,7 @@ uint64_t CacheMgr::ItemCount() const { bool CacheMgr::ItemExists(const std::string& key) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return false; } @@ -36,6 +39,7 @@ bool CacheMgr::ItemExists(const std::string& key) { DataObjPtr CacheMgr::GetItem(const std::string& key) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return nullptr; } server::Metrics::GetInstance().CacheAccessTotalIncrement(); @@ -45,6 +49,7 @@ DataObjPtr CacheMgr::GetItem(const std::string& key) { engine::Index_ptr CacheMgr::GetIndex(const std::string& key) { DataObjPtr obj = GetItem(key); if(obj != nullptr) { + SERVER_LOG_ERROR << "Can't get object from key: " << key; return obj->data(); } @@ -53,6 +58,7 @@ engine::Index_ptr CacheMgr::GetIndex(const std::string& key) { void CacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -62,6 +68,7 @@ void CacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) { void CacheMgr::InsertItem(const std::string& key, const engine::Index_ptr& index) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -72,6 +79,7 @@ void CacheMgr::InsertItem(const std::string& key, const engine::Index_ptr& index void CacheMgr::EraseItem(const std::string& key) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -81,6 +89,7 @@ void CacheMgr::EraseItem(const std::string& key) { void CacheMgr::PrintInfo() { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -89,6 +98,7 @@ void CacheMgr::PrintInfo() { void CacheMgr::ClearCache() { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -97,6 +107,7 @@ void CacheMgr::ClearCache() { int64_t CacheMgr::CacheUsage() const { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } @@ -105,6 +116,7 @@ int64_t CacheMgr::CacheUsage() const { int64_t CacheMgr::CacheCapacity() const { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } @@ -113,6 +125,7 @@ int64_t CacheMgr::CacheCapacity() const { void CacheMgr::SetCapacity(int64_t capacity) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } cache_->set_capacity(capacity); diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index a90f8537..167f91f5 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -12,10 +12,14 @@ namespace zilliz { namespace milvus { namespace cache { +namespace { + constexpr int64_t unit = 1024 * 1024 * 1024; +} + CpuCacheMgr::CpuCacheMgr() { server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16); - cap *= 1024*1024*1024; + cap *= unit; cache_ = std::make_shared(cap, 1UL<<32); double free_percent = config.GetDoubleValue(server::CACHE_FREE_PERCENT, 0.85); diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index b3e73fdc..13eec4f2 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -11,10 +11,14 @@ namespace zilliz { namespace milvus { namespace cache { +namespace { + constexpr int64_t unit = 1024 * 1024 * 1024; +} + GpuCacheMgr::GpuCacheMgr() { server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); int64_t cap = config.GetInt64Value(server::CONFIG_GPU_CACHE_CAPACITY, 1); - cap *= 1024*1024*1024; + cap *= unit; cache_ = std::make_shared(cap, 1UL<<32); } diff --git a/cpp/src/config/ConfigNode.cpp b/cpp/src/config/ConfigNode.cpp index e17eea1d..a6b91bf6 100644 --- a/cpp/src/config/ConfigNode.cpp +++ b/cpp/src/config/ConfigNode.cpp @@ -94,7 +94,7 @@ double ConfigNode::GetDoubleValue(const std::string ¶m_key, double default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { - return std::strtold(val.c_str(), nullptr); + return std::strtod(val.c_str(), nullptr); } else { return default_val; } diff --git a/cpp/src/db/Constants.h b/cpp/src/db/Constants.h index 055b10ca..e94dfa6a 100644 --- a/cpp/src/db/Constants.h +++ b/cpp/src/db/Constants.h @@ -9,14 +9,14 @@ namespace zilliz { namespace milvus { namespace engine { -const size_t K = 1024UL; -const size_t M = K * K; -const size_t G = K * M; -const size_t T = K * G; +constexpr size_t K = 1024UL; +constexpr size_t M = K * K; +constexpr size_t G = K * M; +constexpr size_t T = K * G; -const size_t MAX_TABLE_FILE_MEM = 128 * M; +constexpr size_t MAX_TABLE_FILE_MEM = 128 * M; -const int VECTOR_TYPE_SIZE = sizeof(float); +constexpr int VECTOR_TYPE_SIZE = sizeof(float); } // namespace engine } // namespace milvus diff --git a/cpp/src/db/DB.cpp b/cpp/src/db/DB.cpp index ca08bbd9..1365d4e1 100644 --- a/cpp/src/db/DB.cpp +++ b/cpp/src/db/DB.cpp @@ -12,11 +12,8 @@ namespace zilliz { namespace milvus { namespace engine { -DB::~DB() {} - void DB::Open(const Options& options, DB** dbptr) { *dbptr = DBFactory::Build(options); - return; } } // namespace engine diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index a4163124..3bfd2281 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -52,7 +52,7 @@ public: DB(const DB&) = delete; DB& operator=(const DB&) = delete; - virtual ~DB(); + virtual ~DB() = 0; }; // DB } // namespace engine diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 98a42c00..6187ad7e 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -8,67 +8,88 @@ #include "Meta.h" #include "Options.h" + namespace zilliz { namespace milvus { namespace engine { namespace meta { -auto StoragePrototype(const std::string& path); +auto StoragePrototype(const std::string &path); class DBMetaImpl : public Meta { -public: - DBMetaImpl(const DBMetaOptions& options_); + public: + explicit DBMetaImpl(const DBMetaOptions &options_); + + Status + CreateTable(TableSchema &table_schema) override; + + Status + DescribeTable(TableSchema &group_info_) override; + + Status + HasTable(const std::string &table_id, bool &has_or_not) override; + + Status + AllTables(std::vector &table_schema_array) override; + + Status + DeleteTable(const std::string &table_id) override; - virtual Status CreateTable(TableSchema& table_schema) override; - virtual Status DescribeTable(TableSchema& group_info_) override; - virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; - virtual Status AllTables(std::vector& table_schema_array) override; + Status + DeleteTableFiles(const std::string &table_id) override; - virtual Status DeleteTable(const std::string& table_id) override; - virtual Status DeleteTableFiles(const std::string& table_id) override; + Status + CreateTableFile(TableFileSchema &file_schema) override; - virtual Status CreateTableFile(TableFileSchema& file_schema) override; - virtual Status DropPartitionsByDates(const std::string& table_id, - const DatesT& dates) override; + Status + DropPartitionsByDates(const std::string &table_id, const DatesT &dates) override; - virtual Status GetTableFiles(const std::string& table_id, - const std::vector& ids, - TableFilesSchema& table_files) override; + Status + GetTableFiles(const std::string &table_id, const std::vector &ids, TableFilesSchema &table_files) override; - virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override; + Status + HasNonIndexFiles(const std::string &table_id, bool &has) override; - virtual Status UpdateTableFilesToIndex(const std::string& table_id) override; + Status + UpdateTableFilesToIndex(const std::string &table_id) override; - virtual Status UpdateTableFile(TableFileSchema& file_schema) override; + Status + UpdateTableFile(TableFileSchema &file_schema) override; - virtual Status UpdateTableFiles(TableFilesSchema& files) override; + Status + UpdateTableFiles(TableFilesSchema &files) override; - virtual Status FilesToSearch(const std::string& table_id, - const DatesT& partition, - DatePartionedTableFilesSchema& files) override; + Status + FilesToSearch(const std::string &table_id, const DatesT &partition, DatePartionedTableFilesSchema &files) override; - virtual Status FilesToMerge(const std::string& table_id, - DatePartionedTableFilesSchema& files) override; + Status + FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) override; - virtual Status FilesToIndex(TableFilesSchema&) override; + Status + FilesToIndex(TableFilesSchema &) override; - virtual Status Archive() override; + Status + Archive() override; - virtual Status Size(uint64_t& result) override; + Status + Size(uint64_t &result) override; - virtual Status CleanUp() override; + Status + CleanUp() override; - virtual Status CleanUpFilesWithTTL(uint16_t seconds) override; + Status + CleanUpFilesWithTTL(uint16_t seconds) override; - virtual Status DropAll() override; + Status + DropAll() override; - virtual Status Count(const std::string& table_id, uint64_t& result) override; + Status Count(const std::string &table_id, uint64_t &result) override; - virtual ~DBMetaImpl(); + ~DBMetaImpl() override; -private: - Status NextFileId(std::string& file_id); - Status NextTableId(std::string& table_id); + private: + Status NextFileId(std::string &file_id); + Status NextTableId(std::string &table_id); Status DiscardFiles(long to_discard_size); Status Initialize(); diff --git a/cpp/src/db/IDGenerator.cpp b/cpp/src/db/IDGenerator.cpp index c4f9787b..70d71817 100644 --- a/cpp/src/db/IDGenerator.cpp +++ b/cpp/src/db/IDGenerator.cpp @@ -13,7 +13,7 @@ namespace zilliz { namespace milvus { namespace engine { -IDGenerator::~IDGenerator() {} +constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO; IDNumber SimpleIDGenerator::GetNextIDNumber() { auto now = std::chrono::system_clock::now(); diff --git a/cpp/src/db/IDGenerator.h b/cpp/src/db/IDGenerator.h index 07948506..502b2865 100644 --- a/cpp/src/db/IDGenerator.h +++ b/cpp/src/db/IDGenerator.h @@ -10,28 +10,34 @@ #include #include + namespace zilliz { namespace milvus { namespace engine { class IDGenerator { -public: + public: virtual IDNumber GetNextIDNumber() = 0; - virtual void GetNextIDNumbers(size_t n, IDNumbers& ids) = 0; - - virtual ~IDGenerator(); - + virtual void GetNextIDNumbers(size_t n, IDNumbers &ids) = 0; + virtual ~IDGenerator() = 0; }; // IDGenerator class SimpleIDGenerator : public IDGenerator { -public: - virtual IDNumber GetNextIDNumber() override; - virtual void GetNextIDNumbers(size_t n, IDNumbers& ids) override; + public: + ~SimpleIDGenerator() override = default; + + IDNumber + GetNextIDNumber() override; + + void + GetNextIDNumbers(size_t n, IDNumbers &ids) override; + + private: + void + NextIDNumbers(size_t n, IDNumbers &ids); -private: - void NextIDNumbers(size_t n, IDNumbers& ids); - const size_t MAX_IDS_PER_MICRO = 1000; + static constexpr size_t MAX_IDS_PER_MICRO = 1000; }; // SimpleIDGenerator diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 1d83817f..b18cea35 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -23,6 +23,7 @@ class Meta { public: using Ptr = std::shared_ptr; + virtual ~Meta() = 0; virtual Status CreateTable(TableSchema& table_schema) = 0; virtual Status DescribeTable(TableSchema& table_schema) = 0; virtual Status HasTable(const std::string& table_id, bool& has_or_not) = 0; diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index 403d5b0b..87bc1783 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -12,79 +12,80 @@ #include "mysql++/mysql++.h" #include + namespace zilliz { namespace milvus { namespace engine { namespace meta { // auto StoragePrototype(const std::string& path); - using namespace mysqlpp; +using namespace mysqlpp; - class MySQLMetaImpl : public Meta { - public: - MySQLMetaImpl(const DBMetaOptions& options_, const int& mode); +class MySQLMetaImpl : public Meta { + public: + MySQLMetaImpl(const DBMetaOptions &options_, const int &mode); - virtual Status CreateTable(TableSchema& table_schema) override; - virtual Status DescribeTable(TableSchema& group_info_) override; - virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; - virtual Status AllTables(std::vector& table_schema_array) override; + Status CreateTable(TableSchema &table_schema) override; + Status DescribeTable(TableSchema &group_info_) override; + Status HasTable(const std::string &table_id, bool &has_or_not) override; + Status AllTables(std::vector &table_schema_array) override; - virtual Status DeleteTable(const std::string& table_id) override; - virtual Status DeleteTableFiles(const std::string& table_id) override; + Status DeleteTable(const std::string &table_id) override; + Status DeleteTableFiles(const std::string &table_id) override; - virtual Status CreateTableFile(TableFileSchema& file_schema) override; - virtual Status DropPartitionsByDates(const std::string& table_id, - const DatesT& dates) override; + Status CreateTableFile(TableFileSchema &file_schema) override; + Status DropPartitionsByDates(const std::string &table_id, + const DatesT &dates) override; - virtual Status GetTableFiles(const std::string& table_id, - const std::vector& ids, - TableFilesSchema& table_files) override; + Status GetTableFiles(const std::string &table_id, + const std::vector &ids, + TableFilesSchema &table_files) override; - virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override; + Status HasNonIndexFiles(const std::string &table_id, bool &has) override; - virtual Status UpdateTableFile(TableFileSchema& file_schema) override; + Status UpdateTableFile(TableFileSchema &file_schema) override; - virtual Status UpdateTableFilesToIndex(const std::string& table_id) override; + Status UpdateTableFilesToIndex(const std::string &table_id) override; - virtual Status UpdateTableFiles(TableFilesSchema& files) override; + Status UpdateTableFiles(TableFilesSchema &files) override; - virtual Status FilesToSearch(const std::string& table_id, - const DatesT& partition, - DatePartionedTableFilesSchema& files) override; + Status FilesToSearch(const std::string &table_id, + const DatesT &partition, + DatePartionedTableFilesSchema &files) override; - virtual Status FilesToMerge(const std::string& table_id, - DatePartionedTableFilesSchema& files) override; + Status FilesToMerge(const std::string &table_id, + DatePartionedTableFilesSchema &files) override; - virtual Status FilesToIndex(TableFilesSchema&) override; + Status FilesToIndex(TableFilesSchema &) override; - virtual Status Archive() override; + Status Archive() override; - virtual Status Size(uint64_t& result) override; + Status Size(uint64_t &result) override; - virtual Status CleanUp() override; + Status CleanUp() override; - virtual Status CleanUpFilesWithTTL(uint16_t seconds) override; + Status CleanUpFilesWithTTL(uint16_t seconds) override; - virtual Status DropAll() override; + Status DropAll() override; - virtual Status Count(const std::string& table_id, uint64_t& result) override; + Status Count(const std::string &table_id, uint64_t &result) override; - virtual ~MySQLMetaImpl(); + virtual ~MySQLMetaImpl(); - private: - Status NextFileId(std::string& file_id); - Status NextTableId(std::string& table_id); - Status DiscardFiles(long long to_discard_size); - Status Initialize(); + private: + Status NextFileId(std::string &file_id); + Status NextTableId(std::string &table_id); + Status DiscardFiles(long long to_discard_size); + Status Initialize(); - const DBMetaOptions options_; - const int mode_; + const DBMetaOptions options_; + const int mode_; - std::shared_ptr mysql_connection_pool_; - bool safe_grab = false; + std::shared_ptr mysql_connection_pool_; + bool safe_grab = false; // std::mutex connectionMutex_; - }; // DBMetaImpl +}; // DBMetaImpl } // namespace meta } // namespace engine diff --git a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp index 73c46942..38c94631 100644 --- a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp +++ b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp @@ -20,6 +20,7 @@ class ReuseCacheIndexStrategy { public: bool Schedule(const SearchContextPtr &context, std::list& task_list) { if(context == nullptr) { + ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist"; return false; } @@ -64,6 +65,7 @@ class DeleteTableStrategy { public: bool Schedule(const DeleteContextPtr &context, std::list &task_list) { if (context == nullptr) { + ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist"; return false; } @@ -103,6 +105,7 @@ public: bool TaskDispatchStrategy::Schedule(const ScheduleContextPtr &context_ptr, std::list &task_list) { if(context_ptr == nullptr) { + ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist"; return false; } diff --git a/cpp/src/db/scheduler/TaskScheduler.cpp b/cpp/src/db/scheduler/TaskScheduler.cpp index 04d6762a..f29c3d4c 100644 --- a/cpp/src/db/scheduler/TaskScheduler.cpp +++ b/cpp/src/db/scheduler/TaskScheduler.cpp @@ -31,6 +31,7 @@ TaskScheduler& TaskScheduler::GetInstance() { bool TaskScheduler::Start() { if(!stopped_) { + SERVER_LOG_INFO << "Task Scheduler isn't started"; return true; } @@ -47,6 +48,7 @@ TaskScheduler::Start() { bool TaskScheduler::Stop() { if(stopped_) { + SERVER_LOG_INFO << "Task Scheduler already stopped"; return true; } @@ -80,7 +82,7 @@ TaskScheduler::TaskDispatchWorker() { ScheduleTaskPtr task_ptr = task_dispatch_queue_.Take(); if(task_ptr == nullptr) { SERVER_LOG_INFO << "Stop db task dispatch thread"; - break;//exit + return true; } //execute task @@ -98,8 +100,8 @@ TaskScheduler::TaskWorker() { while(true) { ScheduleTaskPtr task_ptr = task_queue_.Take(); if(task_ptr == nullptr) { - SERVER_LOG_INFO << "Stop db task thread"; - break;//exit + SERVER_LOG_INFO << "Stop db task worker thread"; + return true; } //execute task -- GitLab