diff --git a/cpp/src/cache/Cache.cpp b/cpp/src/cache/Cache.cpp index a1f252030296a27567f580383a7fd6b61cf4f3d3..336bf4e82ce7c67b1ad80bee5de03860f9c102d2 100644 --- a/cpp/src/cache/Cache.cpp +++ b/cpp/src/cache/Cache.cpp @@ -89,7 +89,7 @@ void Cache::erase(const std::string& key) { const DataObjPtr& data_ptr = obj_ptr->data_; usage_ -= data_ptr->size(); - SERVER_LOG_DEBUG << "Erase " << key << " from cache"; + SERVER_LOG_DEBUG << "Erase " << key << " size: " << data_ptr->size(); lru_.erase(key); } diff --git a/cpp/src/cache/CacheMgr.cpp b/cpp/src/cache/CacheMgr.cpp index 14efb3a0bbf9eb8a572544b8bc22b53e5093d491..95deb4c80d8a5857203a949cf81277d403a0a1c7 100644 --- a/cpp/src/cache/CacheMgr.cpp +++ b/cpp/src/cache/CacheMgr.cpp @@ -4,6 +4,7 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// +#include "utils/Log.h" #include "CacheMgr.h" #include "metrics/Metrics.h" @@ -20,6 +21,7 @@ CacheMgr::~CacheMgr() { uint64_t CacheMgr::ItemCount() const { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } @@ -28,6 +30,7 @@ uint64_t CacheMgr::ItemCount() const { bool CacheMgr::ItemExists(const std::string& key) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return false; } @@ -36,6 +39,7 @@ bool CacheMgr::ItemExists(const std::string& key) { DataObjPtr CacheMgr::GetItem(const std::string& key) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return nullptr; } server::Metrics::GetInstance().CacheAccessTotalIncrement(); @@ -45,6 +49,7 @@ DataObjPtr CacheMgr::GetItem(const std::string& key) { engine::Index_ptr CacheMgr::GetIndex(const std::string& key) { DataObjPtr obj = GetItem(key); if(obj != nullptr) { + SERVER_LOG_ERROR << "Can't get object from key: " << key; return obj->data(); } @@ -53,6 +58,7 @@ engine::Index_ptr CacheMgr::GetIndex(const std::string& key) { void CacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -62,6 +68,7 @@ void CacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) { void CacheMgr::InsertItem(const std::string& key, const engine::Index_ptr& index) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -72,6 +79,7 @@ void CacheMgr::InsertItem(const std::string& key, const engine::Index_ptr& index void CacheMgr::EraseItem(const std::string& key) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -81,6 +89,7 @@ void CacheMgr::EraseItem(const std::string& key) { void CacheMgr::PrintInfo() { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -89,6 +98,7 @@ void CacheMgr::PrintInfo() { void CacheMgr::ClearCache() { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -97,6 +107,7 @@ void CacheMgr::ClearCache() { int64_t CacheMgr::CacheUsage() const { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } @@ -105,6 +116,7 @@ int64_t CacheMgr::CacheUsage() const { int64_t CacheMgr::CacheCapacity() const { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } @@ -113,6 +125,7 @@ int64_t CacheMgr::CacheCapacity() const { void CacheMgr::SetCapacity(int64_t capacity) { if(cache_ == nullptr) { + SERVER_LOG_ERROR << "Cache doesn't exist"; return; } cache_->set_capacity(capacity); diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index a90f8537b443c27d561235a34a45753a9f4478ce..167f91f5e596220c47457925b8cb6ed7763bbfab 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -12,10 +12,14 @@ namespace zilliz { namespace milvus { namespace cache { +namespace { + constexpr int64_t unit = 1024 * 1024 * 1024; +} + CpuCacheMgr::CpuCacheMgr() { server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16); - cap *= 1024*1024*1024; + cap *= unit; cache_ = std::make_shared(cap, 1UL<<32); double free_percent = config.GetDoubleValue(server::CACHE_FREE_PERCENT, 0.85); diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index b3e73fdcdc1bf291376aacd7c4fc617bb53abde6..13eec4f2b661e148640492eabb40ad701a1d0f50 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -11,10 +11,14 @@ namespace zilliz { namespace milvus { namespace cache { +namespace { + constexpr int64_t unit = 1024 * 1024 * 1024; +} + GpuCacheMgr::GpuCacheMgr() { server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); int64_t cap = config.GetInt64Value(server::CONFIG_GPU_CACHE_CAPACITY, 1); - cap *= 1024*1024*1024; + cap *= unit; cache_ = std::make_shared(cap, 1UL<<32); } diff --git a/cpp/src/config/ConfigNode.cpp b/cpp/src/config/ConfigNode.cpp index e17eea1dafc1e81f86d241f185d0d1aa0ea911af..a6b91bf67d60a899dd735f022cb35c2ef3eac29b 100644 --- a/cpp/src/config/ConfigNode.cpp +++ b/cpp/src/config/ConfigNode.cpp @@ -94,7 +94,7 @@ double ConfigNode::GetDoubleValue(const std::string ¶m_key, double default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { - return std::strtold(val.c_str(), nullptr); + return std::strtod(val.c_str(), nullptr); } else { return default_val; } diff --git a/cpp/src/db/Constants.h b/cpp/src/db/Constants.h index 055b10ca9a9156d1809dca519738c72aadc32b8c..e94dfa6aea8710948908b0ea3fe4bb2e2793ee5e 100644 --- a/cpp/src/db/Constants.h +++ b/cpp/src/db/Constants.h @@ -9,14 +9,14 @@ namespace zilliz { namespace milvus { namespace engine { -const size_t K = 1024UL; -const size_t M = K * K; -const size_t G = K * M; -const size_t T = K * G; +constexpr size_t K = 1024UL; +constexpr size_t M = K * K; +constexpr size_t G = K * M; +constexpr size_t T = K * G; -const size_t MAX_TABLE_FILE_MEM = 128 * M; +constexpr size_t MAX_TABLE_FILE_MEM = 128 * M; -const int VECTOR_TYPE_SIZE = sizeof(float); +constexpr int VECTOR_TYPE_SIZE = sizeof(float); } // namespace engine } // namespace milvus diff --git a/cpp/src/db/DB.cpp b/cpp/src/db/DB.cpp index ca08bbd9b40bed332f4529de3c0f02beb4e7d62d..1365d4e116ed00859e473fd139f1ea6a1fbc0d01 100644 --- a/cpp/src/db/DB.cpp +++ b/cpp/src/db/DB.cpp @@ -12,11 +12,8 @@ namespace zilliz { namespace milvus { namespace engine { -DB::~DB() {} - void DB::Open(const Options& options, DB** dbptr) { *dbptr = DBFactory::Build(options); - return; } } // namespace engine diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index a416312485d423bdfc47b9411eab13186da2f0fb..3bfd2281e2268d8ae8e6a9d18ed5f0e9247d8bed 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -52,7 +52,7 @@ public: DB(const DB&) = delete; DB& operator=(const DB&) = delete; - virtual ~DB(); + virtual ~DB() = 0; }; // DB } // namespace engine diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 98a42c000dfa46242ecef9b8a1e2e0b3e0cbccea..6187ad7eae3dc30a48473cd5a518b1ca102f6c2d 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -8,67 +8,88 @@ #include "Meta.h" #include "Options.h" + namespace zilliz { namespace milvus { namespace engine { namespace meta { -auto StoragePrototype(const std::string& path); +auto StoragePrototype(const std::string &path); class DBMetaImpl : public Meta { -public: - DBMetaImpl(const DBMetaOptions& options_); + public: + explicit DBMetaImpl(const DBMetaOptions &options_); + + Status + CreateTable(TableSchema &table_schema) override; + + Status + DescribeTable(TableSchema &group_info_) override; + + Status + HasTable(const std::string &table_id, bool &has_or_not) override; + + Status + AllTables(std::vector &table_schema_array) override; + + Status + DeleteTable(const std::string &table_id) override; - virtual Status CreateTable(TableSchema& table_schema) override; - virtual Status DescribeTable(TableSchema& group_info_) override; - virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; - virtual Status AllTables(std::vector& table_schema_array) override; + Status + DeleteTableFiles(const std::string &table_id) override; - virtual Status DeleteTable(const std::string& table_id) override; - virtual Status DeleteTableFiles(const std::string& table_id) override; + Status + CreateTableFile(TableFileSchema &file_schema) override; - virtual Status CreateTableFile(TableFileSchema& file_schema) override; - virtual Status DropPartitionsByDates(const std::string& table_id, - const DatesT& dates) override; + Status + DropPartitionsByDates(const std::string &table_id, const DatesT &dates) override; - virtual Status GetTableFiles(const std::string& table_id, - const std::vector& ids, - TableFilesSchema& table_files) override; + Status + GetTableFiles(const std::string &table_id, const std::vector &ids, TableFilesSchema &table_files) override; - virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override; + Status + HasNonIndexFiles(const std::string &table_id, bool &has) override; - virtual Status UpdateTableFilesToIndex(const std::string& table_id) override; + Status + UpdateTableFilesToIndex(const std::string &table_id) override; - virtual Status UpdateTableFile(TableFileSchema& file_schema) override; + Status + UpdateTableFile(TableFileSchema &file_schema) override; - virtual Status UpdateTableFiles(TableFilesSchema& files) override; + Status + UpdateTableFiles(TableFilesSchema &files) override; - virtual Status FilesToSearch(const std::string& table_id, - const DatesT& partition, - DatePartionedTableFilesSchema& files) override; + Status + FilesToSearch(const std::string &table_id, const DatesT &partition, DatePartionedTableFilesSchema &files) override; - virtual Status FilesToMerge(const std::string& table_id, - DatePartionedTableFilesSchema& files) override; + Status + FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) override; - virtual Status FilesToIndex(TableFilesSchema&) override; + Status + FilesToIndex(TableFilesSchema &) override; - virtual Status Archive() override; + Status + Archive() override; - virtual Status Size(uint64_t& result) override; + Status + Size(uint64_t &result) override; - virtual Status CleanUp() override; + Status + CleanUp() override; - virtual Status CleanUpFilesWithTTL(uint16_t seconds) override; + Status + CleanUpFilesWithTTL(uint16_t seconds) override; - virtual Status DropAll() override; + Status + DropAll() override; - virtual Status Count(const std::string& table_id, uint64_t& result) override; + Status Count(const std::string &table_id, uint64_t &result) override; - virtual ~DBMetaImpl(); + ~DBMetaImpl() override; -private: - Status NextFileId(std::string& file_id); - Status NextTableId(std::string& table_id); + private: + Status NextFileId(std::string &file_id); + Status NextTableId(std::string &table_id); Status DiscardFiles(long to_discard_size); Status Initialize(); diff --git a/cpp/src/db/IDGenerator.cpp b/cpp/src/db/IDGenerator.cpp index c4f9787ba76870152ae25d979741175cbfe2c210..70d71817fcb7488cac13098beee4d73b99fbd3b1 100644 --- a/cpp/src/db/IDGenerator.cpp +++ b/cpp/src/db/IDGenerator.cpp @@ -13,7 +13,7 @@ namespace zilliz { namespace milvus { namespace engine { -IDGenerator::~IDGenerator() {} +constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO; IDNumber SimpleIDGenerator::GetNextIDNumber() { auto now = std::chrono::system_clock::now(); diff --git a/cpp/src/db/IDGenerator.h b/cpp/src/db/IDGenerator.h index 079485060cdf80697ea3f9625b49e41659265549..502b2865f1d764195c106953c079e2966a64c854 100644 --- a/cpp/src/db/IDGenerator.h +++ b/cpp/src/db/IDGenerator.h @@ -10,28 +10,34 @@ #include #include + namespace zilliz { namespace milvus { namespace engine { class IDGenerator { -public: + public: virtual IDNumber GetNextIDNumber() = 0; - virtual void GetNextIDNumbers(size_t n, IDNumbers& ids) = 0; - - virtual ~IDGenerator(); - + virtual void GetNextIDNumbers(size_t n, IDNumbers &ids) = 0; + virtual ~IDGenerator() = 0; }; // IDGenerator class SimpleIDGenerator : public IDGenerator { -public: - virtual IDNumber GetNextIDNumber() override; - virtual void GetNextIDNumbers(size_t n, IDNumbers& ids) override; + public: + ~SimpleIDGenerator() override = default; + + IDNumber + GetNextIDNumber() override; + + void + GetNextIDNumbers(size_t n, IDNumbers &ids) override; + + private: + void + NextIDNumbers(size_t n, IDNumbers &ids); -private: - void NextIDNumbers(size_t n, IDNumbers& ids); - const size_t MAX_IDS_PER_MICRO = 1000; + static constexpr size_t MAX_IDS_PER_MICRO = 1000; }; // SimpleIDGenerator diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 1d83817f5d7db03479cf18b668689763a1b36be2..b18cea35e114656081c5cf7e42178db040d0856a 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -23,6 +23,7 @@ class Meta { public: using Ptr = std::shared_ptr; + virtual ~Meta() = 0; virtual Status CreateTable(TableSchema& table_schema) = 0; virtual Status DescribeTable(TableSchema& table_schema) = 0; virtual Status HasTable(const std::string& table_id, bool& has_or_not) = 0; diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index 403d5b0b594e953a491fc3e83ff6a7c0f3b73ab1..87bc1783c764ae64f80552c0c5b7621701807c04 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -12,79 +12,80 @@ #include "mysql++/mysql++.h" #include + namespace zilliz { namespace milvus { namespace engine { namespace meta { // auto StoragePrototype(const std::string& path); - using namespace mysqlpp; +using namespace mysqlpp; - class MySQLMetaImpl : public Meta { - public: - MySQLMetaImpl(const DBMetaOptions& options_, const int& mode); +class MySQLMetaImpl : public Meta { + public: + MySQLMetaImpl(const DBMetaOptions &options_, const int &mode); - virtual Status CreateTable(TableSchema& table_schema) override; - virtual Status DescribeTable(TableSchema& group_info_) override; - virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; - virtual Status AllTables(std::vector& table_schema_array) override; + Status CreateTable(TableSchema &table_schema) override; + Status DescribeTable(TableSchema &group_info_) override; + Status HasTable(const std::string &table_id, bool &has_or_not) override; + Status AllTables(std::vector &table_schema_array) override; - virtual Status DeleteTable(const std::string& table_id) override; - virtual Status DeleteTableFiles(const std::string& table_id) override; + Status DeleteTable(const std::string &table_id) override; + Status DeleteTableFiles(const std::string &table_id) override; - virtual Status CreateTableFile(TableFileSchema& file_schema) override; - virtual Status DropPartitionsByDates(const std::string& table_id, - const DatesT& dates) override; + Status CreateTableFile(TableFileSchema &file_schema) override; + Status DropPartitionsByDates(const std::string &table_id, + const DatesT &dates) override; - virtual Status GetTableFiles(const std::string& table_id, - const std::vector& ids, - TableFilesSchema& table_files) override; + Status GetTableFiles(const std::string &table_id, + const std::vector &ids, + TableFilesSchema &table_files) override; - virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override; + Status HasNonIndexFiles(const std::string &table_id, bool &has) override; - virtual Status UpdateTableFile(TableFileSchema& file_schema) override; + Status UpdateTableFile(TableFileSchema &file_schema) override; - virtual Status UpdateTableFilesToIndex(const std::string& table_id) override; + Status UpdateTableFilesToIndex(const std::string &table_id) override; - virtual Status UpdateTableFiles(TableFilesSchema& files) override; + Status UpdateTableFiles(TableFilesSchema &files) override; - virtual Status FilesToSearch(const std::string& table_id, - const DatesT& partition, - DatePartionedTableFilesSchema& files) override; + Status FilesToSearch(const std::string &table_id, + const DatesT &partition, + DatePartionedTableFilesSchema &files) override; - virtual Status FilesToMerge(const std::string& table_id, - DatePartionedTableFilesSchema& files) override; + Status FilesToMerge(const std::string &table_id, + DatePartionedTableFilesSchema &files) override; - virtual Status FilesToIndex(TableFilesSchema&) override; + Status FilesToIndex(TableFilesSchema &) override; - virtual Status Archive() override; + Status Archive() override; - virtual Status Size(uint64_t& result) override; + Status Size(uint64_t &result) override; - virtual Status CleanUp() override; + Status CleanUp() override; - virtual Status CleanUpFilesWithTTL(uint16_t seconds) override; + Status CleanUpFilesWithTTL(uint16_t seconds) override; - virtual Status DropAll() override; + Status DropAll() override; - virtual Status Count(const std::string& table_id, uint64_t& result) override; + Status Count(const std::string &table_id, uint64_t &result) override; - virtual ~MySQLMetaImpl(); + virtual ~MySQLMetaImpl(); - private: - Status NextFileId(std::string& file_id); - Status NextTableId(std::string& table_id); - Status DiscardFiles(long long to_discard_size); - Status Initialize(); + private: + Status NextFileId(std::string &file_id); + Status NextTableId(std::string &table_id); + Status DiscardFiles(long long to_discard_size); + Status Initialize(); - const DBMetaOptions options_; - const int mode_; + const DBMetaOptions options_; + const int mode_; - std::shared_ptr mysql_connection_pool_; - bool safe_grab = false; + std::shared_ptr mysql_connection_pool_; + bool safe_grab = false; // std::mutex connectionMutex_; - }; // DBMetaImpl +}; // DBMetaImpl } // namespace meta } // namespace engine diff --git a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp index 73c46942c88daa867ba53ea06ebc15fa4ae5c876..38c9463117f97ed8c7bc475c61b31bb56d652d7e 100644 --- a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp +++ b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp @@ -20,6 +20,7 @@ class ReuseCacheIndexStrategy { public: bool Schedule(const SearchContextPtr &context, std::list& task_list) { if(context == nullptr) { + ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist"; return false; } @@ -64,6 +65,7 @@ class DeleteTableStrategy { public: bool Schedule(const DeleteContextPtr &context, std::list &task_list) { if (context == nullptr) { + ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist"; return false; } @@ -103,6 +105,7 @@ public: bool TaskDispatchStrategy::Schedule(const ScheduleContextPtr &context_ptr, std::list &task_list) { if(context_ptr == nullptr) { + ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist"; return false; } diff --git a/cpp/src/db/scheduler/TaskScheduler.cpp b/cpp/src/db/scheduler/TaskScheduler.cpp index 04d6762aaeddc6045186b431f2781310aa3b8227..f29c3d4caa1db02113c5f03f447f17b314678216 100644 --- a/cpp/src/db/scheduler/TaskScheduler.cpp +++ b/cpp/src/db/scheduler/TaskScheduler.cpp @@ -31,6 +31,7 @@ TaskScheduler& TaskScheduler::GetInstance() { bool TaskScheduler::Start() { if(!stopped_) { + SERVER_LOG_INFO << "Task Scheduler isn't started"; return true; } @@ -47,6 +48,7 @@ TaskScheduler::Start() { bool TaskScheduler::Stop() { if(stopped_) { + SERVER_LOG_INFO << "Task Scheduler already stopped"; return true; } @@ -80,7 +82,7 @@ TaskScheduler::TaskDispatchWorker() { ScheduleTaskPtr task_ptr = task_dispatch_queue_.Take(); if(task_ptr == nullptr) { SERVER_LOG_INFO << "Stop db task dispatch thread"; - break;//exit + return true; } //execute task @@ -98,8 +100,8 @@ TaskScheduler::TaskWorker() { while(true) { ScheduleTaskPtr task_ptr = task_queue_.Take(); if(task_ptr == nullptr) { - SERVER_LOG_INFO << "Stop db task thread"; - break;//exit + SERVER_LOG_INFO << "Stop db task worker thread"; + return true; } //execute task