diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 5ca80f972e59c87e581165c4687cc7c7c11f2199..4dc4e562c8aab2018ddac014015f4fcaeccaa259 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -11,6 +11,7 @@ #include "Types.h" #include +#include namespace zilliz { namespace milvus { @@ -61,6 +62,8 @@ public: }; // DB +using DBPtr = std::shared_ptr; + } // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/DBFactory.cpp b/cpp/src/db/DBFactory.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b04067c38a0281beca50c9cba6d78fbe2b75bf27 --- /dev/null +++ b/cpp/src/db/DBFactory.cpp @@ -0,0 +1,37 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// + +#include "DBFactory.h" +#include "DBImpl.h" +#include "Exception.h" +#include "meta/MetaFactory.h" +#include "meta/SqliteMetaImpl.h" +#include "meta/MySQLMetaImpl.h" + +#include +#include +#include +#include +#include + +namespace zilliz { +namespace milvus { +namespace engine { + +Options DBFactory::BuildOption() { + auto meta = MetaFactory::BuildOption(); + Options options; + options.meta = meta; + return options; +} + +DBPtr DBFactory::Build(const Options& options) { + return std::make_shared(options); +} + +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/DBFactory.h b/cpp/src/db/DBFactory.h new file mode 100644 index 0000000000000000000000000000000000000000..a9404ccd1b1e3421681b65a3abc5db421d4f8a9b --- /dev/null +++ b/cpp/src/db/DBFactory.h @@ -0,0 +1,28 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#pragma once + +#include "DB.h" +#include "Options.h" + +#include +#include + +namespace zilliz { +namespace milvus { +namespace engine { + +class DBFactory { +public: + static Options BuildOption(); + + static DBPtr Build(const Options& options); +}; + + +} +} +} diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index add4b898fda0fdc6b25a1db8d961a84b3b74ed21..2924d1e7185465d924fb796986bb5002512b5e45 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -8,7 +8,8 @@ #include "Log.h" #include "Utils.h" #include "engine/EngineFactory.h" -#include "Factories.h" +#include "insert/MemMenagerFactory.h" +#include "meta/MetaFactory.h" #include "metrics/Metrics.h" #include "scheduler/TaskScheduler.h" @@ -44,7 +45,7 @@ DBImpl::DBImpl(const Options& options) shutting_down_(true), compact_thread_pool_(1, 1), index_thread_pool_(1, 1) { - meta_ptr_ = DBMetaImplFactory::Build(options.meta, options.mode); + meta_ptr_ = MetaFactory::Build(options.meta, options.mode); mem_mgr_ = MemManagerFactory::Build(meta_ptr_, options_); Start(); } diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 79a18e3ee8e56c7e4eb7a0d16f501c65b2e867c2..e4bfbd5d340cad9b6131171d19ec1dba63ae693d 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -32,8 +32,6 @@ class Meta; class DBImpl : public DB { public: - using MetaPtr = meta::Meta::Ptr; - explicit DBImpl(const Options &options); ~DBImpl(); @@ -126,8 +124,8 @@ class DBImpl : public DB { std::thread bg_timer_thread_; - MetaPtr meta_ptr_; - MemManagerAbstractPtr mem_mgr_; + meta::MetaPtr meta_ptr_; + MemManagerPtr mem_mgr_; std::mutex mem_serialize_mutex_; server::ThreadPool compact_thread_pool_; diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp deleted file mode 100644 index 58883d5c7b712b3aa5eab20933cb0cb36ee5293c..0000000000000000000000000000000000000000 --- a/cpp/src/db/Factories.cpp +++ /dev/null @@ -1,110 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// - -#include "Factories.h" -#include "DBImpl.h" -#include "src/db/insert/MemManagerImpl.h" -#include "Exception.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace zilliz { -namespace milvus { -namespace engine { - -DBMetaOptions DBMetaOptionsFactory::Build(const std::string& path) { - auto p = path; - if(p == "") { - srand(time(nullptr)); - std::stringstream ss; - ss << "/tmp/" << rand(); - p = ss.str(); - } - - DBMetaOptions meta; - meta.path = p; - return meta; -} - -Options OptionsFactory::Build() { - auto meta = DBMetaOptionsFactory::Build(); - Options options; - options.meta = meta; - return options; -} - -std::shared_ptr DBMetaImplFactory::Build() { - DBMetaOptions options = DBMetaOptionsFactory::Build(); - return std::shared_ptr(new meta::SqliteMetaImpl(options)); -} - -std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOptions, - const int& mode) { - - std::string uri = metaOptions.backend_uri; - - std::string dialectRegex = "(.*)"; - std::string usernameRegex = "(.*)"; - std::string passwordRegex = "(.*)"; - std::string hostRegex = "(.*)"; - std::string portRegex = "(.*)"; - std::string dbNameRegex = "(.*)"; - std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + - usernameRegex + "\\:" + - passwordRegex + "\\@" + - hostRegex + "\\:" + - portRegex + "\\/" + - dbNameRegex; - std::regex uriRegex(uriRegexStr); - std::smatch pieces_match; - - if (std::regex_match(uri, pieces_match, uriRegex)) { - std::string dialect = pieces_match[1].str(); - std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); - if (dialect.find("mysql") != std::string::npos) { - ENGINE_LOG_INFO << "Using MySQL"; - return std::make_shared(metaOptions, mode); - } else if (dialect.find("sqlite") != std::string::npos) { - ENGINE_LOG_INFO << "Using SQLite"; - return std::make_shared(metaOptions); - } else { - ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << dialect; - throw InvalidArgumentException("URI dialect is not mysql / sqlite"); - } - } else { - ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri; - throw InvalidArgumentException("Wrong URI format "); - } -} - -//std::shared_ptr DBFactory::Build() { -// auto options = OptionsFactory::Build(); -// auto db = DBFactory::Build(options); -// return std::shared_ptr(db); -//} - -DB* DBFactory::Build(const Options& options) { - return new DBImpl(options); -} - -MemManagerAbstractPtr MemManagerFactory::Build(const std::shared_ptr& meta, - const Options& options) { - return std::make_shared(meta, options); -} - -} // namespace engine -} // namespace milvus -} // namespace zilliz diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h deleted file mode 100644 index 3c3479e51234ea06bd96bb1b09c2297dc7963cf3..0000000000000000000000000000000000000000 --- a/cpp/src/db/Factories.h +++ /dev/null @@ -1,46 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// -#pragma once - -#include "DB.h" -#include "src/db/meta/SqliteMetaImpl.h" -#include "meta/MySQLMetaImpl.h" -#include "Options.h" -#include "src/db/insert/MemManager.h" - -#include -#include - - -namespace zilliz { -namespace milvus { -namespace engine { - -struct DBMetaOptionsFactory { - static DBMetaOptions Build(const std::string &path = ""); -}; - -struct OptionsFactory { - static Options Build(); -}; - -struct DBMetaImplFactory { - static std::shared_ptr Build(); - static std::shared_ptr Build(const DBMetaOptions &metaOptions, const int &mode); -}; - -struct DBFactory { - //static std::shared_ptr Build(); - static DB *Build(const Options &); -}; - -struct MemManagerFactory { - static MemManagerAbstractPtr Build(const std::shared_ptr &meta, const Options &options); -}; - -} // namespace engine -} // namespace milvus -} // namespace zilliz diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 45bc0d0ecdd295843e61f986f80b56f4e2a24ff9..9ddb864e0d0d76a97ea354ef5b5ec8048d648c08 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -320,8 +320,8 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { using namespace zilliz::milvus::server; ServerConfig &config = ServerConfig::GetInstance(); - ConfigNode server_config = config.GetConfig(CONFIG_SERVER); - gpu_num_ = server_config.GetInt32Value("gpu_index", 0); + ConfigNode server_config = config.GetConfig(CONFIG_DB); + gpu_num_ = server_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0); return Status::OK(); } diff --git a/cpp/src/db/insert/MemManager.h b/cpp/src/db/insert/MemManager.h index 7c91a47c7e2902735edb57e850cc7775d2464064..81fd3c1108b3d3a6a79e6c2bb529479d4d8d5687 100644 --- a/cpp/src/db/insert/MemManager.h +++ b/cpp/src/db/insert/MemManager.h @@ -1,7 +1,10 @@ #pragma once -#include +#include "db/Status.h" +#include "db/Types.h" +#include +#include namespace zilliz { namespace milvus { @@ -25,7 +28,7 @@ class MemManager { }; // MemManagerAbstract -using MemManagerAbstractPtr = std::shared_ptr; +using MemManagerPtr = std::shared_ptr; } // namespace engine } // namespace milvus diff --git a/cpp/src/db/insert/MemManagerImpl.cpp b/cpp/src/db/insert/MemManagerImpl.cpp index 7c0110e56b5f80aac783dc1f115c60b092072e10..240b2f2bb06678cdb634d90557e04a274820a755 100644 --- a/cpp/src/db/insert/MemManagerImpl.cpp +++ b/cpp/src/db/insert/MemManagerImpl.cpp @@ -10,7 +10,7 @@ namespace zilliz { namespace milvus { namespace engine { -MemManagerImpl::MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) { +MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) { auto memIt = mem_id_map_.find(table_id); if (memIt != mem_id_map_.end()) { return memIt->second; @@ -40,7 +40,7 @@ Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id, IDNumbers &vector_ids) { MemTablePtr mem = GetMemByTable(table_id); - VectorSource::Ptr source = std::make_shared(n, vectors); + VectorSourcePtr source = std::make_shared(n, vectors); auto status = mem->Add(source, vector_ids); if (status.ok()) { diff --git a/cpp/src/db/insert/MemManagerImpl.h b/cpp/src/db/insert/MemManagerImpl.h index 57b3a5f876716517860590aed00ea64075f13150..a17a90c2cd707bf221e02f696c950e5115fe2e49 100644 --- a/cpp/src/db/insert/MemManagerImpl.h +++ b/cpp/src/db/insert/MemManagerImpl.h @@ -18,11 +18,9 @@ namespace engine { class MemManagerImpl : public MemManager { public: - using MetaPtr = meta::Meta::Ptr; using Ptr = std::shared_ptr; - using MemTablePtr = typename MemTable::Ptr; - MemManagerImpl(const std::shared_ptr &meta, const Options &options) + MemManagerImpl(const meta::MetaPtr &meta, const Options &options) : meta_(meta), options_(options) {} Status InsertVectors(const std::string &table_id, @@ -49,7 +47,7 @@ class MemManagerImpl : public MemManager { using MemList = std::vector; MemIdMap mem_id_map_; MemList immu_mem_list_; - MetaPtr meta_; + meta::MetaPtr meta_; Options options_; std::mutex mutex_; std::mutex serialization_mtx_; diff --git a/cpp/src/db/insert/MemMenagerFactory.cpp b/cpp/src/db/insert/MemMenagerFactory.cpp new file mode 100644 index 0000000000000000000000000000000000000000..965ba596bd33e8c15e19913142ce54b2e51897c4 --- /dev/null +++ b/cpp/src/db/insert/MemMenagerFactory.cpp @@ -0,0 +1,30 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// + +#include "MemMenagerFactory.h" +#include "MemManagerImpl.h" +#include "db/Log.h" +#include "db/Exception.h" + +#include +#include +#include +#include +#include +#include + +namespace zilliz { +namespace milvus { +namespace engine { + +MemManagerPtr MemManagerFactory::Build(const std::shared_ptr& meta, + const Options& options) { + return std::make_shared(meta, options); +} + +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemMenagerFactory.h b/cpp/src/db/insert/MemMenagerFactory.h new file mode 100644 index 0000000000000000000000000000000000000000..f20103a0eb69ea6e52c3baf098e797cd06e8f660 --- /dev/null +++ b/cpp/src/db/insert/MemMenagerFactory.h @@ -0,0 +1,22 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#pragma once + +#include "MemManager.h" +#include "db/meta/Meta.h" + +namespace zilliz { +namespace milvus { +namespace engine { + +class MemManagerFactory { +public: + static MemManagerPtr Build(const std::shared_ptr &meta, const Options &options); +}; + +} +} +} diff --git a/cpp/src/db/insert/MemTable.cpp b/cpp/src/db/insert/MemTable.cpp index 0bb67d13870117e02fa3d09916b7a7d7e3194c9a..fe6cdf0967e62d8ef7f04f211e4950e4591ad772 100644 --- a/cpp/src/db/insert/MemTable.cpp +++ b/cpp/src/db/insert/MemTable.cpp @@ -7,7 +7,7 @@ namespace milvus { namespace engine { MemTable::MemTable(const std::string &table_id, - const std::shared_ptr &meta, + const meta::MetaPtr &meta, const Options &options) : table_id_(table_id), meta_(meta), @@ -15,18 +15,18 @@ MemTable::MemTable(const std::string &table_id, } -Status MemTable::Add(VectorSource::Ptr &source, IDNumbers &vector_ids) { +Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) { while (!source->AllAdded()) { - MemTableFile::Ptr current_mem_table_file; + MemTableFilePtr current_mem_table_file; if (!mem_table_file_list_.empty()) { current_mem_table_file = mem_table_file_list_.back(); } Status status; if (mem_table_file_list_.empty() || current_mem_table_file->IsFull()) { - MemTableFile::Ptr new_mem_table_file = std::make_shared(table_id_, meta_, options_); + MemTableFilePtr new_mem_table_file = std::make_shared(table_id_, meta_, options_); status = new_mem_table_file->Add(source, vector_ids); if (status.ok()) { mem_table_file_list_.emplace_back(new_mem_table_file); @@ -44,7 +44,7 @@ Status MemTable::Add(VectorSource::Ptr &source, IDNumbers &vector_ids) { return Status::OK(); } -void MemTable::GetCurrentMemTableFile(MemTableFile::Ptr &mem_table_file) { +void MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) { mem_table_file = mem_table_file_list_.back(); } diff --git a/cpp/src/db/insert/MemTable.h b/cpp/src/db/insert/MemTable.h index 7b2d93ffe8d2536bf04a776629fad7f1d83947a9..54744e521464cb410e25a121e52821d2032e024e 100644 --- a/cpp/src/db/insert/MemTable.h +++ b/cpp/src/db/insert/MemTable.h @@ -12,18 +12,14 @@ namespace milvus { namespace engine { class MemTable { - public: + using MemTableFileList = std::vector; - using Ptr = std::shared_ptr; - using MemTableFileList = std::vector; - using MetaPtr = meta::Meta::Ptr; - - MemTable(const std::string &table_id, const std::shared_ptr &meta, const Options &options); + MemTable(const std::string &table_id, const meta::MetaPtr &meta, const Options &options); - Status Add(VectorSource::Ptr &source, IDNumbers &vector_ids); + Status Add(VectorSourcePtr &source, IDNumbers &vector_ids); - void GetCurrentMemTableFile(MemTableFile::Ptr &mem_table_file); + void GetCurrentMemTableFile(MemTableFilePtr &mem_table_file); size_t GetTableFileCount(); @@ -40,7 +36,7 @@ class MemTable { MemTableFileList mem_table_file_list_; - MetaPtr meta_; + meta::MetaPtr meta_; Options options_; @@ -48,6 +44,8 @@ class MemTable { }; //MemTable +using MemTablePtr = std::shared_ptr; + } // namespace engine } // namespace milvus } // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 5d55bebbaeb7c9f6c9b0532c53769d03be73d1a3..946b658b4b8f5769022fc8ec5f6b0df01cba0391 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -12,7 +12,7 @@ namespace milvus { namespace engine { MemTableFile::MemTableFile(const std::string &table_id, - const std::shared_ptr &meta, + const meta::MetaPtr &meta, const Options &options) : table_id_(table_id), meta_(meta), @@ -43,7 +43,7 @@ Status MemTableFile::CreateTableFile() { return status; } -Status MemTableFile::Add(const VectorSource::Ptr &source, IDNumbers& vector_ids) { +Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) { if (table_file_schema_.dimension_ <= 0) { std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + diff --git a/cpp/src/db/insert/MemTableFile.h b/cpp/src/db/insert/MemTableFile.h index d754b030713ba8a75a7369f7be5d5e6e87e41b7e..b816e9cdddc55f267f991f3aadd09bf123260fbc 100644 --- a/cpp/src/db/insert/MemTableFile.h +++ b/cpp/src/db/insert/MemTableFile.h @@ -13,13 +13,9 @@ namespace engine { class MemTableFile { public: + MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const Options &options); - using Ptr = std::shared_ptr; - using MetaPtr = meta::Meta::Ptr; - - MemTableFile(const std::string &table_id, const std::shared_ptr &meta, const Options &options); - - Status Add(const VectorSource::Ptr &source, IDNumbers& vector_ids); + Status Add(const VectorSourcePtr &source, IDNumbers& vector_ids); size_t GetCurrentMem(); @@ -37,7 +33,7 @@ class MemTableFile { meta::TableFileSchema table_file_schema_; - MetaPtr meta_; + meta::MetaPtr meta_; Options options_; @@ -47,6 +43,8 @@ class MemTableFile { }; //MemTableFile +using MemTableFilePtr = std::shared_ptr; + } // namespace engine } // namespace milvus } // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/insert/VectorSource.h b/cpp/src/db/insert/VectorSource.h index 4c350c78bcb789ace5a91f8232d06546c5ad4360..23cce9bc2cde6d0f4f43cb33a097a04692a7f998 100644 --- a/cpp/src/db/insert/VectorSource.h +++ b/cpp/src/db/insert/VectorSource.h @@ -11,11 +11,7 @@ namespace milvus { namespace engine { class VectorSource { - public: - - using Ptr = std::shared_ptr; - VectorSource(const size_t &n, const float *vectors); Status Add(const ExecutionEnginePtr &execution_engine, @@ -42,6 +38,8 @@ class VectorSource { }; //VectorSource +using VectorSourcePtr = std::shared_ptr; + } // namespace engine } // namespace milvus } // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index b59ca3de5a6864c987f857299a922d395cc42e0a..35f5e938a0b4ff30959d9e7ce369c6f4e09bde4a 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -20,8 +20,6 @@ namespace meta { class Meta { public: - using Ptr = std::shared_ptr; - virtual ~Meta() = default; virtual Status CreateTable(TableSchema &table_schema) = 0; @@ -85,6 +83,8 @@ class Meta { }; // MetaData +using MetaPtr = std::shared_ptr; + } // namespace meta } // namespace engine } // namespace milvus diff --git a/cpp/src/db/meta/MetaFactory.cpp b/cpp/src/db/meta/MetaFactory.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9e025658aff88b505233b7ed6afdbd2ad568ee62 --- /dev/null +++ b/cpp/src/db/meta/MetaFactory.cpp @@ -0,0 +1,77 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// + +#include "MetaFactory.h" +#include "SqliteMetaImpl.h" +#include "MySQLMetaImpl.h" +#include "db/Log.h" +#include "db/Exception.h" + +#include +#include +#include +#include +#include +#include + +namespace zilliz { +namespace milvus { +namespace engine { + + DBMetaOptions MetaFactory::BuildOption(const std::string &path) { + auto p = path; + if(p == "") { + srand(time(nullptr)); + std::stringstream ss; + ss << "/tmp/" << rand(); + p = ss.str(); + } + + DBMetaOptions meta; + meta.path = p; + return meta; + } + + meta::MetaPtr MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) { + std::string uri = metaOptions.backend_uri; + + std::string dialectRegex = "(.*)"; + std::string usernameRegex = "(.*)"; + std::string passwordRegex = "(.*)"; + std::string hostRegex = "(.*)"; + std::string portRegex = "(.*)"; + std::string dbNameRegex = "(.*)"; + std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + + usernameRegex + "\\:" + + passwordRegex + "\\@" + + hostRegex + "\\:" + + portRegex + "\\/" + + dbNameRegex; + std::regex uriRegex(uriRegexStr); + std::smatch pieces_match; + + if (std::regex_match(uri, pieces_match, uriRegex)) { + std::string dialect = pieces_match[1].str(); + std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); + if (dialect.find("mysql") != std::string::npos) { + ENGINE_LOG_INFO << "Using MySQL"; + return std::make_shared(metaOptions, mode); + } else if (dialect.find("sqlite") != std::string::npos) { + ENGINE_LOG_INFO << "Using SQLite"; + return std::make_shared(metaOptions); + } else { + ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << dialect; + throw InvalidArgumentException("URI dialect is not mysql / sqlite"); + } + } else { + ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri; + throw InvalidArgumentException("Wrong URI format "); + } + } + +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MetaFactory.h b/cpp/src/db/meta/MetaFactory.h new file mode 100644 index 0000000000000000000000000000000000000000..fa964c309477a57d118388fb848089d5ea7520ad --- /dev/null +++ b/cpp/src/db/meta/MetaFactory.h @@ -0,0 +1,25 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#pragma once + +#include "Meta.h" +#include "db/Options.h" + +namespace zilliz { +namespace milvus { +namespace engine { + +class MetaFactory { +public: + static DBMetaOptions BuildOption(const std::string &path = ""); + + static meta::MetaPtr Build(const DBMetaOptions &metaOptions, const int &mode); +}; + + +} +} +} diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 7906e02b7fb7d6a21ca56488ffaaffca7144fdb0..6c89471741f34d5203da61c975fe5986d9773fd0 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -8,7 +8,6 @@ #include "db/Utils.h" #include "db/Log.h" #include "MetaConsts.h" -#include "db/Factories.h" #include "metrics/Metrics.h" #include diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index a69408c0b2b358c6bea7623636016d7498e27a50..38941abf38d231f15a64ac953c427578dffbef4a 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -8,7 +8,6 @@ #include "db/Utils.h" #include "db/Log.h" #include "MetaConsts.h" -#include "db/Factories.h" #include "metrics/Metrics.h" #include diff --git a/cpp/src/db/scheduler/context/DeleteContext.cpp b/cpp/src/db/scheduler/context/DeleteContext.cpp index 26ab02265a33c8cb805becc84a04c8ff271eaeff..9207697c5f3934ba32b6e0e024aec4560917c0be 100644 --- a/cpp/src/db/scheduler/context/DeleteContext.cpp +++ b/cpp/src/db/scheduler/context/DeleteContext.cpp @@ -11,7 +11,7 @@ namespace zilliz { namespace milvus { namespace engine { -DeleteContext::DeleteContext(const std::string &table_id, meta::Meta::Ptr &meta_ptr, uint64_t num_resource) +DeleteContext::DeleteContext(const std::string &table_id, meta::MetaPtr &meta_ptr, uint64_t num_resource) : IScheduleContext(ScheduleContextType::kDelete), table_id_(table_id), meta_ptr_(meta_ptr), diff --git a/cpp/src/db/scheduler/context/DeleteContext.h b/cpp/src/db/scheduler/context/DeleteContext.h index e9ff2f1be2f9d7c2a724516ed5ca29cf4930ae73..6c21d1e65a02bcbafc9994800179badca6d704a3 100644 --- a/cpp/src/db/scheduler/context/DeleteContext.h +++ b/cpp/src/db/scheduler/context/DeleteContext.h @@ -16,16 +16,16 @@ namespace engine { class DeleteContext : public IScheduleContext { public: - DeleteContext(const std::string& table_id, meta::Meta::Ptr& meta_ptr, uint64_t num_resource); + DeleteContext(const std::string& table_id, meta::MetaPtr& meta_ptr, uint64_t num_resource); std::string table_id() const { return table_id_; } - meta::Meta::Ptr meta() const { return meta_ptr_; } + meta::MetaPtr meta() const { return meta_ptr_; } void WaitAndDelete(); void ResourceDone(); private: std::string table_id_; - meta::Meta::Ptr meta_ptr_; + meta::MetaPtr meta_ptr_; uint64_t num_resource_; uint64_t done_resource = 0; diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 305c5ba0ef13e67f0237061462466831479f9071..25f57419b897af8ffd2d0dffa654cf4f3e6d24cb 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -6,7 +6,7 @@ #include "DBWrapper.h" #include "ServerConfig.h" -#include "db/Factories.h" +#include "db/DBFactory.h" #include "utils/CommonUtil.h" #include "utils/Log.h" #include "utils/StringHelpFunctions.h" @@ -96,8 +96,7 @@ ErrorCode DBWrapper::StartService() { //create db instance std::string msg = opt.meta.path; try { - engine::DB* db = engine::DBFactory::Build(opt); - db_.reset(db); + db_ = engine::DBFactory::Build(opt); } catch(std::exception& ex) { msg = ex.what(); } diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index 6d0d3f77aaddaa1577e25c2ba7f49caa1382587d..b47b18b1f7c8b6e21555ed144f738054a0a1deb5 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -25,19 +25,19 @@ public: return wrapper; } - static std::shared_ptr DB() { + static engine::DBPtr DB() { return GetInstance().EngineDB(); } ErrorCode StartService(); ErrorCode StopService(); - std::shared_ptr EngineDB() { + engine::DBPtr EngineDB() { return db_; } private: - std::shared_ptr db_; + engine::DBPtr db_; }; } diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 91b68210719f98a4ab95d0590bef18e04a10735e..bf177475a7a206425e13340bb3c4c0780a23b1f2 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -7,7 +7,7 @@ #include "db/DB.h" #include "db/DBImpl.h" #include "db/meta/MetaConsts.h" -#include "db/Factories.h" +#include "db/DBFactory.h" #include "cache/CpuCacheMgr.h" #include "utils/CommonUtil.h" @@ -467,11 +467,6 @@ TEST_F(DBTest2, DELETE_TEST) { }; TEST_F(DBTest2, DELETE_BY_RANGE_TEST) { - auto options = engine::OptionsFactory::Build(); - options.meta.path = "/tmp/milvus_test"; - options.meta.backend_uri = "sqlite://:@:/"; - auto db_ = engine::DBFactory::Build(options); - engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 1429038a2ba216347f3eeb8c0cb16c3e5b8a7090..2760711df32e3b7d8ab2e4d75466830ebf1f62b3 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -3,7 +3,6 @@ #include "db/insert/VectorSource.h" #include "db/insert/MemTableFile.h" #include "db/insert/MemTable.h" -#include "db/Factories.h" #include "db/Constants.h" #include "db/engine/EngineFactory.h" #include "db/meta/MetaConsts.h" @@ -106,7 +105,7 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) { std::vector vectors_100; BuildVectors(n_100, vectors_100); - engine::VectorSource::Ptr source = std::make_shared(n_100, vectors_100.data()); + engine::VectorSourcePtr source = std::make_shared(n_100, vectors_100.data()); engine::IDNumbers vector_ids; status = mem_table_file.Add(source, vector_ids); @@ -124,7 +123,7 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) { std::vector vectors_128M; BuildVectors(n_max, vectors_128M); - engine::VectorSource::Ptr source_128M = std::make_shared(n_max, vectors_128M.data()); + engine::VectorSourcePtr source_128M = std::make_shared(n_max, vectors_128M.data()); vector_ids.clear(); status = mem_table_file.Add(source_128M, vector_ids); @@ -145,7 +144,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { std::vector vectors_100; BuildVectors(n_100, vectors_100); - engine::VectorSource::Ptr source_100 = std::make_shared(n_100, vectors_100.data()); + engine::VectorSourcePtr source_100 = std::make_shared(n_100, vectors_100.data()); engine::MemTable mem_table(TABLE_NAME, impl_, options); @@ -155,7 +154,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { vector_ids = source_100->GetVectorIds(); ASSERT_EQ(vector_ids.size(), 100); - engine::MemTableFile::Ptr mem_table_file; + engine::MemTableFilePtr mem_table_file; mem_table.GetCurrentMemTableFile(mem_table_file); size_t singleVectorMem = sizeof(float) * TABLE_DIM; ASSERT_EQ(mem_table_file->GetCurrentMem(), n_100 * singleVectorMem); @@ -165,7 +164,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { BuildVectors(n_max, vectors_128M); vector_ids.clear(); - engine::VectorSource::Ptr source_128M = std::make_shared(n_max, vectors_128M.data()); + engine::VectorSourcePtr source_128M = std::make_shared(n_max, vectors_128M.data()); status = mem_table.Add(source_128M, vector_ids); ASSERT_TRUE(status.ok()); @@ -181,7 +180,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { std::vector vectors_1G; BuildVectors(n_1G, vectors_1G); - engine::VectorSource::Ptr source_1G = std::make_shared(n_1G, vectors_1G.data()); + engine::VectorSourcePtr source_1G = std::make_shared(n_1G, vectors_1G.data()); vector_ids.clear(); status = mem_table.Add(source_1G, vector_ids); diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 60b6eab310ce45c21672b6bf7a26a38b466c00fd..b23a634b7b290bbac2359dd0ae9aaaeedac20e2d 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -11,7 +11,6 @@ #include "utils.h" #include "db/meta/SqliteMetaImpl.h" -#include "db/Factories.h" #include "db/Utils.h" #include "db/meta/MetaConsts.h" diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 677b310a600ab8ff6fbbb961e96198dd19ba1bb9..79925c9235fa1f962011042fcf0f197e0413c619 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -7,7 +7,6 @@ #include "db/DB.h" #include "db/DBImpl.h" #include "db/meta/MetaConsts.h" -#include "db/Factories.h" #include #include diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index ede879c3bae6f971f22d08d2032c3565cc13f473..1db24652dbcc2870ee7480c8e57909b567270ae9 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -11,7 +11,6 @@ #include "utils.h" #include "db/meta/MySQLMetaImpl.h" -#include "db/Factories.h" #include "db/Utils.h" #include "db/meta/MetaConsts.h" diff --git a/cpp/unittest/db/scheduler_test.cpp b/cpp/unittest/db/scheduler_test.cpp index dfb90fc3dfa186765d602fc6cfaf16b155167d4d..dc23623dc16151e71d67cf6fd8150326c28e7c51 100644 --- a/cpp/unittest/db/scheduler_test.cpp +++ b/cpp/unittest/db/scheduler_test.cpp @@ -99,7 +99,7 @@ TEST(DBSchedulerTest, DELETE_SCHEDULER_TEST) { task_list.push_back(task_ptr); } - engine::meta::Meta::Ptr meta_ptr; + engine::meta::MetaPtr meta_ptr; engine::DeleteContextPtr context_ptr = std::make_shared(table_id, meta_ptr, 0); ret = engine::TaskDispatchStrategy::Schedule(context_ptr, task_list); ASSERT_TRUE(ret); diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 873028b65a8837618acab7a02e1b8658c761dd67..47f21d5d7abddca5d66fe9729ecc950782ffeb3e 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -5,12 +5,11 @@ //////////////////////////////////////////////////////////////////////////////// #include -#include #include #include #include "utils.h" -#include "db/Factories.h" +#include "db/DBFactory.h" #include "db/Options.h" #include "server/ServerConfig.h" #include "knowhere/index/vector_index/gpu_ivf.h" @@ -56,7 +55,7 @@ void BaseTest::TearDown() { } engine::Options BaseTest::GetOptions() { - auto options = engine::OptionsFactory::Build(); + auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.backend_uri = "sqlite://:@:/"; return options; @@ -86,7 +85,6 @@ void DBTest::SetUp() { void DBTest::TearDown() { db_->Stop(); db_->DropAll(); - delete db_; BaseTest::TearDown(); @@ -99,7 +97,7 @@ void DBTest::TearDown() { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// engine::Options DBTest2::GetOptions() { - auto options = engine::OptionsFactory::Build(); + auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); options.meta.backend_uri = "sqlite://:@:/"; @@ -125,7 +123,7 @@ void MetaTest::TearDown() { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// engine::Options MySqlDBTest::GetOptions() { - auto options = engine::OptionsFactory::Build(); + auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.backend_uri = DBTestEnvironment::getURI(); @@ -154,7 +152,7 @@ void MySqlMetaTest::TearDown() { } zilliz::milvus::engine::Options MySqlMetaTest::GetOptions() { - auto options = engine::OptionsFactory::Build(); + auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.backend_uri = DBTestEnvironment::getURI(); diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index 8cb006ecfc135f02c15edbc15232f1195d1a171e..5d8cca3bd17631260272ab1985f0c1915d83b6a1 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -44,7 +44,7 @@ protected: ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DBTest : public BaseTest { protected: - zilliz::milvus::engine::DB *db_; + zilliz::milvus::engine::DBPtr db_; virtual void SetUp() override; virtual void TearDown() override; diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index edbbb5803572844136aaa2dfda82ddc1cd446b81..dc9469a18cc3fec7dbaf47b0e60c8e93ffbd225d 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -18,7 +18,6 @@ #include "utils.h" #include "db/DB.h" #include "db/meta/SqliteMetaImpl.h" -#include "db/Factories.h" using namespace zilliz::milvus; diff --git a/cpp/unittest/metrics/utils.cpp b/cpp/unittest/metrics/utils.cpp index 094b911da84dd787cdf15536c1ecfe093e4b490c..307572e9e84d79a5511220e89c15d154350f76dc 100644 --- a/cpp/unittest/metrics/utils.cpp +++ b/cpp/unittest/metrics/utils.cpp @@ -5,13 +5,11 @@ //////////////////////////////////////////////////////////////////////////////// #include -#include #include #include #include "utils.h" -#include "db/Factories.h" -#include "db/Options.h" +#include "db/DBFactory.h" INITIALIZE_EASYLOGGINGPP @@ -43,7 +41,7 @@ void MetricTest::InitLog() { } engine::Options MetricTest::GetOptions() { - auto options = engine::OptionsFactory::Build(); + auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.backend_uri = "sqlite://:@:/"; return options; @@ -56,7 +54,7 @@ void MetricTest::SetUp() { } void MetricTest::TearDown() { - delete db_; + db_->Stop(); boost::filesystem::remove_all("/tmp/milvus_test"); } diff --git a/cpp/unittest/metrics/utils.h b/cpp/unittest/metrics/utils.h index 4980b91a1ec713866e0be45278c4c14e017a711f..73e7310094a01a5747027489ca9f5cb70761e011 100644 --- a/cpp/unittest/metrics/utils.h +++ b/cpp/unittest/metrics/utils.h @@ -55,7 +55,7 @@ void ASSERT_STATS(zilliz::milvus::engine::Status& stat); class MetricTest : public ::testing::Test { protected: - zilliz::milvus::engine::DB* db_; + zilliz::milvus::engine::DBPtr db_; void InitLog(); virtual void SetUp() override;