diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 2e6ea844857cadc56a90755ff4f521db5d50a89d..6fae123644c7578b96ac681b5f468f34ad7580e1 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -9,8 +9,10 @@ aux_source_directory(cache cache_files) aux_source_directory(config config_files) aux_source_directory(server server_files) aux_source_directory(utils utils_files) -aux_source_directory(db db_files) -#aux_source_directory(wrapper wrapper_files) +aux_source_directory(db db_main_files) +aux_source_directory(db/engine db_engine_files) +aux_source_directory(db/insert db_insert_files) +aux_source_directory(db/meta db_meta_files) aux_source_directory(metrics metrics_files) aux_source_directory(wrapper/knowhere knowhere_files) @@ -33,12 +35,14 @@ set(service_files server/ThreadPoolServer.h ) -set(engine_files +set(db_files ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp ${cache_files} - ${db_files} + ${db_main_files} + ${db_engine_files} + ${db_insert_files} + ${db_meta_files} ${db_scheduler_files} - ${wrapper_files} ${metrics_files} ${knowhere_files} ) @@ -126,7 +130,7 @@ if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") endif () -cuda_add_library(milvus_engine STATIC ${engine_files}) +cuda_add_library(milvus_engine STATIC ${db_files}) target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs}) add_library(metrics STATIC ${metrics_files}) diff --git a/cpp/src/db/DB.cpp b/cpp/src/db/DB.cpp index 3b4f03d247da8bea5b6641d351baadbc6fb6fecf..2d922dd33b26cdbd5aac6b3e817e4a36d096319a 100644 --- a/cpp/src/db/DB.cpp +++ b/cpp/src/db/DB.cpp @@ -5,7 +5,6 @@ ******************************************************************************/ #include "DBImpl.h" -#include "DBMetaImpl.h" #include "Factories.h" namespace zilliz { diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 3bfd2281e2268d8ae8e6a9d18ed5f0e9247d8bed..f1d3a77a8a7ad28315a87cc158d6dd11e1852f5c 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -6,7 +6,7 @@ #pragma once #include "Options.h" -#include "Meta.h" +#include "meta/Meta.h" #include "Status.h" #include "Types.h" diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 17537556b5b1a5e4f905ca6bed52dac0cb13b9ee..5df588bda4127fd36b1fa79b7387a7230f540436 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -4,16 +4,16 @@ * Proprietary and confidential. ******************************************************************************/ #include "DBImpl.h" -#include "DBMetaImpl.h" +#include "src/db/meta/SqliteMetaImpl.h" #include "Log.h" -#include "EngineFactory.h" +#include "engine/EngineFactory.h" #include "Factories.h" #include "metrics/Metrics.h" #include "scheduler/TaskScheduler.h" #include "scheduler/context/DeleteContext.h" #include "utils/TimeRecorder.h" -#include "MetaConsts.h" +#include "meta/MetaConsts.h" #include #include diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 27b0b66ffaca861befe61e991e3bb49344fe5d51..ccd7f541bdb80111976e285dfc98846bf96d80c9 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -8,7 +8,7 @@ #include "DB.h" #include "Types.h" #include "utils/ThreadPool.h" -#include "MemManagerAbstract.h" +#include "src/db/insert/MemManager.h" #include #include diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 05a1ad733831426771870d9fb2a11dc787696420..bb1056e3c22003b699ecf02a924e7759633fe575 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -6,7 +6,7 @@ #include "Factories.h" #include "DBImpl.h" -#include "NewMemManager.h" +#include "src/db/insert/MemManagerImpl.h" #include "Exception.h" #include @@ -46,9 +46,9 @@ Options OptionsFactory::Build() { return options; } -std::shared_ptr DBMetaImplFactory::Build() { +std::shared_ptr DBMetaImplFactory::Build() { DBMetaOptions options = DBMetaOptionsFactory::Build(); - return std::shared_ptr(new meta::DBMetaImpl(options)); + return std::shared_ptr(new meta::SqliteMetaImpl(options)); } std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOptions, @@ -79,7 +79,7 @@ std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOp return std::make_shared(metaOptions, mode); } else if (dialect.find("sqlite") != std::string::npos) { ENGINE_LOG_INFO << "Using SQLite"; - return std::make_shared(metaOptions); + return std::make_shared(metaOptions); } else { ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << dialect; throw InvalidArgumentException("URI dialect is not mysql / sqlite"); @@ -102,7 +102,7 @@ DB* DBFactory::Build(const Options& options) { MemManagerAbstractPtr MemManagerFactory::Build(const std::shared_ptr& meta, const Options& options) { - return std::make_shared(meta, options); + return std::make_shared(meta, options); } } // namespace engine diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index 8b6e7b100f6fa86863508091c1dc07b3969199c8..0e6823c385c62187a43b41fb9c2333f8a45a2b17 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -6,11 +6,10 @@ #pragma once #include "DB.h" -#include "DBMetaImpl.h" -#include "MySQLMetaImpl.h" +#include "src/db/meta/SqliteMetaImpl.h" +#include "meta/MySQLMetaImpl.h" #include "Options.h" -#include "ExecutionEngine.h" -#include "MemManagerAbstract.h" +#include "src/db/insert/MemManager.h" #include #include @@ -29,7 +28,7 @@ struct OptionsFactory { }; struct DBMetaImplFactory { - static std::shared_ptr Build(); + static std::shared_ptr Build(); static std::shared_ptr Build(const DBMetaOptions &metaOptions, const int &mode); }; diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index 81f97978eaaaef515f2fb98aa6111de98a139a2c..287f9f074d35f90844d90ae46c89026ef27739c9 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -9,7 +9,7 @@ #include #include "Options.h" -#include "DBMetaImpl.h" +#include "db/meta/SqliteMetaImpl.h" #include "Exception.h" namespace zilliz { diff --git a/cpp/src/db/Utils.h b/cpp/src/db/Utils.h index a5a0f8ff8c278827ac60ebb4ed117d741cbe7788..8329f5a1fcab00288d5e98d183d96e94f6850ef2 100644 --- a/cpp/src/db/Utils.h +++ b/cpp/src/db/Utils.h @@ -6,7 +6,7 @@ #pragma once #include "Options.h" -#include "MetaTypes.h" +#include "db/meta/MetaTypes.h" #include diff --git a/cpp/src/db/EngineFactory.cpp b/cpp/src/db/engine/EngineFactory.cpp similarity index 99% rename from cpp/src/db/EngineFactory.cpp rename to cpp/src/db/engine/EngineFactory.cpp index 32e10beb8ddb5d166ec1fb40a697a69168101e37..d09e9f8b973a557c6015990279ff16cca190b5e5 100644 --- a/cpp/src/db/EngineFactory.cpp +++ b/cpp/src/db/engine/EngineFactory.cpp @@ -6,7 +6,7 @@ #include "EngineFactory.h" //#include "FaissExecutionEngine.h" #include "ExecutionEngineImpl.h" -#include "Log.h" +#include "db/Log.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/db/EngineFactory.h b/cpp/src/db/engine/EngineFactory.h similarity index 96% rename from cpp/src/db/EngineFactory.h rename to cpp/src/db/engine/EngineFactory.h index 9d667bd5aa0d723a905a14dfd88e1d8ec1b8be00..d8c35468daa22f9e5ccfec5b0773fa8335ef85f8 100644 --- a/cpp/src/db/EngineFactory.h +++ b/cpp/src/db/engine/EngineFactory.h @@ -5,7 +5,7 @@ ******************************************************************************/ #pragma once -#include "Status.h" +#include "db/Status.h" #include "ExecutionEngine.h" namespace zilliz { diff --git a/cpp/src/db/ExecutionEngine.cpp b/cpp/src/db/engine/ExecutionEngine.cpp similarity index 100% rename from cpp/src/db/ExecutionEngine.cpp rename to cpp/src/db/engine/ExecutionEngine.cpp diff --git a/cpp/src/db/ExecutionEngine.h b/cpp/src/db/engine/ExecutionEngine.h similarity index 98% rename from cpp/src/db/ExecutionEngine.h rename to cpp/src/db/engine/ExecutionEngine.h index f942e1faa6ad0c0995a3b829bc66f58c397e0fbd..5871be7f57b986ae90d841558fddf75eed887a38 100644 --- a/cpp/src/db/ExecutionEngine.h +++ b/cpp/src/db/engine/ExecutionEngine.h @@ -5,7 +5,7 @@ ******************************************************************************/ #pragma once -#include "Status.h" +#include "db/Status.h" #include #include diff --git a/cpp/src/db/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp similarity index 98% rename from cpp/src/db/ExecutionEngineImpl.cpp rename to cpp/src/db/engine/ExecutionEngineImpl.cpp index 1164116f38c3c30755a1591caf39a63d42975c53..21878377e66010c5c97ff7bd22a453035df790e6 100644 --- a/cpp/src/db/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -5,9 +5,9 @@ ******************************************************************************/ #include -#include -#include -#include "Log.h" +#include "src/server/ServerConfig.h" +#include "src/metrics/Metrics.h" +#include "db/Log.h" #include "utils/CommonUtil.h" #include "src/cache/CpuCacheMgr.h" @@ -15,7 +15,7 @@ #include "wrapper/knowhere/vec_index.h" #include "wrapper/knowhere/vec_impl.h" #include "knowhere/common/exception.h" -#include "Exception.h" +#include "db/Exception.h" namespace zilliz { diff --git a/cpp/src/db/ExecutionEngineImpl.h b/cpp/src/db/engine/ExecutionEngineImpl.h similarity index 100% rename from cpp/src/db/ExecutionEngineImpl.h rename to cpp/src/db/engine/ExecutionEngineImpl.h diff --git a/cpp/src/db/MemManagerAbstract.h b/cpp/src/db/insert/MemManager.h similarity index 83% rename from cpp/src/db/MemManagerAbstract.h rename to cpp/src/db/insert/MemManager.h index 943c454e465bda005a2e28d44c6b95708dffacc4..7c91a47c7e2902735edb57e850cc7775d2464064 100644 --- a/cpp/src/db/MemManagerAbstract.h +++ b/cpp/src/db/insert/MemManager.h @@ -7,7 +7,7 @@ namespace zilliz { namespace milvus { namespace engine { -class MemManagerAbstract { +class MemManager { public: virtual Status InsertVectors(const std::string &table_id, @@ -25,7 +25,7 @@ class MemManagerAbstract { }; // MemManagerAbstract -using MemManagerAbstractPtr = std::shared_ptr; +using MemManagerAbstractPtr = std::shared_ptr; } // namespace engine } // namespace milvus diff --git a/cpp/src/db/NewMemManager.cpp b/cpp/src/db/insert/MemManagerImpl.cpp similarity index 81% rename from cpp/src/db/NewMemManager.cpp rename to cpp/src/db/insert/MemManagerImpl.cpp index c7291354ad2c133ad81ac88999377b0e1d7efebf..bd71f874da9a0e4b4fbe6c07c9e6c7b73b2f7286 100644 --- a/cpp/src/db/NewMemManager.cpp +++ b/cpp/src/db/insert/MemManagerImpl.cpp @@ -1,7 +1,7 @@ -#include "NewMemManager.h" +#include "MemManagerImpl.h" #include "VectorSource.h" -#include "Log.h" -#include "Constants.h" +#include "db/Log.h" +#include "db/Constants.h" #include @@ -10,7 +10,7 @@ namespace zilliz { namespace milvus { namespace engine { -NewMemManager::MemTablePtr NewMemManager::GetMemByTable(const std::string &table_id) { +MemManagerImpl::MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) { auto memIt = mem_id_map_.find(table_id); if (memIt != mem_id_map_.end()) { return memIt->second; @@ -20,7 +20,7 @@ NewMemManager::MemTablePtr NewMemManager::GetMemByTable(const std::string &table return mem_id_map_[table_id]; } -Status NewMemManager::InsertVectors(const std::string &table_id_, +Status MemManagerImpl::InsertVectors(const std::string &table_id_, size_t n_, const float *vectors_, IDNumbers &vector_ids_) { @@ -34,7 +34,7 @@ Status NewMemManager::InsertVectors(const std::string &table_id_, return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); } -Status NewMemManager::InsertVectorsNoLock(const std::string &table_id, +Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id, size_t n, const float *vectors, IDNumbers &vector_ids) { @@ -49,7 +49,7 @@ Status NewMemManager::InsertVectorsNoLock(const std::string &table_id, return status; } -Status NewMemManager::ToImmutable() { +Status MemManagerImpl::ToImmutable() { std::unique_lock lock(mutex_); MemIdMap temp_map; for (auto &kv: mem_id_map_) { @@ -65,7 +65,7 @@ Status NewMemManager::ToImmutable() { return Status::OK(); } -Status NewMemManager::Serialize(std::set &table_ids) { +Status MemManagerImpl::Serialize(std::set &table_ids) { ToImmutable(); std::unique_lock lock(serialization_mtx_); table_ids.clear(); @@ -77,7 +77,7 @@ Status NewMemManager::Serialize(std::set &table_ids) { return Status::OK(); } -Status NewMemManager::EraseMemVector(const std::string &table_id) { +Status MemManagerImpl::EraseMemVector(const std::string &table_id) { {//erase MemVector from rapid-insert cache std::unique_lock lock(mutex_); mem_id_map_.erase(table_id); @@ -97,7 +97,7 @@ Status NewMemManager::EraseMemVector(const std::string &table_id) { return Status::OK(); } -size_t NewMemManager::GetCurrentMutableMem() { +size_t MemManagerImpl::GetCurrentMutableMem() { size_t total_mem = 0; for (auto &kv : mem_id_map_) { auto memTable = kv.second; @@ -106,7 +106,7 @@ size_t NewMemManager::GetCurrentMutableMem() { return total_mem; } -size_t NewMemManager::GetCurrentImmutableMem() { +size_t MemManagerImpl::GetCurrentImmutableMem() { size_t total_mem = 0; for (auto &mem_table : immu_mem_list_) { total_mem += mem_table->GetCurrentMem(); @@ -114,7 +114,7 @@ size_t NewMemManager::GetCurrentImmutableMem() { return total_mem; } -size_t NewMemManager::GetCurrentMem() { +size_t MemManagerImpl::GetCurrentMem() { return GetCurrentMutableMem() + GetCurrentImmutableMem(); } diff --git a/cpp/src/db/NewMemManager.h b/cpp/src/db/insert/MemManagerImpl.h similarity index 82% rename from cpp/src/db/NewMemManager.h rename to cpp/src/db/insert/MemManagerImpl.h index 5b933c94ca39a2bb6018858a0c90806eeaaa53ad..57b3a5f876716517860590aed00ea64075f13150 100644 --- a/cpp/src/db/NewMemManager.h +++ b/cpp/src/db/insert/MemManagerImpl.h @@ -1,9 +1,9 @@ #pragma once -#include "Meta.h" +#include "db/meta/Meta.h" #include "MemTable.h" -#include "Status.h" -#include "MemManagerAbstract.h" +#include "db/Status.h" +#include "MemManager.h" #include #include @@ -16,13 +16,13 @@ namespace zilliz { namespace milvus { namespace engine { -class NewMemManager : public MemManagerAbstract { +class MemManagerImpl : public MemManager { public: using MetaPtr = meta::Meta::Ptr; - using Ptr = std::shared_ptr; + using Ptr = std::shared_ptr; using MemTablePtr = typename MemTable::Ptr; - NewMemManager(const std::shared_ptr &meta, const Options &options) + MemManagerImpl(const std::shared_ptr &meta, const Options &options) : meta_(meta), options_(options) {} Status InsertVectors(const std::string &table_id, diff --git a/cpp/src/db/MemTable.cpp b/cpp/src/db/insert/MemTable.cpp similarity index 98% rename from cpp/src/db/MemTable.cpp rename to cpp/src/db/insert/MemTable.cpp index e05aa058ac7f23cb1691736b1ecdf8118a54639d..38206e25fd8615903cdae774b394402240733a0d 100644 --- a/cpp/src/db/MemTable.cpp +++ b/cpp/src/db/insert/MemTable.cpp @@ -1,5 +1,5 @@ #include "MemTable.h" -#include "Log.h" +#include "db/Log.h" namespace zilliz { diff --git a/cpp/src/db/MemTable.h b/cpp/src/db/insert/MemTable.h similarity index 95% rename from cpp/src/db/MemTable.h rename to cpp/src/db/insert/MemTable.h index 198fcc228ab9b99b7c7a31781a30ba442f8d595a..4f0cdb7d2a113fd75ad0a3f34ba9dd0d435c2088 100644 --- a/cpp/src/db/MemTable.h +++ b/cpp/src/db/insert/MemTable.h @@ -1,6 +1,6 @@ #pragma once -#include "Status.h" +#include "db/Status.h" #include "MemTableFile.h" #include "VectorSource.h" diff --git a/cpp/src/db/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp similarity index 97% rename from cpp/src/db/MemTableFile.cpp rename to cpp/src/db/insert/MemTableFile.cpp index f36223f097bb1a9df472e3fc167efc16a1c2761d..1d7053ab5a2c9b33e111dd74aaf52c8a6f9da84b 100644 --- a/cpp/src/db/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -1,7 +1,7 @@ #include "MemTableFile.h" -#include "Constants.h" -#include "Log.h" -#include "EngineFactory.h" +#include "db/Constants.h" +#include "db/Log.h" +#include "db/engine/EngineFactory.h" #include "metrics/Metrics.h" #include diff --git a/cpp/src/db/MemTableFile.h b/cpp/src/db/insert/MemTableFile.h similarity index 87% rename from cpp/src/db/MemTableFile.h rename to cpp/src/db/insert/MemTableFile.h index 4d0011b36233f769e7aaf945a3fa0b5bb54c86a2..b582152299fb90e64a75d10c03404ab941d966f7 100644 --- a/cpp/src/db/MemTableFile.h +++ b/cpp/src/db/insert/MemTableFile.h @@ -1,9 +1,9 @@ #pragma once -#include "Status.h" -#include "Meta.h" +#include "db/Status.h" +#include "db/meta/Meta.h" #include "VectorSource.h" -#include "ExecutionEngine.h" +#include "db/engine/ExecutionEngine.h" namespace zilliz { diff --git a/cpp/src/db/VectorSource.cpp b/cpp/src/db/insert/VectorSource.cpp similarity index 94% rename from cpp/src/db/VectorSource.cpp rename to cpp/src/db/insert/VectorSource.cpp index 74c07ae1f67a54e4301474a2d350091ce196de3b..f36eeb09bd4e5ae8d01fac1d429b875c71c5674c 100644 --- a/cpp/src/db/VectorSource.cpp +++ b/cpp/src/db/insert/VectorSource.cpp @@ -1,7 +1,7 @@ #include "VectorSource.h" -#include "ExecutionEngine.h" -#include "EngineFactory.h" -#include "Log.h" +#include "db/engine/ExecutionEngine.h" +#include "db/engine/EngineFactory.h" +#include "db/Log.h" #include "metrics/Metrics.h" diff --git a/cpp/src/db/VectorSource.h b/cpp/src/db/insert/VectorSource.h similarity index 84% rename from cpp/src/db/VectorSource.h rename to cpp/src/db/insert/VectorSource.h index 7092805a6d79de741de24c078ad5e02cdfa4d15d..9792772d800ac77931f5a797ce5bea59ea7b96be 100644 --- a/cpp/src/db/VectorSource.h +++ b/cpp/src/db/insert/VectorSource.h @@ -1,9 +1,9 @@ #pragma once -#include "Status.h" -#include "Meta.h" -#include "IDGenerator.h" -#include "ExecutionEngine.h" +#include "db/Status.h" +#include "db/meta/Meta.h" +#include "db/IDGenerator.h" +#include "db/engine/ExecutionEngine.h" namespace zilliz { diff --git a/cpp/src/db/Meta.cpp b/cpp/src/db/meta/Meta.cpp similarity index 100% rename from cpp/src/db/Meta.cpp rename to cpp/src/db/meta/Meta.cpp diff --git a/cpp/src/db/Meta.h b/cpp/src/db/meta/Meta.h similarity index 98% rename from cpp/src/db/Meta.h rename to cpp/src/db/meta/Meta.h index 7e826f63354599a161444ee644a5fb66e6e9861f..e88761b4460613cf8899453c19178c0fb9c04099 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -6,8 +6,8 @@ #pragma once #include "MetaTypes.h" -#include "Options.h" -#include "Status.h" +#include "db/Options.h" +#include "db/Status.h" #include #include diff --git a/cpp/src/db/MetaConsts.h b/cpp/src/db/meta/MetaConsts.h similarity index 100% rename from cpp/src/db/MetaConsts.h rename to cpp/src/db/meta/MetaConsts.h diff --git a/cpp/src/db/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h similarity index 97% rename from cpp/src/db/MetaTypes.h rename to cpp/src/db/meta/MetaTypes.h index d7039685e3afb921aa3a5856f19e6e2b78a6708a..9f58734f39d8ce77a0343756dd34ad1026d7fa5a 100644 --- a/cpp/src/db/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -5,7 +5,7 @@ ******************************************************************************/ #pragma once -#include "ExecutionEngine.h" +#include "db/engine/ExecutionEngine.h" #include #include diff --git a/cpp/src/db/MySQLConnectionPool.cpp b/cpp/src/db/meta/MySQLConnectionPool.cpp similarity index 100% rename from cpp/src/db/MySQLConnectionPool.cpp rename to cpp/src/db/meta/MySQLConnectionPool.cpp diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/meta/MySQLConnectionPool.h similarity index 97% rename from cpp/src/db/MySQLConnectionPool.h rename to cpp/src/db/meta/MySQLConnectionPool.h index 5112993b94fd18073f76a2cbcd4d16eff307258d..62afd2ddbf6fa650c16c8b9a49d9c466507afae1 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/meta/MySQLConnectionPool.h @@ -4,7 +4,7 @@ #include #include -#include "Log.h" +#include "db/Log.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp similarity index 99% rename from cpp/src/db/MySQLMetaImpl.cpp rename to cpp/src/db/meta/MySQLMetaImpl.cpp index e9835f24b6720c47d14d51b359e3467f5e23a7af..bf4589351ecaece83406d8989890ecd32c80ae8d 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -4,11 +4,11 @@ * Proprietary and confidential. ******************************************************************************/ #include "MySQLMetaImpl.h" -#include "IDGenerator.h" -#include "Utils.h" -#include "Log.h" +#include "db/IDGenerator.h" +#include "db/Utils.h" +#include "db/Log.h" #include "MetaConsts.h" -#include "Factories.h" +#include "db/Factories.h" #include "metrics/Metrics.h" #include diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h similarity index 99% rename from cpp/src/db/MySQLMetaImpl.h rename to cpp/src/db/meta/MySQLMetaImpl.h index 7822b99f6488e7af70bc2ab58cab401150f2cfd7..30695423ddd39a1ab4f5d63edacf43e5a22ad319 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -6,7 +6,7 @@ #pragma once #include "Meta.h" -#include "Options.h" +#include "db/Options.h" #include "MySQLConnectionPool.h" #include "mysql++/mysql++.h" diff --git a/cpp/src/db/meta/DBMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp similarity index 95% rename from cpp/src/db/meta/DBMetaImpl.cpp rename to cpp/src/db/meta/SqliteMetaImpl.cpp index c629f378161020a9eb680911a61b4dca0efe44ec..9411a5e82b0a88eccbc0351d275e334dfa5ad8b3 100644 --- a/cpp/src/db/meta/DBMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -3,12 +3,12 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ -#include "DBMetaImpl.h" -#include "IDGenerator.h" -#include "Utils.h" -#include "Log.h" +#include "SqliteMetaImpl.h" +#include "db/IDGenerator.h" +#include "db/Utils.h" +#include "db/Log.h" #include "MetaConsts.h" -#include "Factories.h" +#include "db/Factories.h" #include "metrics/Metrics.h" #include @@ -83,7 +83,7 @@ using ConnectorT = decltype(StoragePrototype("")); static std::unique_ptr ConnectorPtr; using ConditionT = decltype(c(&TableFileSchema::id_) == 1UL); -Status DBMetaImpl::NextTableId(std::string &table_id) { +Status SqliteMetaImpl::NextTableId(std::string &table_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -91,7 +91,7 @@ Status DBMetaImpl::NextTableId(std::string &table_id) { return Status::OK(); } -Status DBMetaImpl::NextFileId(std::string &file_id) { +Status SqliteMetaImpl::NextFileId(std::string &file_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -99,12 +99,12 @@ Status DBMetaImpl::NextFileId(std::string &file_id) { return Status::OK(); } -DBMetaImpl::DBMetaImpl(const DBMetaOptions &options_) +SqliteMetaImpl::SqliteMetaImpl(const DBMetaOptions &options_) : options_(options_) { Initialize(); } -Status DBMetaImpl::Initialize() { +Status SqliteMetaImpl::Initialize() { if (!boost::filesystem::is_directory(options_.path)) { auto ret = boost::filesystem::create_directory(options_.path); if (!ret) { @@ -125,7 +125,7 @@ Status DBMetaImpl::Initialize() { } // PXU TODO: Temp solution. Will fix later -Status DBMetaImpl::DropPartitionsByDates(const std::string &table_id, +Status SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id, const DatesT &dates) { if (dates.size() == 0) { return Status::OK(); @@ -165,7 +165,7 @@ Status DBMetaImpl::DropPartitionsByDates(const std::string &table_id, return Status::OK(); } -Status DBMetaImpl::CreateTable(TableSchema &table_schema) { +Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) { try { MetricCollector metric; @@ -209,7 +209,7 @@ Status DBMetaImpl::CreateTable(TableSchema &table_schema) { return Status::OK(); } -Status DBMetaImpl::DeleteTable(const std::string& table_id) { +Status SqliteMetaImpl::DeleteTable(const std::string& table_id) { try { MetricCollector metric; @@ -244,7 +244,7 @@ Status DBMetaImpl::DeleteTable(const std::string& table_id) { return Status::OK(); } -Status DBMetaImpl::DeleteTableFiles(const std::string& table_id) { +Status SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) { try { MetricCollector metric; @@ -269,7 +269,7 @@ Status DBMetaImpl::DeleteTableFiles(const std::string& table_id) { return Status::OK(); } -Status DBMetaImpl::DescribeTable(TableSchema &table_schema) { +Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { try { MetricCollector metric; @@ -299,7 +299,7 @@ Status DBMetaImpl::DescribeTable(TableSchema &table_schema) { return Status::OK(); } -Status DBMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) { +Status SqliteMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) { has = false; try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, @@ -353,7 +353,7 @@ Status DBMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) { return Status::OK(); } -Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { +Status SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { has_or_not = false; try { @@ -374,7 +374,7 @@ Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { return Status::OK(); } -Status DBMetaImpl::AllTables(std::vector& table_schema_array) { +Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { try { MetricCollector metric; @@ -404,7 +404,7 @@ Status DBMetaImpl::AllTables(std::vector& table_schema_array) { return Status::OK(); } -Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) { +Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { if (file_schema.date_ == EmptyDate) { file_schema.date_ = Meta::GetDate(); } @@ -440,7 +440,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) { return Status::OK(); } -Status DBMetaImpl::FilesToIndex(TableFilesSchema &files) { +Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { files.clear(); try { @@ -490,7 +490,7 @@ Status DBMetaImpl::FilesToIndex(TableFilesSchema &files) { return Status::OK(); } -Status DBMetaImpl::FilesToSearch(const std::string &table_id, +Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, const DatesT &partition, DatePartionedTableFilesSchema &files) { files.clear(); @@ -589,7 +589,7 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id, return Status::OK(); } -Status DBMetaImpl::FilesToSearch(const std::string &table_id, +Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, const std::vector &ids, const DatesT &partition, DatePartionedTableFilesSchema &files) { @@ -662,7 +662,7 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id, return Status::OK(); } -Status DBMetaImpl::FilesToMerge(const std::string &table_id, +Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) { files.clear(); @@ -710,7 +710,7 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id, return Status::OK(); } -Status DBMetaImpl::GetTableFiles(const std::string& table_id, +Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, const std::vector& ids, TableFilesSchema& table_files) { try { @@ -754,7 +754,7 @@ Status DBMetaImpl::GetTableFiles(const std::string& table_id, } // PXU TODO: Support Swap -Status DBMetaImpl::Archive() { +Status SqliteMetaImpl::Archive() { auto &criterias = options_.archive_conf.GetCriterias(); if (criterias.size() == 0) { return Status::OK(); @@ -794,7 +794,7 @@ Status DBMetaImpl::Archive() { return Status::OK(); } -Status DBMetaImpl::Size(uint64_t &result) { +Status SqliteMetaImpl::Size(uint64_t &result) { result = 0; try { auto files = ConnectorPtr->select(columns(&TableFileSchema::size_, @@ -821,7 +821,7 @@ Status DBMetaImpl::Size(uint64_t &result) { return Status::OK(); } -Status DBMetaImpl::DiscardFiles(long to_discard_size) { +Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { if (to_discard_size <= 0) { return Status::OK(); } @@ -883,7 +883,7 @@ Status DBMetaImpl::DiscardFiles(long to_discard_size) { return DiscardFiles(to_discard_size); } -Status DBMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { +Status SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); try { MetricCollector metric; @@ -910,7 +910,7 @@ Status DBMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { return Status::OK(); } -Status DBMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { +Status SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { try { MetricCollector metric; @@ -932,7 +932,7 @@ Status DBMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { return Status::OK(); } -Status DBMetaImpl::UpdateTableFiles(TableFilesSchema &files) { +Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { try { MetricCollector metric; @@ -977,7 +977,7 @@ Status DBMetaImpl::UpdateTableFiles(TableFilesSchema &files) { return Status::OK(); } -Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { +Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { auto now = utils::GetMicroSecTimeStamp(); try { MetricCollector metric; @@ -1052,7 +1052,7 @@ Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return Status::OK(); } -Status DBMetaImpl::CleanUp() { +Status SqliteMetaImpl::CleanUp() { try { MetricCollector metric; @@ -1086,7 +1086,7 @@ Status DBMetaImpl::CleanUp() { return Status::OK(); } -Status DBMetaImpl::Count(const std::string &table_id, uint64_t &result) { +Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { try { MetricCollector metric; @@ -1120,14 +1120,14 @@ Status DBMetaImpl::Count(const std::string &table_id, uint64_t &result) { return Status::OK(); } -Status DBMetaImpl::DropAll() { +Status SqliteMetaImpl::DropAll() { if (boost::filesystem::is_directory(options_.path)) { boost::filesystem::remove_all(options_.path); } return Status::OK(); } -DBMetaImpl::~DBMetaImpl() { +SqliteMetaImpl::~SqliteMetaImpl() { CleanUp(); } diff --git a/cpp/src/db/meta/DBMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h similarity index 94% rename from cpp/src/db/meta/DBMetaImpl.h rename to cpp/src/db/meta/SqliteMetaImpl.h index 7f2a87fcb4814fac4a33a0ba7d25fa3cc84e044e..1525f27e4b776afc7757c893199b7575e2f96fda 100644 --- a/cpp/src/db/meta/DBMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -6,7 +6,7 @@ #pragma once #include "Meta.h" -#include "Options.h" +#include "db/Options.h" #include @@ -17,9 +17,9 @@ namespace meta { auto StoragePrototype(const std::string &path); -class DBMetaImpl : public Meta { +class SqliteMetaImpl : public Meta { public: - explicit DBMetaImpl(const DBMetaOptions &options_); + explicit SqliteMetaImpl(const DBMetaOptions &options_); Status CreateTable(TableSchema &table_schema) override; @@ -91,7 +91,7 @@ class DBMetaImpl : public Meta { Status Count(const std::string &table_id, uint64_t &result) override; - ~DBMetaImpl() override; + ~SqliteMetaImpl() override; private: Status NextFileId(std::string &file_id); diff --git a/cpp/src/db/scheduler/TaskScheduler.cpp b/cpp/src/db/scheduler/TaskScheduler.cpp index f29c3d4caa1db02113c5f03f447f17b314678216..28d46da76c6c12a67bcd85db93e7409e5e1efdcf 100644 --- a/cpp/src/db/scheduler/TaskScheduler.cpp +++ b/cpp/src/db/scheduler/TaskScheduler.cpp @@ -8,7 +8,7 @@ #include "TaskDispatchQueue.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" -#include "db/EngineFactory.h" +#include "db/engine/EngineFactory.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/db/scheduler/context/DeleteContext.h b/cpp/src/db/scheduler/context/DeleteContext.h index b909ecb69ea36ee98f0214ad822937235dee466e..efcc3bb6a7ca6a820832ac74142d870da5ef7f31 100644 --- a/cpp/src/db/scheduler/context/DeleteContext.h +++ b/cpp/src/db/scheduler/context/DeleteContext.h @@ -6,7 +6,7 @@ #pragma once #include "IScheduleContext.h" -#include "db/Meta.h" +#include "db/meta/Meta.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/db/scheduler/context/SearchContext.h b/cpp/src/db/scheduler/context/SearchContext.h index 3cea23d96d68ac408aec47de99cff3c6b343f647..466fc435b462d5889d8bb24fac9ecee402877b09 100644 --- a/cpp/src/db/scheduler/context/SearchContext.h +++ b/cpp/src/db/scheduler/context/SearchContext.h @@ -6,7 +6,7 @@ #pragma once #include "IScheduleContext.h" -#include "db/MetaTypes.h" +#include "db/meta/MetaTypes.h" #include #include diff --git a/cpp/src/db/scheduler/task/IndexLoadTask.cpp b/cpp/src/db/scheduler/task/IndexLoadTask.cpp index f3aacf83528aa2cd6b8fea646c0fc833f54883cd..2c319396d83f785fa940295bc7bc6b3dc84a9dba 100644 --- a/cpp/src/db/scheduler/task/IndexLoadTask.cpp +++ b/cpp/src/db/scheduler/task/IndexLoadTask.cpp @@ -7,7 +7,7 @@ #include "IndexLoadTask.h" #include "SearchTask.h" #include "db/Log.h" -#include "db/EngineFactory.h" +#include "db/engine/EngineFactory.h" #include "utils/TimeRecorder.h" #include "metrics/Metrics.h" diff --git a/cpp/src/db/scheduler/task/SearchTask.h b/cpp/src/db/scheduler/task/SearchTask.h index 7e0aa52af11c5e5eca34beb0b59dd531b178c2af..034b53d4dc3431514812b1a435286681c6bdfbff 100644 --- a/cpp/src/db/scheduler/task/SearchTask.h +++ b/cpp/src/db/scheduler/task/SearchTask.h @@ -7,7 +7,7 @@ #include "IScheduleTask.h" #include "db/scheduler/context/SearchContext.h" -#include "db/ExecutionEngine.h" +#include "db/engine/ExecutionEngine.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index 5bd09bd0f4e32b5dcdb4798fc63c46948db7e11a..fdde4b157cfbaa71aa093966282acf8ae91d5e4a 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -6,7 +6,6 @@ #pragma once #include "db/DB.h" -#include "db/Meta.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index bfdf35cee27047f2657e7eee6ff93cd9119dd332..fa0cd8397d6f2b6f486f5d5a2f12f0dbeaaa4b3b 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -1,4 +1,4 @@ -#include "db/ExecutionEngine.h" +#include "db/engine/ExecutionEngine.h" #include "ValidationUtil.h" #include "Log.h" diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 458ad7589dbff4edd89dcb92cf8ceb382706e47f..aa96fa6e25871980079057bbf8017e73702fa8e0 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -40,7 +40,6 @@ set(unittest_libs add_subdirectory(server) add_subdirectory(db) -add_subdirectory(index_wrapper) +add_subdirectory(knowhere) add_subdirectory(metrics) -add_subdirectory(storage) -add_subdirectory(utils) \ No newline at end of file +#add_subdirectory(storage) \ No newline at end of file diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 6066c410958c1ddb571a2a1e4fbef9b7786419ee..6c451bf20f76397a4953e8e1440b3918848cec61 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -3,10 +3,12 @@ # Unauthorized copying of this file, via any medium is strictly prohibited. # Proprietary and confidential. #------------------------------------------------------------------------------- -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) +aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) -#aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src) aux_source_directory(./ test_srcs) @@ -32,14 +34,17 @@ include_directories(/usr/include/mysql) set(db_test_src ${config_files} ${cache_srcs} - ${db_srcs} + ${db_main_files} + ${db_engine_files} + ${db_insert_files} + ${db_meta_files} ${db_scheduler_srcs} ${wrapper_src} ${knowhere_src} ${util_files} ${require_files} ${test_srcs} - ) + ) cuda_add_executable(db_test ${db_test_src}) diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 81f2279d1cf5f4230f9737cfdd7bf942d2b80e80..19d7828c6f2c37383e2fbc5157fc63dbb66d92b9 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -6,7 +6,7 @@ #include "utils.h" #include "db/DB.h" #include "db/DBImpl.h" -#include "db/MetaConsts.h" +#include "db/meta/MetaConsts.h" #include "db/Factories.h" #include diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 17d87030cd3fed6e3e827d412acc61640e030fe7..b410b0bcbd8806b3131b249e8c4c27c56bed07ff 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -1,16 +1,16 @@ #include "gtest/gtest.h" -#include "db/VectorSource.h" -#include "db/MemTableFile.h" -#include "db/MemTable.h" +#include "db/insert/VectorSource.h" +#include "db/insert/MemTableFile.h" +#include "db/insert/MemTable.h" #include "utils.h" #include "db/Factories.h" #include "db/Constants.h" -#include "db/EngineFactory.h" +#include "db/engine/EngineFactory.h" #include "metrics/Metrics.h" -#include "db/MetaConsts.h" -#include "boost/filesystem.hpp" +#include "db/meta/MetaConsts.h" +#include #include #include #include @@ -47,7 +47,7 @@ void BuildVectors(int64_t n, std::vector &vectors) { TEST_F(NewMemManagerTest, VECTOR_SOURCE_TEST) { - std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); + std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); engine::meta::TableSchema table_schema = BuildTableSchema(); auto status = impl_->CreateTable(table_schema); @@ -90,7 +90,7 @@ TEST_F(NewMemManagerTest, VECTOR_SOURCE_TEST) { TEST_F(NewMemManagerTest, MEM_TABLE_FILE_TEST) { - std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); + std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); auto options = engine::OptionsFactory::Build(); engine::meta::TableSchema table_schema = BuildTableSchema(); @@ -134,7 +134,7 @@ TEST_F(NewMemManagerTest, MEM_TABLE_FILE_TEST) { TEST_F(NewMemManagerTest, MEM_TABLE_TEST) { - std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); + std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); auto options = engine::OptionsFactory::Build(); engine::meta::TableSchema table_schema = BuildTableSchema(); diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 36643ead947baaa7893772ca162588889cd7fad8..0e981f6ae420f395d233161263b430051cf16c0d 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -10,10 +10,10 @@ #include #include "utils.h" -#include "db/DBMetaImpl.h" +#include "db/meta/SqliteMetaImpl.h" #include "db/Factories.h" #include "db/Utils.h" -#include "db/MetaConsts.h" +#include "db/meta/MetaConsts.h" using namespace zilliz::milvus::engine; @@ -113,7 +113,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { ss << "days:" << days_num; options.archive_conf = ArchiveConf("delete", ss.str()); - meta::DBMetaImpl impl(options); + meta::SqliteMetaImpl impl(options); auto table_id = "meta_test_table"; meta::TableSchema table; @@ -163,7 +163,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { options.path = "/tmp/milvus_test"; options.archive_conf = ArchiveConf("delete", "disk:11"); - meta::DBMetaImpl impl(options); + meta::SqliteMetaImpl impl(options); auto table_id = "meta_test_group"; meta::TableSchema table; diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index d968b69b8ee060071c0def1a83d83b13b8cc3f77..6433a224ff26e8d6baf6f1aaa076ac7c4d665c03 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -11,8 +11,8 @@ #include "db/Exception.h" #include "db/Status.h" #include "db/Options.h" -#include "db/DBMetaImpl.h" -#include "db/EngineFactory.h" +#include "db/meta/SqliteMetaImpl.h" +#include "db/engine/EngineFactory.h" #include "db/Utils.h" #include @@ -100,7 +100,7 @@ TEST(DBMiscTest, OPTIONS_TEST) { TEST(DBMiscTest, META_TEST) { engine::DBMetaOptions options; options.path = "/tmp/milvus_test"; - engine::meta::DBMetaImpl impl(options); + engine::meta::SqliteMetaImpl impl(options); time_t tt; time( &tt ); diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 21160eed58fdf7bcd89e5a202096af18bdbe64ed..c3752438b93331ddcf178cd2b9db9f6bd12ca337 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -6,7 +6,7 @@ #include "utils.h" #include "db/DB.h" #include "db/DBImpl.h" -#include "db/MetaConsts.h" +#include "db/meta/MetaConsts.h" #include "db/Factories.h" #include diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 9e4b817b92fa97a7d9e9549ade9530896bca124a..059fe9a2c79b34dfca3932d5274f3cb36448d75f 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -10,10 +10,10 @@ #include #include "utils.h" -#include "db/MySQLMetaImpl.h" +#include "db/meta/MySQLMetaImpl.h" #include "db/Factories.h" #include "db/Utils.h" -#include "db/MetaConsts.h" +#include "db/meta/MetaConsts.h" #include "mysql++/mysql++.h" diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index 08d1caaaf5d20b70487a85f5d3c0dcff1a05f2cf..4d3e318faa4d22b2406bf4ce89f001198b98dee6 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -11,8 +11,8 @@ //#include #include "db/DB.h" -#include "db/DBMetaImpl.h" -#include "db/MySQLMetaImpl.h" +#include "db/meta/SqliteMetaImpl.h" +#include "db/meta/MySQLMetaImpl.h" #define TIMING @@ -71,7 +71,7 @@ class DBTest2 : public DBTest { class MetaTest : public DBTest { protected: - std::shared_ptr impl_; + std::shared_ptr impl_; virtual void SetUp() override; virtual void TearDown() override; diff --git a/cpp/unittest/index_wrapper/CMakeLists.txt b/cpp/unittest/knowhere/CMakeLists.txt similarity index 100% rename from cpp/unittest/index_wrapper/CMakeLists.txt rename to cpp/unittest/knowhere/CMakeLists.txt diff --git a/cpp/unittest/index_wrapper/knowhere_test.cpp b/cpp/unittest/knowhere/knowhere_test.cpp similarity index 100% rename from cpp/unittest/index_wrapper/knowhere_test.cpp rename to cpp/unittest/knowhere/knowhere_test.cpp diff --git a/cpp/unittest/index_wrapper/utils.cpp b/cpp/unittest/knowhere/utils.cpp similarity index 100% rename from cpp/unittest/index_wrapper/utils.cpp rename to cpp/unittest/knowhere/utils.cpp diff --git a/cpp/unittest/index_wrapper/utils.h b/cpp/unittest/knowhere/utils.h similarity index 100% rename from cpp/unittest/index_wrapper/utils.h rename to cpp/unittest/knowhere/utils.h diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 02fe3009b99248373cc9ba677a498c1d4a34896c..47b86d11078ada8a268047ce1ee249a1e1890f9a 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -11,8 +11,10 @@ include_directories(../../src) - -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) +aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) @@ -44,7 +46,10 @@ include_directories(/usr/include/mysql) set(count_test_src ${config_files} ${cache_srcs} - ${db_srcs} + ${db_main_files} + ${db_engine_files} + ${db_insert_files} + ${db_meta_files} ${db_scheduler_srcs} ${wrapper_src} ${knowhere_src} diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index 883e63ed03298d3184d5d84bb07074b81aa08a18..db9b06b78820d1d8710299728c379afabefca9b9 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -17,7 +17,7 @@ #include "metrics/Metrics.h" #include "utils.h" #include "db/DB.h" -#include "db/DBMetaImpl.h" +#include "db/meta/SqliteMetaImpl.h" #include "db/Factories.h" diff --git a/cpp/unittest/metrics/utils.h b/cpp/unittest/metrics/utils.h index 1badce00f217698f73bac53bb8f794952654f4a1..4980b91a1ec713866e0be45278c4c14e017a711f 100644 --- a/cpp/unittest/metrics/utils.h +++ b/cpp/unittest/metrics/utils.h @@ -11,8 +11,8 @@ //#include #include "db/DB.h" -#include "db/DBMetaImpl.h" -#include "db/MySQLMetaImpl.h" +#include "db/meta/SqliteMetaImpl.h" +#include "db/meta/MySQLMetaImpl.h" #define TIMING diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index c60e2dbe2c0e5966d91814c4be310fb94d3b376e..33c7fcdea252707f673906aeadf255cc99a8ff13 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -3,15 +3,15 @@ # Unauthorized copying of this file, via any medium is strictly prohibited. # Proprietary and confidential. #------------------------------------------------------------------------------- -include_directories(../../src) +include_directories(${MILVUS_ENGINE_SRC}/) include_directories(/usr/include) include_directories(/usr/local/cuda/include) link_directories(/usr/local/cuda/lib64) -aux_source_directory(../../src/config config_files) -aux_source_directory(../../src/cache cache_srcs) -aux_source_directory(../../src/wrapper wrapper_src) +aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) +aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) aux_source_directory(./ test_srcs) set(utils_srcs diff --git a/cpp/unittest/server/log_test.cpp b/cpp/unittest/server/log_test.cpp deleted file mode 100644 index 22410f78c97ae276056f79b340f081f05637676f..0000000000000000000000000000000000000000 --- a/cpp/unittest/server/log_test.cpp +++ /dev/null @@ -1,20 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// -#include -#include "easylogging++.h" - -class LogTest: public testing::Test { -protected: - void SetUp() override { - el::Configurations conf("../../../conf/log_config.conf"); - el::Loggers::reconfigureAllLoggers(conf); - } -}; - -TEST_F(LogTest, TEST) { - EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer)); - EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::LogDetailedCrashReason)); -} \ No newline at end of file diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index e4948fc5e828cb596442f72846626b59f0735f9b..ea59847245d29d1dba223f52e8f4f6ba2b4214d8 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -12,6 +12,8 @@ #include "utils/TimeRecorder.h" #include "utils/BlockingQueue.h" #include "utils/LogUtil.h" +#include "utils/ValidationUtil.h" +#include "db/engine/ExecutionEngine.h" using namespace zilliz::milvus; @@ -148,6 +150,57 @@ TEST(UtilTest, LOG_TEST) { int32_t res = server::InitLog(LOG_FILE_PATH); ASSERT_EQ(res, 0); + EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer)); + EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::LogDetailedCrashReason)); + std::string fname = server::GetFileName(LOG_FILE_PATH); ASSERT_EQ(fname, "log_config.conf"); } + +TEST(UtilTest, VALIDATE_TABLENAME_TEST) { + std::string table_name = "Normal123_"; + server:: ServerError res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_SUCCESS); + + table_name = "12sds"; + res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + + table_name = ""; + res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + + table_name = "_asdasd"; + res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_SUCCESS); + + table_name = "!@#!@"; + res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + + table_name = "中文"; + res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + + + table_name = std::string('a', 32768); + res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); +} + +TEST(UtilTest, VALIDATE_DIMENSIONTEST) { + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(-1), server::SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(0), server::SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16385), server::SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16384), server::SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(1), server::SERVER_SUCCESS); +} + +TEST(UtilTest, VALIDATE_INDEXTYPE_TEST) { + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), server::SERVER_INVALID_INDEX_TYPE); + for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) { + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType(i), server::SERVER_SUCCESS); + } + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), server::SERVER_INVALID_INDEX_TYPE); +} + diff --git a/cpp/unittest/utils/CMakeLists.txt b/cpp/unittest/utils/CMakeLists.txt deleted file mode 100644 index 61e4459462ea21b4f4b6da8fab8dbc033a90b13f..0000000000000000000000000000000000000000 --- a/cpp/unittest/utils/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -#------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. -#------------------------------------------------------------------------------- - -# Make sure that your call to link_directories takes place before your call to the relevant add_executable. -include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") -link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") - -set(validation_util_src - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.h) - -set(validation_util_test_src - ${unittest_srcs} - ${validation_util_src} - ${require_files} - ValidationUtilTest.cpp - ) - -add_executable(valication_util_test - ${validation_util_test_src} - ${config_files}) - -target_link_libraries(valication_util_test - ${unittest_libs} - boost_filesystem_static) - -install(TARGETS valication_util_test DESTINATION bin) \ No newline at end of file diff --git a/cpp/unittest/utils/ValidationUtilTest.cpp b/cpp/unittest/utils/ValidationUtilTest.cpp deleted file mode 100644 index 871172e6c9d5ee31e3a1bfeb95720748a2a14120..0000000000000000000000000000000000000000 --- a/cpp/unittest/utils/ValidationUtilTest.cpp +++ /dev/null @@ -1,63 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// -#include - -#include "utils/ValidationUtil.h" -#include "utils/Error.h" -#include "db/ExecutionEngine.h" - -#include - -using namespace zilliz::milvus; -using namespace zilliz::milvus::server; - -TEST(ValidationUtilTest, TableNameTest) { - std::string table_name = "Normal123_"; - ServerError res = ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_SUCCESS); - - table_name = "12sds"; - res = ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); - - table_name = ""; - res = ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); - - table_name = "_asdasd"; - res = ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_SUCCESS); - - table_name = "!@#!@"; - res = ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); - - table_name = "中文"; - res = ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); - - - table_name = std::string('a', 32768); - res = ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); -} - - -TEST(ValidationUtilTest, TableDimensionTest) { - ASSERT_EQ(ValidationUtil::ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ValidationUtil::ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ValidationUtil::ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ValidationUtil::ValidateTableDimension(16384), SERVER_SUCCESS); - ASSERT_EQ(ValidationUtil::ValidateTableDimension(1), SERVER_SUCCESS); -} - -TEST(ValidationUtilTest, TableIndexTypeTest) { - ASSERT_EQ(ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE); - for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) { - ASSERT_EQ(ValidationUtil::ValidateTableIndexType(i), SERVER_SUCCESS); - } - ASSERT_EQ(ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE); -}