From 39499d2f77a7126504dcdc104e58ef2fbeaf2c9d Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 8 Aug 2019 15:36:00 +0800 Subject: [PATCH] MS-327 clean code Former-commit-id: 46b960c0a67766c4d67b36826ec88b47ae470206 --- cpp/src/CMakeLists.txt | 16 +++-- cpp/src/db/DB.cpp | 1 - cpp/src/db/DB.h | 2 +- cpp/src/db/DBImpl.cpp | 6 +- cpp/src/db/DBImpl.h | 2 +- cpp/src/db/Factories.cpp | 10 +-- cpp/src/db/Factories.h | 9 ++- cpp/src/db/Options.cpp | 2 +- cpp/src/db/Utils.h | 2 +- cpp/src/db/{ => engine}/EngineFactory.cpp | 2 +- cpp/src/db/{ => engine}/EngineFactory.h | 2 +- cpp/src/db/{ => engine}/ExecutionEngine.cpp | 0 cpp/src/db/{ => engine}/ExecutionEngine.h | 2 +- .../db/{ => engine}/ExecutionEngineImpl.cpp | 8 +-- cpp/src/db/{ => engine}/ExecutionEngineImpl.h | 0 .../MemManager.h} | 4 +- .../MemManagerImpl.cpp} | 24 +++---- .../MemManagerImpl.h} | 12 ++-- cpp/src/db/{ => insert}/MemTable.cpp | 2 +- cpp/src/db/{ => insert}/MemTable.h | 2 +- cpp/src/db/{ => insert}/MemTableFile.cpp | 6 +- cpp/src/db/{ => insert}/MemTableFile.h | 6 +- cpp/src/db/{ => insert}/VectorSource.cpp | 6 +- cpp/src/db/{ => insert}/VectorSource.h | 8 +-- cpp/src/db/{ => meta}/Meta.cpp | 0 cpp/src/db/{ => meta}/Meta.h | 4 +- cpp/src/db/{ => meta}/MetaConsts.h | 0 cpp/src/db/{ => meta}/MetaTypes.h | 2 +- cpp/src/db/{ => meta}/MySQLConnectionPool.cpp | 0 cpp/src/db/{ => meta}/MySQLConnectionPool.h | 2 +- cpp/src/db/{ => meta}/MySQLMetaImpl.cpp | 8 +-- cpp/src/db/{ => meta}/MySQLMetaImpl.h | 2 +- .../{DBMetaImpl.cpp => SqliteMetaImpl.cpp} | 68 +++++++++---------- .../meta/{DBMetaImpl.h => SqliteMetaImpl.h} | 8 +-- cpp/src/db/scheduler/TaskScheduler.cpp | 2 +- cpp/src/db/scheduler/context/DeleteContext.h | 2 +- cpp/src/db/scheduler/context/SearchContext.h | 2 +- cpp/src/db/scheduler/task/IndexLoadTask.cpp | 2 +- cpp/src/db/scheduler/task/SearchTask.h | 2 +- cpp/src/server/DBWrapper.h | 1 - cpp/src/utils/ValidationUtil.cpp | 2 +- cpp/unittest/CMakeLists.txt | 5 +- cpp/unittest/db/CMakeLists.txt | 13 ++-- cpp/unittest/db/db_tests.cpp | 2 +- cpp/unittest/db/mem_test.cpp | 18 ++--- cpp/unittest/db/meta_tests.cpp | 8 +-- cpp/unittest/db/misc_test.cpp | 6 +- cpp/unittest/db/mysql_db_test.cpp | 2 +- cpp/unittest/db/mysql_meta_test.cpp | 4 +- cpp/unittest/db/utils.h | 6 +- .../CMakeLists.txt | 0 .../knowhere_test.cpp | 0 .../{index_wrapper => knowhere}/utils.cpp | 0 .../{index_wrapper => knowhere}/utils.h | 0 cpp/unittest/metrics/CMakeLists.txt | 11 ++- cpp/unittest/metrics/metrics_test.cpp | 2 +- cpp/unittest/metrics/utils.h | 4 +- cpp/unittest/server/CMakeLists.txt | 8 +-- cpp/unittest/server/log_test.cpp | 20 ------ cpp/unittest/server/util_test.cpp | 53 +++++++++++++++ cpp/unittest/utils/CMakeLists.txt | 30 -------- cpp/unittest/utils/ValidationUtilTest.cpp | 63 ----------------- 62 files changed, 223 insertions(+), 273 deletions(-) rename cpp/src/db/{ => engine}/EngineFactory.cpp (99%) rename cpp/src/db/{ => engine}/EngineFactory.h (96%) rename cpp/src/db/{ => engine}/ExecutionEngine.cpp (100%) rename cpp/src/db/{ => engine}/ExecutionEngine.h (98%) rename cpp/src/db/{ => engine}/ExecutionEngineImpl.cpp (98%) rename cpp/src/db/{ => engine}/ExecutionEngineImpl.h (100%) rename cpp/src/db/{MemManagerAbstract.h => insert/MemManager.h} (83%) rename cpp/src/db/{NewMemManager.cpp => insert/MemManagerImpl.cpp} (81%) rename cpp/src/db/{NewMemManager.h => insert/MemManagerImpl.h} (82%) rename cpp/src/db/{ => insert}/MemTable.cpp (98%) rename cpp/src/db/{ => insert}/MemTable.h (95%) rename cpp/src/db/{ => insert}/MemTableFile.cpp (97%) rename cpp/src/db/{ => insert}/MemTableFile.h (87%) rename cpp/src/db/{ => insert}/VectorSource.cpp (94%) rename cpp/src/db/{ => insert}/VectorSource.h (84%) rename cpp/src/db/{ => meta}/Meta.cpp (100%) rename cpp/src/db/{ => meta}/Meta.h (98%) rename cpp/src/db/{ => meta}/MetaConsts.h (100%) rename cpp/src/db/{ => meta}/MetaTypes.h (97%) rename cpp/src/db/{ => meta}/MySQLConnectionPool.cpp (100%) rename cpp/src/db/{ => meta}/MySQLConnectionPool.h (97%) rename cpp/src/db/{ => meta}/MySQLMetaImpl.cpp (99%) rename cpp/src/db/{ => meta}/MySQLMetaImpl.h (99%) rename cpp/src/db/meta/{DBMetaImpl.cpp => SqliteMetaImpl.cpp} (95%) rename cpp/src/db/meta/{DBMetaImpl.h => SqliteMetaImpl.h} (94%) rename cpp/unittest/{index_wrapper => knowhere}/CMakeLists.txt (100%) rename cpp/unittest/{index_wrapper => knowhere}/knowhere_test.cpp (100%) rename cpp/unittest/{index_wrapper => knowhere}/utils.cpp (100%) rename cpp/unittest/{index_wrapper => knowhere}/utils.h (100%) delete mode 100644 cpp/unittest/server/log_test.cpp delete mode 100644 cpp/unittest/utils/CMakeLists.txt delete mode 100644 cpp/unittest/utils/ValidationUtilTest.cpp diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 2e6ea844..6fae1236 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -9,8 +9,10 @@ aux_source_directory(cache cache_files) aux_source_directory(config config_files) aux_source_directory(server server_files) aux_source_directory(utils utils_files) -aux_source_directory(db db_files) -#aux_source_directory(wrapper wrapper_files) +aux_source_directory(db db_main_files) +aux_source_directory(db/engine db_engine_files) +aux_source_directory(db/insert db_insert_files) +aux_source_directory(db/meta db_meta_files) aux_source_directory(metrics metrics_files) aux_source_directory(wrapper/knowhere knowhere_files) @@ -33,12 +35,14 @@ set(service_files server/ThreadPoolServer.h ) -set(engine_files +set(db_files ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp ${cache_files} - ${db_files} + ${db_main_files} + ${db_engine_files} + ${db_insert_files} + ${db_meta_files} ${db_scheduler_files} - ${wrapper_files} ${metrics_files} ${knowhere_files} ) @@ -126,7 +130,7 @@ if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") endif () -cuda_add_library(milvus_engine STATIC ${engine_files}) +cuda_add_library(milvus_engine STATIC ${db_files}) target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs}) add_library(metrics STATIC ${metrics_files}) diff --git a/cpp/src/db/DB.cpp b/cpp/src/db/DB.cpp index 3b4f03d2..2d922dd3 100644 --- a/cpp/src/db/DB.cpp +++ b/cpp/src/db/DB.cpp @@ -5,7 +5,6 @@ ******************************************************************************/ #include "DBImpl.h" -#include "DBMetaImpl.h" #include "Factories.h" namespace zilliz { diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 3bfd2281..f1d3a77a 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -6,7 +6,7 @@ #pragma once #include "Options.h" -#include "Meta.h" +#include "meta/Meta.h" #include "Status.h" #include "Types.h" diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 17537556..5df588bd 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -4,16 +4,16 @@ * Proprietary and confidential. ******************************************************************************/ #include "DBImpl.h" -#include "DBMetaImpl.h" +#include "src/db/meta/SqliteMetaImpl.h" #include "Log.h" -#include "EngineFactory.h" +#include "engine/EngineFactory.h" #include "Factories.h" #include "metrics/Metrics.h" #include "scheduler/TaskScheduler.h" #include "scheduler/context/DeleteContext.h" #include "utils/TimeRecorder.h" -#include "MetaConsts.h" +#include "meta/MetaConsts.h" #include #include diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 27b0b66f..ccd7f541 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -8,7 +8,7 @@ #include "DB.h" #include "Types.h" #include "utils/ThreadPool.h" -#include "MemManagerAbstract.h" +#include "src/db/insert/MemManager.h" #include #include diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 05a1ad73..bb1056e3 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -6,7 +6,7 @@ #include "Factories.h" #include "DBImpl.h" -#include "NewMemManager.h" +#include "src/db/insert/MemManagerImpl.h" #include "Exception.h" #include @@ -46,9 +46,9 @@ Options OptionsFactory::Build() { return options; } -std::shared_ptr DBMetaImplFactory::Build() { +std::shared_ptr DBMetaImplFactory::Build() { DBMetaOptions options = DBMetaOptionsFactory::Build(); - return std::shared_ptr(new meta::DBMetaImpl(options)); + return std::shared_ptr(new meta::SqliteMetaImpl(options)); } std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOptions, @@ -79,7 +79,7 @@ std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOp return std::make_shared(metaOptions, mode); } else if (dialect.find("sqlite") != std::string::npos) { ENGINE_LOG_INFO << "Using SQLite"; - return std::make_shared(metaOptions); + return std::make_shared(metaOptions); } else { ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << dialect; throw InvalidArgumentException("URI dialect is not mysql / sqlite"); @@ -102,7 +102,7 @@ DB* DBFactory::Build(const Options& options) { MemManagerAbstractPtr MemManagerFactory::Build(const std::shared_ptr& meta, const Options& options) { - return std::make_shared(meta, options); + return std::make_shared(meta, options); } } // namespace engine diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index 8b6e7b10..0e6823c3 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -6,11 +6,10 @@ #pragma once #include "DB.h" -#include "DBMetaImpl.h" -#include "MySQLMetaImpl.h" +#include "src/db/meta/SqliteMetaImpl.h" +#include "meta/MySQLMetaImpl.h" #include "Options.h" -#include "ExecutionEngine.h" -#include "MemManagerAbstract.h" +#include "src/db/insert/MemManager.h" #include #include @@ -29,7 +28,7 @@ struct OptionsFactory { }; struct DBMetaImplFactory { - static std::shared_ptr Build(); + static std::shared_ptr Build(); static std::shared_ptr Build(const DBMetaOptions &metaOptions, const int &mode); }; diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index 81f97978..287f9f07 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -9,7 +9,7 @@ #include #include "Options.h" -#include "DBMetaImpl.h" +#include "db/meta/SqliteMetaImpl.h" #include "Exception.h" namespace zilliz { diff --git a/cpp/src/db/Utils.h b/cpp/src/db/Utils.h index a5a0f8ff..8329f5a1 100644 --- a/cpp/src/db/Utils.h +++ b/cpp/src/db/Utils.h @@ -6,7 +6,7 @@ #pragma once #include "Options.h" -#include "MetaTypes.h" +#include "db/meta/MetaTypes.h" #include diff --git a/cpp/src/db/EngineFactory.cpp b/cpp/src/db/engine/EngineFactory.cpp similarity index 99% rename from cpp/src/db/EngineFactory.cpp rename to cpp/src/db/engine/EngineFactory.cpp index 32e10beb..d09e9f8b 100644 --- a/cpp/src/db/EngineFactory.cpp +++ b/cpp/src/db/engine/EngineFactory.cpp @@ -6,7 +6,7 @@ #include "EngineFactory.h" //#include "FaissExecutionEngine.h" #include "ExecutionEngineImpl.h" -#include "Log.h" +#include "db/Log.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/db/EngineFactory.h b/cpp/src/db/engine/EngineFactory.h similarity index 96% rename from cpp/src/db/EngineFactory.h rename to cpp/src/db/engine/EngineFactory.h index 9d667bd5..d8c35468 100644 --- a/cpp/src/db/EngineFactory.h +++ b/cpp/src/db/engine/EngineFactory.h @@ -5,7 +5,7 @@ ******************************************************************************/ #pragma once -#include "Status.h" +#include "db/Status.h" #include "ExecutionEngine.h" namespace zilliz { diff --git a/cpp/src/db/ExecutionEngine.cpp b/cpp/src/db/engine/ExecutionEngine.cpp similarity index 100% rename from cpp/src/db/ExecutionEngine.cpp rename to cpp/src/db/engine/ExecutionEngine.cpp diff --git a/cpp/src/db/ExecutionEngine.h b/cpp/src/db/engine/ExecutionEngine.h similarity index 98% rename from cpp/src/db/ExecutionEngine.h rename to cpp/src/db/engine/ExecutionEngine.h index f942e1fa..5871be7f 100644 --- a/cpp/src/db/ExecutionEngine.h +++ b/cpp/src/db/engine/ExecutionEngine.h @@ -5,7 +5,7 @@ ******************************************************************************/ #pragma once -#include "Status.h" +#include "db/Status.h" #include #include diff --git a/cpp/src/db/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp similarity index 98% rename from cpp/src/db/ExecutionEngineImpl.cpp rename to cpp/src/db/engine/ExecutionEngineImpl.cpp index 1164116f..21878377 100644 --- a/cpp/src/db/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -5,9 +5,9 @@ ******************************************************************************/ #include -#include -#include -#include "Log.h" +#include "src/server/ServerConfig.h" +#include "src/metrics/Metrics.h" +#include "db/Log.h" #include "utils/CommonUtil.h" #include "src/cache/CpuCacheMgr.h" @@ -15,7 +15,7 @@ #include "wrapper/knowhere/vec_index.h" #include "wrapper/knowhere/vec_impl.h" #include "knowhere/common/exception.h" -#include "Exception.h" +#include "db/Exception.h" namespace zilliz { diff --git a/cpp/src/db/ExecutionEngineImpl.h b/cpp/src/db/engine/ExecutionEngineImpl.h similarity index 100% rename from cpp/src/db/ExecutionEngineImpl.h rename to cpp/src/db/engine/ExecutionEngineImpl.h diff --git a/cpp/src/db/MemManagerAbstract.h b/cpp/src/db/insert/MemManager.h similarity index 83% rename from cpp/src/db/MemManagerAbstract.h rename to cpp/src/db/insert/MemManager.h index 943c454e..7c91a47c 100644 --- a/cpp/src/db/MemManagerAbstract.h +++ b/cpp/src/db/insert/MemManager.h @@ -7,7 +7,7 @@ namespace zilliz { namespace milvus { namespace engine { -class MemManagerAbstract { +class MemManager { public: virtual Status InsertVectors(const std::string &table_id, @@ -25,7 +25,7 @@ class MemManagerAbstract { }; // MemManagerAbstract -using MemManagerAbstractPtr = std::shared_ptr; +using MemManagerAbstractPtr = std::shared_ptr; } // namespace engine } // namespace milvus diff --git a/cpp/src/db/NewMemManager.cpp b/cpp/src/db/insert/MemManagerImpl.cpp similarity index 81% rename from cpp/src/db/NewMemManager.cpp rename to cpp/src/db/insert/MemManagerImpl.cpp index c7291354..bd71f874 100644 --- a/cpp/src/db/NewMemManager.cpp +++ b/cpp/src/db/insert/MemManagerImpl.cpp @@ -1,7 +1,7 @@ -#include "NewMemManager.h" +#include "MemManagerImpl.h" #include "VectorSource.h" -#include "Log.h" -#include "Constants.h" +#include "db/Log.h" +#include "db/Constants.h" #include @@ -10,7 +10,7 @@ namespace zilliz { namespace milvus { namespace engine { -NewMemManager::MemTablePtr NewMemManager::GetMemByTable(const std::string &table_id) { +MemManagerImpl::MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) { auto memIt = mem_id_map_.find(table_id); if (memIt != mem_id_map_.end()) { return memIt->second; @@ -20,7 +20,7 @@ NewMemManager::MemTablePtr NewMemManager::GetMemByTable(const std::string &table return mem_id_map_[table_id]; } -Status NewMemManager::InsertVectors(const std::string &table_id_, +Status MemManagerImpl::InsertVectors(const std::string &table_id_, size_t n_, const float *vectors_, IDNumbers &vector_ids_) { @@ -34,7 +34,7 @@ Status NewMemManager::InsertVectors(const std::string &table_id_, return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); } -Status NewMemManager::InsertVectorsNoLock(const std::string &table_id, +Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id, size_t n, const float *vectors, IDNumbers &vector_ids) { @@ -49,7 +49,7 @@ Status NewMemManager::InsertVectorsNoLock(const std::string &table_id, return status; } -Status NewMemManager::ToImmutable() { +Status MemManagerImpl::ToImmutable() { std::unique_lock lock(mutex_); MemIdMap temp_map; for (auto &kv: mem_id_map_) { @@ -65,7 +65,7 @@ Status NewMemManager::ToImmutable() { return Status::OK(); } -Status NewMemManager::Serialize(std::set &table_ids) { +Status MemManagerImpl::Serialize(std::set &table_ids) { ToImmutable(); std::unique_lock lock(serialization_mtx_); table_ids.clear(); @@ -77,7 +77,7 @@ Status NewMemManager::Serialize(std::set &table_ids) { return Status::OK(); } -Status NewMemManager::EraseMemVector(const std::string &table_id) { +Status MemManagerImpl::EraseMemVector(const std::string &table_id) { {//erase MemVector from rapid-insert cache std::unique_lock lock(mutex_); mem_id_map_.erase(table_id); @@ -97,7 +97,7 @@ Status NewMemManager::EraseMemVector(const std::string &table_id) { return Status::OK(); } -size_t NewMemManager::GetCurrentMutableMem() { +size_t MemManagerImpl::GetCurrentMutableMem() { size_t total_mem = 0; for (auto &kv : mem_id_map_) { auto memTable = kv.second; @@ -106,7 +106,7 @@ size_t NewMemManager::GetCurrentMutableMem() { return total_mem; } -size_t NewMemManager::GetCurrentImmutableMem() { +size_t MemManagerImpl::GetCurrentImmutableMem() { size_t total_mem = 0; for (auto &mem_table : immu_mem_list_) { total_mem += mem_table->GetCurrentMem(); @@ -114,7 +114,7 @@ size_t NewMemManager::GetCurrentImmutableMem() { return total_mem; } -size_t NewMemManager::GetCurrentMem() { +size_t MemManagerImpl::GetCurrentMem() { return GetCurrentMutableMem() + GetCurrentImmutableMem(); } diff --git a/cpp/src/db/NewMemManager.h b/cpp/src/db/insert/MemManagerImpl.h similarity index 82% rename from cpp/src/db/NewMemManager.h rename to cpp/src/db/insert/MemManagerImpl.h index 5b933c94..57b3a5f8 100644 --- a/cpp/src/db/NewMemManager.h +++ b/cpp/src/db/insert/MemManagerImpl.h @@ -1,9 +1,9 @@ #pragma once -#include "Meta.h" +#include "db/meta/Meta.h" #include "MemTable.h" -#include "Status.h" -#include "MemManagerAbstract.h" +#include "db/Status.h" +#include "MemManager.h" #include #include @@ -16,13 +16,13 @@ namespace zilliz { namespace milvus { namespace engine { -class NewMemManager : public MemManagerAbstract { +class MemManagerImpl : public MemManager { public: using MetaPtr = meta::Meta::Ptr; - using Ptr = std::shared_ptr; + using Ptr = std::shared_ptr; using MemTablePtr = typename MemTable::Ptr; - NewMemManager(const std::shared_ptr &meta, const Options &options) + MemManagerImpl(const std::shared_ptr &meta, const Options &options) : meta_(meta), options_(options) {} Status InsertVectors(const std::string &table_id, diff --git a/cpp/src/db/MemTable.cpp b/cpp/src/db/insert/MemTable.cpp similarity index 98% rename from cpp/src/db/MemTable.cpp rename to cpp/src/db/insert/MemTable.cpp index e05aa058..38206e25 100644 --- a/cpp/src/db/MemTable.cpp +++ b/cpp/src/db/insert/MemTable.cpp @@ -1,5 +1,5 @@ #include "MemTable.h" -#include "Log.h" +#include "db/Log.h" namespace zilliz { diff --git a/cpp/src/db/MemTable.h b/cpp/src/db/insert/MemTable.h similarity index 95% rename from cpp/src/db/MemTable.h rename to cpp/src/db/insert/MemTable.h index 198fcc22..4f0cdb7d 100644 --- a/cpp/src/db/MemTable.h +++ b/cpp/src/db/insert/MemTable.h @@ -1,6 +1,6 @@ #pragma once -#include "Status.h" +#include "db/Status.h" #include "MemTableFile.h" #include "VectorSource.h" diff --git a/cpp/src/db/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp similarity index 97% rename from cpp/src/db/MemTableFile.cpp rename to cpp/src/db/insert/MemTableFile.cpp index f36223f0..1d7053ab 100644 --- a/cpp/src/db/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -1,7 +1,7 @@ #include "MemTableFile.h" -#include "Constants.h" -#include "Log.h" -#include "EngineFactory.h" +#include "db/Constants.h" +#include "db/Log.h" +#include "db/engine/EngineFactory.h" #include "metrics/Metrics.h" #include diff --git a/cpp/src/db/MemTableFile.h b/cpp/src/db/insert/MemTableFile.h similarity index 87% rename from cpp/src/db/MemTableFile.h rename to cpp/src/db/insert/MemTableFile.h index 4d0011b3..b5821522 100644 --- a/cpp/src/db/MemTableFile.h +++ b/cpp/src/db/insert/MemTableFile.h @@ -1,9 +1,9 @@ #pragma once -#include "Status.h" -#include "Meta.h" +#include "db/Status.h" +#include "db/meta/Meta.h" #include "VectorSource.h" -#include "ExecutionEngine.h" +#include "db/engine/ExecutionEngine.h" namespace zilliz { diff --git a/cpp/src/db/VectorSource.cpp b/cpp/src/db/insert/VectorSource.cpp similarity index 94% rename from cpp/src/db/VectorSource.cpp rename to cpp/src/db/insert/VectorSource.cpp index 74c07ae1..f36eeb09 100644 --- a/cpp/src/db/VectorSource.cpp +++ b/cpp/src/db/insert/VectorSource.cpp @@ -1,7 +1,7 @@ #include "VectorSource.h" -#include "ExecutionEngine.h" -#include "EngineFactory.h" -#include "Log.h" +#include "db/engine/ExecutionEngine.h" +#include "db/engine/EngineFactory.h" +#include "db/Log.h" #include "metrics/Metrics.h" diff --git a/cpp/src/db/VectorSource.h b/cpp/src/db/insert/VectorSource.h similarity index 84% rename from cpp/src/db/VectorSource.h rename to cpp/src/db/insert/VectorSource.h index 7092805a..9792772d 100644 --- a/cpp/src/db/VectorSource.h +++ b/cpp/src/db/insert/VectorSource.h @@ -1,9 +1,9 @@ #pragma once -#include "Status.h" -#include "Meta.h" -#include "IDGenerator.h" -#include "ExecutionEngine.h" +#include "db/Status.h" +#include "db/meta/Meta.h" +#include "db/IDGenerator.h" +#include "db/engine/ExecutionEngine.h" namespace zilliz { diff --git a/cpp/src/db/Meta.cpp b/cpp/src/db/meta/Meta.cpp similarity index 100% rename from cpp/src/db/Meta.cpp rename to cpp/src/db/meta/Meta.cpp diff --git a/cpp/src/db/Meta.h b/cpp/src/db/meta/Meta.h similarity index 98% rename from cpp/src/db/Meta.h rename to cpp/src/db/meta/Meta.h index 7e826f63..e88761b4 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -6,8 +6,8 @@ #pragma once #include "MetaTypes.h" -#include "Options.h" -#include "Status.h" +#include "db/Options.h" +#include "db/Status.h" #include #include diff --git a/cpp/src/db/MetaConsts.h b/cpp/src/db/meta/MetaConsts.h similarity index 100% rename from cpp/src/db/MetaConsts.h rename to cpp/src/db/meta/MetaConsts.h diff --git a/cpp/src/db/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h similarity index 97% rename from cpp/src/db/MetaTypes.h rename to cpp/src/db/meta/MetaTypes.h index d7039685..9f58734f 100644 --- a/cpp/src/db/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -5,7 +5,7 @@ ******************************************************************************/ #pragma once -#include "ExecutionEngine.h" +#include "db/engine/ExecutionEngine.h" #include #include diff --git a/cpp/src/db/MySQLConnectionPool.cpp b/cpp/src/db/meta/MySQLConnectionPool.cpp similarity index 100% rename from cpp/src/db/MySQLConnectionPool.cpp rename to cpp/src/db/meta/MySQLConnectionPool.cpp diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/meta/MySQLConnectionPool.h similarity index 97% rename from cpp/src/db/MySQLConnectionPool.h rename to cpp/src/db/meta/MySQLConnectionPool.h index 5112993b..62afd2dd 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/meta/MySQLConnectionPool.h @@ -4,7 +4,7 @@ #include #include -#include "Log.h" +#include "db/Log.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp similarity index 99% rename from cpp/src/db/MySQLMetaImpl.cpp rename to cpp/src/db/meta/MySQLMetaImpl.cpp index e9835f24..bf458935 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -4,11 +4,11 @@ * Proprietary and confidential. ******************************************************************************/ #include "MySQLMetaImpl.h" -#include "IDGenerator.h" -#include "Utils.h" -#include "Log.h" +#include "db/IDGenerator.h" +#include "db/Utils.h" +#include "db/Log.h" #include "MetaConsts.h" -#include "Factories.h" +#include "db/Factories.h" #include "metrics/Metrics.h" #include diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h similarity index 99% rename from cpp/src/db/MySQLMetaImpl.h rename to cpp/src/db/meta/MySQLMetaImpl.h index 7822b99f..30695423 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -6,7 +6,7 @@ #pragma once #include "Meta.h" -#include "Options.h" +#include "db/Options.h" #include "MySQLConnectionPool.h" #include "mysql++/mysql++.h" diff --git a/cpp/src/db/meta/DBMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp similarity index 95% rename from cpp/src/db/meta/DBMetaImpl.cpp rename to cpp/src/db/meta/SqliteMetaImpl.cpp index c629f378..9411a5e8 100644 --- a/cpp/src/db/meta/DBMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -3,12 +3,12 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ -#include "DBMetaImpl.h" -#include "IDGenerator.h" -#include "Utils.h" -#include "Log.h" +#include "SqliteMetaImpl.h" +#include "db/IDGenerator.h" +#include "db/Utils.h" +#include "db/Log.h" #include "MetaConsts.h" -#include "Factories.h" +#include "db/Factories.h" #include "metrics/Metrics.h" #include @@ -83,7 +83,7 @@ using ConnectorT = decltype(StoragePrototype("")); static std::unique_ptr ConnectorPtr; using ConditionT = decltype(c(&TableFileSchema::id_) == 1UL); -Status DBMetaImpl::NextTableId(std::string &table_id) { +Status SqliteMetaImpl::NextTableId(std::string &table_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -91,7 +91,7 @@ Status DBMetaImpl::NextTableId(std::string &table_id) { return Status::OK(); } -Status DBMetaImpl::NextFileId(std::string &file_id) { +Status SqliteMetaImpl::NextFileId(std::string &file_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -99,12 +99,12 @@ Status DBMetaImpl::NextFileId(std::string &file_id) { return Status::OK(); } -DBMetaImpl::DBMetaImpl(const DBMetaOptions &options_) +SqliteMetaImpl::SqliteMetaImpl(const DBMetaOptions &options_) : options_(options_) { Initialize(); } -Status DBMetaImpl::Initialize() { +Status SqliteMetaImpl::Initialize() { if (!boost::filesystem::is_directory(options_.path)) { auto ret = boost::filesystem::create_directory(options_.path); if (!ret) { @@ -125,7 +125,7 @@ Status DBMetaImpl::Initialize() { } // PXU TODO: Temp solution. Will fix later -Status DBMetaImpl::DropPartitionsByDates(const std::string &table_id, +Status SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id, const DatesT &dates) { if (dates.size() == 0) { return Status::OK(); @@ -165,7 +165,7 @@ Status DBMetaImpl::DropPartitionsByDates(const std::string &table_id, return Status::OK(); } -Status DBMetaImpl::CreateTable(TableSchema &table_schema) { +Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) { try { MetricCollector metric; @@ -209,7 +209,7 @@ Status DBMetaImpl::CreateTable(TableSchema &table_schema) { return Status::OK(); } -Status DBMetaImpl::DeleteTable(const std::string& table_id) { +Status SqliteMetaImpl::DeleteTable(const std::string& table_id) { try { MetricCollector metric; @@ -244,7 +244,7 @@ Status DBMetaImpl::DeleteTable(const std::string& table_id) { return Status::OK(); } -Status DBMetaImpl::DeleteTableFiles(const std::string& table_id) { +Status SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) { try { MetricCollector metric; @@ -269,7 +269,7 @@ Status DBMetaImpl::DeleteTableFiles(const std::string& table_id) { return Status::OK(); } -Status DBMetaImpl::DescribeTable(TableSchema &table_schema) { +Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { try { MetricCollector metric; @@ -299,7 +299,7 @@ Status DBMetaImpl::DescribeTable(TableSchema &table_schema) { return Status::OK(); } -Status DBMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) { +Status SqliteMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) { has = false; try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, @@ -353,7 +353,7 @@ Status DBMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) { return Status::OK(); } -Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { +Status SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { has_or_not = false; try { @@ -374,7 +374,7 @@ Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { return Status::OK(); } -Status DBMetaImpl::AllTables(std::vector& table_schema_array) { +Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { try { MetricCollector metric; @@ -404,7 +404,7 @@ Status DBMetaImpl::AllTables(std::vector& table_schema_array) { return Status::OK(); } -Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) { +Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { if (file_schema.date_ == EmptyDate) { file_schema.date_ = Meta::GetDate(); } @@ -440,7 +440,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) { return Status::OK(); } -Status DBMetaImpl::FilesToIndex(TableFilesSchema &files) { +Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { files.clear(); try { @@ -490,7 +490,7 @@ Status DBMetaImpl::FilesToIndex(TableFilesSchema &files) { return Status::OK(); } -Status DBMetaImpl::FilesToSearch(const std::string &table_id, +Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, const DatesT &partition, DatePartionedTableFilesSchema &files) { files.clear(); @@ -589,7 +589,7 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id, return Status::OK(); } -Status DBMetaImpl::FilesToSearch(const std::string &table_id, +Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, const std::vector &ids, const DatesT &partition, DatePartionedTableFilesSchema &files) { @@ -662,7 +662,7 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id, return Status::OK(); } -Status DBMetaImpl::FilesToMerge(const std::string &table_id, +Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) { files.clear(); @@ -710,7 +710,7 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id, return Status::OK(); } -Status DBMetaImpl::GetTableFiles(const std::string& table_id, +Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, const std::vector& ids, TableFilesSchema& table_files) { try { @@ -754,7 +754,7 @@ Status DBMetaImpl::GetTableFiles(const std::string& table_id, } // PXU TODO: Support Swap -Status DBMetaImpl::Archive() { +Status SqliteMetaImpl::Archive() { auto &criterias = options_.archive_conf.GetCriterias(); if (criterias.size() == 0) { return Status::OK(); @@ -794,7 +794,7 @@ Status DBMetaImpl::Archive() { return Status::OK(); } -Status DBMetaImpl::Size(uint64_t &result) { +Status SqliteMetaImpl::Size(uint64_t &result) { result = 0; try { auto files = ConnectorPtr->select(columns(&TableFileSchema::size_, @@ -821,7 +821,7 @@ Status DBMetaImpl::Size(uint64_t &result) { return Status::OK(); } -Status DBMetaImpl::DiscardFiles(long to_discard_size) { +Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { if (to_discard_size <= 0) { return Status::OK(); } @@ -883,7 +883,7 @@ Status DBMetaImpl::DiscardFiles(long to_discard_size) { return DiscardFiles(to_discard_size); } -Status DBMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { +Status SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); try { MetricCollector metric; @@ -910,7 +910,7 @@ Status DBMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { return Status::OK(); } -Status DBMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { +Status SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { try { MetricCollector metric; @@ -932,7 +932,7 @@ Status DBMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { return Status::OK(); } -Status DBMetaImpl::UpdateTableFiles(TableFilesSchema &files) { +Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { try { MetricCollector metric; @@ -977,7 +977,7 @@ Status DBMetaImpl::UpdateTableFiles(TableFilesSchema &files) { return Status::OK(); } -Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { +Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { auto now = utils::GetMicroSecTimeStamp(); try { MetricCollector metric; @@ -1052,7 +1052,7 @@ Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return Status::OK(); } -Status DBMetaImpl::CleanUp() { +Status SqliteMetaImpl::CleanUp() { try { MetricCollector metric; @@ -1086,7 +1086,7 @@ Status DBMetaImpl::CleanUp() { return Status::OK(); } -Status DBMetaImpl::Count(const std::string &table_id, uint64_t &result) { +Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { try { MetricCollector metric; @@ -1120,14 +1120,14 @@ Status DBMetaImpl::Count(const std::string &table_id, uint64_t &result) { return Status::OK(); } -Status DBMetaImpl::DropAll() { +Status SqliteMetaImpl::DropAll() { if (boost::filesystem::is_directory(options_.path)) { boost::filesystem::remove_all(options_.path); } return Status::OK(); } -DBMetaImpl::~DBMetaImpl() { +SqliteMetaImpl::~SqliteMetaImpl() { CleanUp(); } diff --git a/cpp/src/db/meta/DBMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h similarity index 94% rename from cpp/src/db/meta/DBMetaImpl.h rename to cpp/src/db/meta/SqliteMetaImpl.h index 7f2a87fc..1525f27e 100644 --- a/cpp/src/db/meta/DBMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -6,7 +6,7 @@ #pragma once #include "Meta.h" -#include "Options.h" +#include "db/Options.h" #include @@ -17,9 +17,9 @@ namespace meta { auto StoragePrototype(const std::string &path); -class DBMetaImpl : public Meta { +class SqliteMetaImpl : public Meta { public: - explicit DBMetaImpl(const DBMetaOptions &options_); + explicit SqliteMetaImpl(const DBMetaOptions &options_); Status CreateTable(TableSchema &table_schema) override; @@ -91,7 +91,7 @@ class DBMetaImpl : public Meta { Status Count(const std::string &table_id, uint64_t &result) override; - ~DBMetaImpl() override; + ~SqliteMetaImpl() override; private: Status NextFileId(std::string &file_id); diff --git a/cpp/src/db/scheduler/TaskScheduler.cpp b/cpp/src/db/scheduler/TaskScheduler.cpp index f29c3d4c..28d46da7 100644 --- a/cpp/src/db/scheduler/TaskScheduler.cpp +++ b/cpp/src/db/scheduler/TaskScheduler.cpp @@ -8,7 +8,7 @@ #include "TaskDispatchQueue.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" -#include "db/EngineFactory.h" +#include "db/engine/EngineFactory.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/db/scheduler/context/DeleteContext.h b/cpp/src/db/scheduler/context/DeleteContext.h index b909ecb6..efcc3bb6 100644 --- a/cpp/src/db/scheduler/context/DeleteContext.h +++ b/cpp/src/db/scheduler/context/DeleteContext.h @@ -6,7 +6,7 @@ #pragma once #include "IScheduleContext.h" -#include "db/Meta.h" +#include "db/meta/Meta.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/db/scheduler/context/SearchContext.h b/cpp/src/db/scheduler/context/SearchContext.h index 3cea23d9..466fc435 100644 --- a/cpp/src/db/scheduler/context/SearchContext.h +++ b/cpp/src/db/scheduler/context/SearchContext.h @@ -6,7 +6,7 @@ #pragma once #include "IScheduleContext.h" -#include "db/MetaTypes.h" +#include "db/meta/MetaTypes.h" #include #include diff --git a/cpp/src/db/scheduler/task/IndexLoadTask.cpp b/cpp/src/db/scheduler/task/IndexLoadTask.cpp index f3aacf83..2c319396 100644 --- a/cpp/src/db/scheduler/task/IndexLoadTask.cpp +++ b/cpp/src/db/scheduler/task/IndexLoadTask.cpp @@ -7,7 +7,7 @@ #include "IndexLoadTask.h" #include "SearchTask.h" #include "db/Log.h" -#include "db/EngineFactory.h" +#include "db/engine/EngineFactory.h" #include "utils/TimeRecorder.h" #include "metrics/Metrics.h" diff --git a/cpp/src/db/scheduler/task/SearchTask.h b/cpp/src/db/scheduler/task/SearchTask.h index 7e0aa52a..034b53d4 100644 --- a/cpp/src/db/scheduler/task/SearchTask.h +++ b/cpp/src/db/scheduler/task/SearchTask.h @@ -7,7 +7,7 @@ #include "IScheduleTask.h" #include "db/scheduler/context/SearchContext.h" -#include "db/ExecutionEngine.h" +#include "db/engine/ExecutionEngine.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index 5bd09bd0..fdde4b15 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -6,7 +6,6 @@ #pragma once #include "db/DB.h" -#include "db/Meta.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index bfdf35ce..fa0cd839 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -1,4 +1,4 @@ -#include "db/ExecutionEngine.h" +#include "db/engine/ExecutionEngine.h" #include "ValidationUtil.h" #include "Log.h" diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 458ad758..aa96fa6e 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -40,7 +40,6 @@ set(unittest_libs add_subdirectory(server) add_subdirectory(db) -add_subdirectory(index_wrapper) +add_subdirectory(knowhere) add_subdirectory(metrics) -add_subdirectory(storage) -add_subdirectory(utils) \ No newline at end of file +#add_subdirectory(storage) \ No newline at end of file diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 6066c410..6c451bf2 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -3,10 +3,12 @@ # Unauthorized copying of this file, via any medium is strictly prohibited. # Proprietary and confidential. #------------------------------------------------------------------------------- -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) +aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) -#aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src) aux_source_directory(./ test_srcs) @@ -32,14 +34,17 @@ include_directories(/usr/include/mysql) set(db_test_src ${config_files} ${cache_srcs} - ${db_srcs} + ${db_main_files} + ${db_engine_files} + ${db_insert_files} + ${db_meta_files} ${db_scheduler_srcs} ${wrapper_src} ${knowhere_src} ${util_files} ${require_files} ${test_srcs} - ) + ) cuda_add_executable(db_test ${db_test_src}) diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 81f2279d..19d7828c 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -6,7 +6,7 @@ #include "utils.h" #include "db/DB.h" #include "db/DBImpl.h" -#include "db/MetaConsts.h" +#include "db/meta/MetaConsts.h" #include "db/Factories.h" #include diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 17d87030..b410b0bc 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -1,16 +1,16 @@ #include "gtest/gtest.h" -#include "db/VectorSource.h" -#include "db/MemTableFile.h" -#include "db/MemTable.h" +#include "db/insert/VectorSource.h" +#include "db/insert/MemTableFile.h" +#include "db/insert/MemTable.h" #include "utils.h" #include "db/Factories.h" #include "db/Constants.h" -#include "db/EngineFactory.h" +#include "db/engine/EngineFactory.h" #include "metrics/Metrics.h" -#include "db/MetaConsts.h" -#include "boost/filesystem.hpp" +#include "db/meta/MetaConsts.h" +#include #include #include #include @@ -47,7 +47,7 @@ void BuildVectors(int64_t n, std::vector &vectors) { TEST_F(NewMemManagerTest, VECTOR_SOURCE_TEST) { - std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); + std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); engine::meta::TableSchema table_schema = BuildTableSchema(); auto status = impl_->CreateTable(table_schema); @@ -90,7 +90,7 @@ TEST_F(NewMemManagerTest, VECTOR_SOURCE_TEST) { TEST_F(NewMemManagerTest, MEM_TABLE_FILE_TEST) { - std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); + std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); auto options = engine::OptionsFactory::Build(); engine::meta::TableSchema table_schema = BuildTableSchema(); @@ -134,7 +134,7 @@ TEST_F(NewMemManagerTest, MEM_TABLE_FILE_TEST) { TEST_F(NewMemManagerTest, MEM_TABLE_TEST) { - std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); + std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); auto options = engine::OptionsFactory::Build(); engine::meta::TableSchema table_schema = BuildTableSchema(); diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 36643ead..0e981f6a 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -10,10 +10,10 @@ #include #include "utils.h" -#include "db/DBMetaImpl.h" +#include "db/meta/SqliteMetaImpl.h" #include "db/Factories.h" #include "db/Utils.h" -#include "db/MetaConsts.h" +#include "db/meta/MetaConsts.h" using namespace zilliz::milvus::engine; @@ -113,7 +113,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { ss << "days:" << days_num; options.archive_conf = ArchiveConf("delete", ss.str()); - meta::DBMetaImpl impl(options); + meta::SqliteMetaImpl impl(options); auto table_id = "meta_test_table"; meta::TableSchema table; @@ -163,7 +163,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { options.path = "/tmp/milvus_test"; options.archive_conf = ArchiveConf("delete", "disk:11"); - meta::DBMetaImpl impl(options); + meta::SqliteMetaImpl impl(options); auto table_id = "meta_test_group"; meta::TableSchema table; diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index d968b69b..6433a224 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -11,8 +11,8 @@ #include "db/Exception.h" #include "db/Status.h" #include "db/Options.h" -#include "db/DBMetaImpl.h" -#include "db/EngineFactory.h" +#include "db/meta/SqliteMetaImpl.h" +#include "db/engine/EngineFactory.h" #include "db/Utils.h" #include @@ -100,7 +100,7 @@ TEST(DBMiscTest, OPTIONS_TEST) { TEST(DBMiscTest, META_TEST) { engine::DBMetaOptions options; options.path = "/tmp/milvus_test"; - engine::meta::DBMetaImpl impl(options); + engine::meta::SqliteMetaImpl impl(options); time_t tt; time( &tt ); diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 21160eed..c3752438 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -6,7 +6,7 @@ #include "utils.h" #include "db/DB.h" #include "db/DBImpl.h" -#include "db/MetaConsts.h" +#include "db/meta/MetaConsts.h" #include "db/Factories.h" #include diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 9e4b817b..059fe9a2 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -10,10 +10,10 @@ #include #include "utils.h" -#include "db/MySQLMetaImpl.h" +#include "db/meta/MySQLMetaImpl.h" #include "db/Factories.h" #include "db/Utils.h" -#include "db/MetaConsts.h" +#include "db/meta/MetaConsts.h" #include "mysql++/mysql++.h" diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index 08d1caaa..4d3e318f 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -11,8 +11,8 @@ //#include #include "db/DB.h" -#include "db/DBMetaImpl.h" -#include "db/MySQLMetaImpl.h" +#include "db/meta/SqliteMetaImpl.h" +#include "db/meta/MySQLMetaImpl.h" #define TIMING @@ -71,7 +71,7 @@ class DBTest2 : public DBTest { class MetaTest : public DBTest { protected: - std::shared_ptr impl_; + std::shared_ptr impl_; virtual void SetUp() override; virtual void TearDown() override; diff --git a/cpp/unittest/index_wrapper/CMakeLists.txt b/cpp/unittest/knowhere/CMakeLists.txt similarity index 100% rename from cpp/unittest/index_wrapper/CMakeLists.txt rename to cpp/unittest/knowhere/CMakeLists.txt diff --git a/cpp/unittest/index_wrapper/knowhere_test.cpp b/cpp/unittest/knowhere/knowhere_test.cpp similarity index 100% rename from cpp/unittest/index_wrapper/knowhere_test.cpp rename to cpp/unittest/knowhere/knowhere_test.cpp diff --git a/cpp/unittest/index_wrapper/utils.cpp b/cpp/unittest/knowhere/utils.cpp similarity index 100% rename from cpp/unittest/index_wrapper/utils.cpp rename to cpp/unittest/knowhere/utils.cpp diff --git a/cpp/unittest/index_wrapper/utils.h b/cpp/unittest/knowhere/utils.h similarity index 100% rename from cpp/unittest/index_wrapper/utils.h rename to cpp/unittest/knowhere/utils.h diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 02fe3009..47b86d11 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -11,8 +11,10 @@ include_directories(../../src) - -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) +aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) @@ -44,7 +46,10 @@ include_directories(/usr/include/mysql) set(count_test_src ${config_files} ${cache_srcs} - ${db_srcs} + ${db_main_files} + ${db_engine_files} + ${db_insert_files} + ${db_meta_files} ${db_scheduler_srcs} ${wrapper_src} ${knowhere_src} diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index 883e63ed..db9b06b7 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -17,7 +17,7 @@ #include "metrics/Metrics.h" #include "utils.h" #include "db/DB.h" -#include "db/DBMetaImpl.h" +#include "db/meta/SqliteMetaImpl.h" #include "db/Factories.h" diff --git a/cpp/unittest/metrics/utils.h b/cpp/unittest/metrics/utils.h index 1badce00..4980b91a 100644 --- a/cpp/unittest/metrics/utils.h +++ b/cpp/unittest/metrics/utils.h @@ -11,8 +11,8 @@ //#include #include "db/DB.h" -#include "db/DBMetaImpl.h" -#include "db/MySQLMetaImpl.h" +#include "db/meta/SqliteMetaImpl.h" +#include "db/meta/MySQLMetaImpl.h" #define TIMING diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index c60e2dbe..33c7fcde 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -3,15 +3,15 @@ # Unauthorized copying of this file, via any medium is strictly prohibited. # Proprietary and confidential. #------------------------------------------------------------------------------- -include_directories(../../src) +include_directories(${MILVUS_ENGINE_SRC}/) include_directories(/usr/include) include_directories(/usr/local/cuda/include) link_directories(/usr/local/cuda/lib64) -aux_source_directory(../../src/config config_files) -aux_source_directory(../../src/cache cache_srcs) -aux_source_directory(../../src/wrapper wrapper_src) +aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) +aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) aux_source_directory(./ test_srcs) set(utils_srcs diff --git a/cpp/unittest/server/log_test.cpp b/cpp/unittest/server/log_test.cpp deleted file mode 100644 index 22410f78..00000000 --- a/cpp/unittest/server/log_test.cpp +++ /dev/null @@ -1,20 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// -#include -#include "easylogging++.h" - -class LogTest: public testing::Test { -protected: - void SetUp() override { - el::Configurations conf("../../../conf/log_config.conf"); - el::Loggers::reconfigureAllLoggers(conf); - } -}; - -TEST_F(LogTest, TEST) { - EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer)); - EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::LogDetailedCrashReason)); -} \ No newline at end of file diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index e4948fc5..ea598472 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -12,6 +12,8 @@ #include "utils/TimeRecorder.h" #include "utils/BlockingQueue.h" #include "utils/LogUtil.h" +#include "utils/ValidationUtil.h" +#include "db/engine/ExecutionEngine.h" using namespace zilliz::milvus; @@ -148,6 +150,57 @@ TEST(UtilTest, LOG_TEST) { int32_t res = server::InitLog(LOG_FILE_PATH); ASSERT_EQ(res, 0); + EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer)); + EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::LogDetailedCrashReason)); + std::string fname = server::GetFileName(LOG_FILE_PATH); ASSERT_EQ(fname, "log_config.conf"); } + +TEST(UtilTest, VALIDATE_TABLENAME_TEST) { + std::string table_name = "Normal123_"; + server:: ServerError res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_SUCCESS); + + table_name = "12sds"; + res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + + table_name = ""; + res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + + table_name = "_asdasd"; + res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_SUCCESS); + + table_name = "!@#!@"; + res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + + table_name = "中文"; + res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + + + table_name = std::string('a', 32768); + res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); +} + +TEST(UtilTest, VALIDATE_DIMENSIONTEST) { + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(-1), server::SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(0), server::SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16385), server::SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16384), server::SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(1), server::SERVER_SUCCESS); +} + +TEST(UtilTest, VALIDATE_INDEXTYPE_TEST) { + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), server::SERVER_INVALID_INDEX_TYPE); + for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) { + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType(i), server::SERVER_SUCCESS); + } + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), server::SERVER_INVALID_INDEX_TYPE); +} + diff --git a/cpp/unittest/utils/CMakeLists.txt b/cpp/unittest/utils/CMakeLists.txt deleted file mode 100644 index 61e44594..00000000 --- a/cpp/unittest/utils/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -#------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. -#------------------------------------------------------------------------------- - -# Make sure that your call to link_directories takes place before your call to the relevant add_executable. -include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") -link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") - -set(validation_util_src - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.h) - -set(validation_util_test_src - ${unittest_srcs} - ${validation_util_src} - ${require_files} - ValidationUtilTest.cpp - ) - -add_executable(valication_util_test - ${validation_util_test_src} - ${config_files}) - -target_link_libraries(valication_util_test - ${unittest_libs} - boost_filesystem_static) - -install(TARGETS valication_util_test DESTINATION bin) \ No newline at end of file diff --git a/cpp/unittest/utils/ValidationUtilTest.cpp b/cpp/unittest/utils/ValidationUtilTest.cpp deleted file mode 100644 index 871172e6..00000000 --- a/cpp/unittest/utils/ValidationUtilTest.cpp +++ /dev/null @@ -1,63 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// -#include - -#include "utils/ValidationUtil.h" -#include "utils/Error.h" -#include "db/ExecutionEngine.h" - -#include - -using namespace zilliz::milvus; -using namespace zilliz::milvus::server; - -TEST(ValidationUtilTest, TableNameTest) { - std::string table_name = "Normal123_"; - ServerError res = ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_SUCCESS); - - table_name = "12sds"; - res = ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); - - table_name = ""; - res = ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); - - table_name = "_asdasd"; - res = ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_SUCCESS); - - table_name = "!@#!@"; - res = ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); - - table_name = "中文"; - res = ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); - - - table_name = std::string('a', 32768); - res = ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); -} - - -TEST(ValidationUtilTest, TableDimensionTest) { - ASSERT_EQ(ValidationUtil::ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ValidationUtil::ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ValidationUtil::ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ValidationUtil::ValidateTableDimension(16384), SERVER_SUCCESS); - ASSERT_EQ(ValidationUtil::ValidateTableDimension(1), SERVER_SUCCESS); -} - -TEST(ValidationUtilTest, TableIndexTypeTest) { - ASSERT_EQ(ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE); - for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) { - ASSERT_EQ(ValidationUtil::ValidateTableIndexType(i), SERVER_SUCCESS); - } - ASSERT_EQ(ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE); -} -- GitLab