Merge branch 'branch-0.3.1' into 'branch-0.3.1'

MS-249 Check machine hardware during initialize

See merge request megasearch/milvus!257

Former-commit-id: aa3222dfeead108fb711cb0b3e922de8e2df148e

cpp/CHANGELOG.md

@@ -35,7 +35,8 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-248 - Support AddVector/SearchVector profiling
 - MS-256 - Add more cache config
 - MS-260 - Refine log
-- MS-261 - update faiss version to 1.5.3 and add BUILD_FAISS_WITH_MKL as an option
+- MS-249 - Check machine hardware during initialize
+- MS-261 - Update faiss version to 1.5.3 and add BUILD_FAISS_WITH_MKL as an option
 
 ## New Feature
 - MS-180 - Add new mem manager

cpp/conf/server_config.template

@@ -17,7 +17,7 @@ db_config:
   archive_disk_threshold: 0        # triger archive action if storage size exceed this value, 0 means no limit, unit: GB
   archive_days_threshold: 0        # files older than x days will be archived, 0 means no limit, unit: day
   insert_buffer_size: 4            # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.
-                                   # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory
+                                   # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB
 
 metric_config:
   is_startup: off      # if monitoring start: on, off

cpp/src/CMakeLists.txt

@@ -87,6 +87,7 @@ set(third_party_libs
         mysqlpp
         ${PROFILER_LIB}
         ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
+        cudart
         )
 
 if (MEGASEARCH_WITH_ARROW STREQUAL "ON")

cpp/src/server/RequestTask.cpp

@@ -162,17 +162,17 @@ ServerError CreateTableTask::OnExecute() {
     try {
         //step 1: check arguments
         ServerError res = SERVER_SUCCESS;
-        res = ValidateTableName(schema_.table_name);
+        res = ValidationUtil::ValidateTableName(schema_.table_name);
         if(res != SERVER_SUCCESS) {
             return SetError(res, "Invalid table name: " + schema_.table_name);
         }
 
-        res = ValidateTableDimension(schema_.dimension);
+        res = ValidationUtil::ValidateTableDimension(schema_.dimension);
         if(res != SERVER_SUCCESS) {
             return SetError(res, "Invalid table dimension: " + std::to_string(schema_.dimension));
         }
 
-        res = ValidateTableIndexType(schema_.index_type);
+        res = ValidationUtil::ValidateTableIndexType(schema_.index_type);
         if(res != SERVER_SUCCESS) {
             return SetError(res, "Invalid index type: " + std::to_string(schema_.index_type));
         }
@@ -217,7 +217,7 @@ ServerError DescribeTableTask::OnExecute() {
     try {
         //step 1: check arguments
         ServerError res = SERVER_SUCCESS;
-        res = ValidateTableName(table_name_);
+        res = ValidationUtil::ValidateTableName(table_name_);
         if(res != SERVER_SUCCESS) {
             return SetError(res, "Invalid table name: " + table_name_);
         }
@@ -260,7 +260,7 @@ ServerError BuildIndexTask::OnExecute() {
 
         //step 1: check arguments
         ServerError res = SERVER_SUCCESS;
-        res = ValidateTableName(table_name_);
+        res = ValidationUtil::ValidateTableName(table_name_);
         if(res != SERVER_SUCCESS) {
             return SetError(res, "Invalid table name: " + table_name_);
         }
@@ -303,7 +303,7 @@ ServerError HasTableTask::OnExecute() {
 
         //step 1: check arguments
         ServerError res = SERVER_SUCCESS;
-        res = ValidateTableName(table_name_);
+        res = ValidationUtil::ValidateTableName(table_name_);
         if(res != SERVER_SUCCESS) {
             return SetError(res, "Invalid table name: " + table_name_);
         }
@@ -339,7 +339,7 @@ ServerError DeleteTableTask::OnExecute() {
 
         //step 1: check arguments
         ServerError res = SERVER_SUCCESS;
-        res = ValidateTableName(table_name_);
+        res = ValidationUtil::ValidateTableName(table_name_);
         if(res != SERVER_SUCCESS) {
             return SetError(res, "Invalid table name: " + table_name_);
         }
@@ -420,7 +420,7 @@ ServerError AddVectorTask::OnExecute() {
 
         //step 1: check arguments
         ServerError res = SERVER_SUCCESS;
-        res = ValidateTableName(table_name_);
+        res = ValidationUtil::ValidateTableName(table_name_);
         if(res != SERVER_SUCCESS) {
             return SetError(res, "Invalid table name: " + table_name_);
         }
@@ -508,7 +508,7 @@ ServerError SearchVectorTaskBase::OnExecute() {
 
         //step 1: check arguments
         ServerError res = SERVER_SUCCESS;
-        res = ValidateTableName(table_name_);
+        res = ValidationUtil::ValidateTableName(table_name_);
         if(res != SERVER_SUCCESS) {
             return SetError(res, "Invalid table name: " + table_name_);
         }
@@ -720,7 +720,7 @@ ServerError GetTableRowCountTask::OnExecute() {
 
         //step 1: check arguments
         ServerError res = SERVER_SUCCESS;
-        res = ValidateTableName(table_name_);
+        res = ValidationUtil::ValidateTableName(table_name_);
         if(res != SERVER_SUCCESS) {
             return SetError(res, "Invalid table name: " + table_name_);
         }

cpp/src/server/Server.cpp

@@ -226,6 +226,10 @@ Server::Stop() {
 ServerError
 Server::LoadConfig() {
     ServerConfig::GetInstance().LoadConfigFile(config_filename_);
+    ServerError err = ServerConfig::GetInstance().ValidateConfig();
+    if(err != SERVER_SUCCESS){
+        exit(0);
+    }
 
     return SERVER_SUCCESS;
 }

cpp/src/server/ServerConfig.cpp

@@ -12,11 +12,16 @@
 #include <iostream>
 
 #include "config/IConfigMgr.h"
+#include "utils/CommonUtil.h"
+#include "utils/ValidationUtil.h"
 
 namespace zilliz {
 namespace milvus {
 namespace server {
 
+constexpr uint64_t MB = 1024*1024;
+constexpr uint64_t GB = MB*1024;
+
 ServerConfig&
 ServerConfig::GetInstance() {
     static ServerConfig config;
@@ -53,6 +58,65 @@ ServerConfig::LoadConfigFile(const std::string& config_filename) {
     return SERVER_SUCCESS;
 }
 
+ServerError ServerConfig::ValidateConfig() const {
+    //server config validation
+    ConfigNode server_config = GetConfig(CONFIG_SERVER);
+    uint32_t gpu_index = (uint32_t)server_config.GetInt32Value(CONFIG_GPU_INDEX);
+    if(ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) {
+        std::cout << "Error: invalid gpu_index " << std::to_string(gpu_index) << std::endl;
+        return SERVER_INVALID_ARGUMENT;
+    }
+
+    //db config validation
+    unsigned long total_mem = 0, free_mem = 0;
+    CommonUtil::GetSystemMemInfo(total_mem, free_mem);
+
+    ConfigNode db_config = GetConfig(CONFIG_DB);
+    uint64_t insert_buffer_size = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INSERT_BUFFER_SIZE);
+    insert_buffer_size *= GB;
+    if(insert_buffer_size >= total_mem) {
+        std::cout << "Error: insert_buffer_size execeed system memory" << std::endl;
+        return SERVER_INVALID_ARGUMENT;
+    }
+
+    uint64_t index_building_threshold = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INDEX_TRIGGER_SIZE);
+    index_building_threshold *= MB;
+
+    size_t gpu_mem = 0;
+    ValidationUtil::GetGpuMemory(gpu_index, gpu_mem);
+    if(index_building_threshold >= gpu_mem) {
+        std::cout << "Error: index_building_threshold execeed gpu memory" << std::endl;
+        return SERVER_INVALID_ARGUMENT;
+    } else if(index_building_threshold >= gpu_mem/3) {
+        std::cout << "Warnning: index_building_threshold is greater than 1/3 of gpu memory, "
+            << "some index type(such as IVFLAT) may cause cuda::bad_alloc() error" << std::endl;
+    }
+
+    //cache config validation
+    ConfigNode cache_config = GetConfig(CONFIG_CACHE);
+    uint64_t cache_cap = (uint64_t)cache_config.GetInt64Value(CONFIG_CPU_CACHE_CAPACITY);
+    cache_cap *= GB;
+    if(cache_cap >= total_mem) {
+        std::cout << "Error: cpu_cache_capacity execeed system memory" << std::endl;
+        return SERVER_INVALID_ARGUMENT;
+    } if(cache_cap > (double)total_mem*0.9) {
+        std::cout << "Warnning: cpu_cache_capacity value is too aggressive" << std::endl;
+    }
+
+    if(insert_buffer_size + cache_cap >= total_mem) {
+        std::cout << "Error: sum of cpu_cache_capacity and insert_buffer_size execeed system memory" << std::endl;
+        return SERVER_INVALID_ARGUMENT;
+    }
+
+    double free_percent = cache_config.GetDoubleValue(server::CACHE_FREE_PERCENT);
+    if(free_percent < std::numeric_limits<double>::epsilon() || free_percent > 1.0) {
+        std::cout << "Error: invalid cache_free_percent " << std::to_string(free_percent) << std::endl;
+        return SERVER_INVALID_ARGUMENT;
+    }
+
+    return SERVER_SUCCESS;
+}
+
 void
 ServerConfig::PrintAll() const {
     if(const IConfigMgr* mgr = IConfigMgr::GetInstance()) {

cpp/src/server/ServerConfig.h

@@ -19,6 +19,7 @@ static const std::string CONFIG_SERVER_ADDRESS = "address";
 static const std::string CONFIG_SERVER_PORT = "port";
 static const std::string CONFIG_SERVER_PROTOCOL = "transfer_protocol";
 static const std::string CONFIG_CLUSTER_MODE = "mode";
+static const std::string CONFIG_GPU_INDEX = "gpu_index";
 
 static const std::string CONFIG_DB = "db_config";
 static const std::string CONFIG_DB_URL = "db_backend_url";
@@ -57,6 +58,7 @@ class ServerConfig {
     static ServerConfig &GetInstance();
 
     ServerError LoadConfigFile(const std::string& config_filename);
+    ServerError ValidateConfig() const;
     void PrintAll() const;
 
     ConfigNode GetConfig(const std::string& name) const;

cpp/src/utils/ValidationUtil.cpp

-#include <src/db/ExecutionEngine.h>
+#include "db/ExecutionEngine.h"
 #include "ValidationUtil.h"
 #include "Log.h"
 
+#include <cuda_runtime.h>
 
 namespace zilliz {
 namespace milvus {
@@ -11,7 +12,7 @@ constexpr size_t table_name_size_limit = 255;
 constexpr int64_t table_dimension_limit = 16384;
 
 ServerError
-ValidateTableName(const std::string &table_name) {
+ValidationUtil::ValidateTableName(const std::string &table_name) {
 
     // Table name shouldn't be empty.
     if (table_name.empty()) {
@@ -45,7 +46,7 @@ ValidateTableName(const std::string &table_name) {
 }
 
 ServerError
-ValidateTableDimension(int64_t dimension) {
+ValidationUtil::ValidateTableDimension(int64_t dimension) {
     if (dimension <= 0 || dimension > table_dimension_limit) {
         SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << table_dimension_limit;
         return SERVER_INVALID_VECTOR_DIMENSION;
@@ -55,7 +56,7 @@ ValidateTableDimension(int64_t dimension) {
 }
 
 ServerError
-ValidateTableIndexType(int32_t index_type) {
+ValidationUtil::ValidateTableIndexType(int32_t index_type) {
     int engine_type = (int)engine::EngineType(index_type);
     if(engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) {
         return SERVER_INVALID_INDEX_TYPE;
@@ -64,6 +65,35 @@ ValidateTableIndexType(int32_t index_type) {
     return SERVER_SUCCESS;
 }
 
+ServerError
+ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
+    int num_devices = 0;
+    auto cuda_err = cudaGetDeviceCount(&num_devices);
+    if (cuda_err) {
+        SERVER_LOG_ERROR << "Failed to count video card: " << std::to_string(cuda_err);
+        return SERVER_UNEXPECTED_ERROR;
+    }
+
+    if(gpu_index >= num_devices) {
+        return SERVER_INVALID_ARGUMENT;
+    }
+
+    return SERVER_SUCCESS;
+}
+
+ServerError
+ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) {
+    cudaDeviceProp deviceProp;
+    auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);
+    if (cuda_err) {
+        SERVER_LOG_ERROR << "Failed to get video card properties: " << std::to_string(cuda_err);
+        return SERVER_UNEXPECTED_ERROR;
+    }
+
+    memory = deviceProp.totalGlobalMem;
+    return SERVER_SUCCESS;
+}
+
 }
 }
 }
\ No newline at end of file

cpp/src/utils/ValidationUtil.h

@@ -6,14 +6,23 @@ namespace zilliz {
 namespace milvus {
 namespace server {
 
-ServerError
-ValidateTableName(const std::string& table_name);
+class ValidationUtil {
+public:
+    static ServerError
+    ValidateTableName(const std::string &table_name);
 
-ServerError
-ValidateTableDimension(int64_t dimension);
+    static ServerError
+    ValidateTableDimension(int64_t dimension);
 
-ServerError
-ValidateTableIndexType(int32_t index_type);
+    static ServerError
+    ValidateTableIndexType(int32_t index_type);
+
+    static ServerError
+    ValidateGpuIndex(uint32_t gpu_index);
+
+    static ServerError
+    GetGpuMemory(uint32_t gpu_index, size_t &memory);
+};
 
 }
 }

cpp/src/wrapper/IndexBuilder.cpp

@@ -37,7 +37,7 @@ class GpuResources {
         using namespace zilliz::milvus::server;
         ServerConfig &config = ServerConfig::GetInstance();
         ConfigNode server_config = config.GetConfig(CONFIG_SERVER);
-        gpu_num = server_config.GetInt32Value("gpu_index", 0);
+        gpu_num = server_config.GetInt32Value(server::CONFIG_GPU_INDEX, 0);
     }
 
     int32_t GetGpu() {

cpp/unittest/CMakeLists.txt

@@ -35,6 +35,7 @@ set(unittest_libs
         dl
         z
         ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
+        cudart
         )
 
 add_subdirectory(server)

cpp/unittest/db/CMakeLists.txt

@@ -9,6 +9,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs)
 aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src)
 aux_source_directory(./ test_srcs)
 
+set(util_files
+        ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
+
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files)
@@ -29,6 +32,7 @@ set(db_test_src
     ${db_srcs}
     ${db_scheduler_srcs}
     ${wrapper_src}
+    ${util_files}
     ${require_files}
     ${test_srcs})
 

cpp/unittest/faiss_wrapper/CMakeLists.txt

@@ -6,6 +6,9 @@
 aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src)
 aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
 
+set(util_files
+        ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
+
 # Make sure that your call to link_directories takes place before your call to the relevant add_executable.
 include_directories(/usr/local/cuda/include)
 link_directories("/usr/local/cuda/lib64")
@@ -14,6 +17,7 @@ set(wrapper_test_src
         ${unittest_srcs}
         ${wrapper_src}
         ${config_files}
+        ${util_files}
         ${require_files}
         wrapper_test.cpp
         )

cpp/unittest/metrics/CMakeLists.txt

@@ -17,6 +17,9 @@ aux_source_directory(../../src/wrapper wrapper_src)
 aux_source_directory(../../src/metrics metrics_src)
 aux_source_directory(./ test_srcs)
 
+set(util_files
+        ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
+
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files)
@@ -43,6 +46,7 @@ set(count_test_src
         ${wrapper_src}
         ${metrics_src}
         ${test_srcs}
+        ${util_files}
         )
 
 

cpp/unittest/server/CMakeLists.txt

@@ -19,6 +19,7 @@ set(utils_srcs
         ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp
         ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp
         ${MILVUS_ENGINE_SRC}/utils/LogUtil.cpp
+        ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
         )
 
 cuda_add_executable(server_test

cpp/unittest/storage/CMakeLists.txt

@@ -5,6 +5,9 @@
 #-------------------------------------------------------------------------------
 aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 s3_client_src)
 
+set(util_files
+        ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
+
 # Make sure that your call to link_directories takes place before your call to the relevant add_executable.
 include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
 link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
@@ -19,7 +22,9 @@ set(s3_client_test_src
 
 add_executable(s3_test
         ${s3_client_test_src}
-        ${config_files})
+        ${config_files}
+        ${util_files}
+        )
 
 set(s3_client_libs
         stdc++

cpp/unittest/utils/ValidationUtilTest.cpp

@@ -16,48 +16,48 @@ using namespace zilliz::milvus::server;
 
 TEST(ValidationUtilTest, TableNameTest) {
     std::string table_name = "Normal123_";
-    ServerError res = ValidateTableName(table_name);
+    ServerError res = ValidationUtil::ValidateTableName(table_name);
     ASSERT_EQ(res, SERVER_SUCCESS);
 
     table_name = "12sds";
-    res = ValidateTableName(table_name);
+    res = ValidationUtil::ValidateTableName(table_name);
     ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
 
     table_name = "";
-    res = ValidateTableName(table_name);
+    res = ValidationUtil::ValidateTableName(table_name);
     ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
 
     table_name = "_asdasd";
-    res = ValidateTableName(table_name);
+    res = ValidationUtil::ValidateTableName(table_name);
     ASSERT_EQ(res, SERVER_SUCCESS);
 
     table_name = "!@#!@";
-    res = ValidateTableName(table_name);
+    res = ValidationUtil::ValidateTableName(table_name);
     ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
 
     table_name = "中文";
-    res = ValidateTableName(table_name);
+    res = ValidationUtil::ValidateTableName(table_name);
     ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
 
 
     table_name = std::string('a', 32768);
-    res = ValidateTableName(table_name);
+    res = ValidationUtil::ValidateTableName(table_name);
     ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
 }
 
 
 TEST(ValidationUtilTest, TableDimensionTest) {
-    ASSERT_EQ(ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION);
-    ASSERT_EQ(ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION);
-    ASSERT_EQ(ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION);
-    ASSERT_EQ(ValidateTableDimension(16384), SERVER_SUCCESS);
-    ASSERT_EQ(ValidateTableDimension(1), SERVER_SUCCESS);
+    ASSERT_EQ(ValidationUtil::ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION);
+    ASSERT_EQ(ValidationUtil::ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION);
+    ASSERT_EQ(ValidationUtil::ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION);
+    ASSERT_EQ(ValidationUtil::ValidateTableDimension(16384), SERVER_SUCCESS);
+    ASSERT_EQ(ValidationUtil::ValidateTableDimension(1), SERVER_SUCCESS);
 }
 
 TEST(ValidationUtilTest, TableIndexTypeTest) {
-    ASSERT_EQ(ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE);
+    ASSERT_EQ(ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE);
     for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) {
-        ASSERT_EQ(ValidateTableIndexType(i), SERVER_SUCCESS);
+        ASSERT_EQ(ValidationUtil::ValidateTableIndexType(i), SERVER_SUCCESS);
     }
-    ASSERT_EQ(ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE);
+    ASSERT_EQ(ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE);
 }