fix conflicts

Former-commit-id: f7a27268673c3544a75eee494e7c612c76c38384

cpp/CHANGELOG.md

@@ -29,11 +29,9 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-492 - Drop index failed if index have been created with index_type: FLAT
 - MS-493 - Knowhere unittest crash
 - MS-453 - GPU search error when nprobe set more than 1024
-<<<<<<< HEAD
 - MS-474 - Create index hang if use branch-0.3.1 server config
-=======
 - MS-510 - unittest out of memory and crashed
->>>>>>> upstream/branch-0.4.0
+- MS-119 - The problem of combining the log files
 
 ## Improvement
 - MS-327 - Clean code for milvus
@@ -114,6 +112,8 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-528 - Hide some config used future
 - MS-530 - Add unittest for SearchTask->Load
 - MS-531 - Disable next version code
+- MS-533 - Update resource_test to cover dump function
+- MS-523 - Config file validation
 
 ## New Feature
 - MS-343 - Implement ResourceMgr

cpp/CMakeLists.txt

@@ -31,7 +31,6 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release")
     set(BUILD_TYPE "release")
 else()
     set(BUILD_TYPE "debug")
-    SET(CMAKE_VERBOSE_MAKEFILE on)
 endif()
 message(STATUS "Build type = ${BUILD_TYPE}")
 

cpp/conf/server_config.template

 server_config:
-  address: 0.0.0.0            # milvus server ip address
+  address: 0.0.0.0            # milvus server ip address (IPv4)
   port: 19530                 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
   gpu_index: 0                # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1
   mode: single                # milvus deployment type: single, cluster, read_only

cpp/src/core/include/knowhere/index/vector_index/gpu_ivf.h

@@ -3,6 +3,7 @@
 #include <faiss/gpu/StandardGpuResources.h>
 
 #include "ivf.h"
+#include "src/utils/BlockingQueue.h"
 
 
 namespace zilliz {
@@ -16,12 +17,15 @@ struct Resource {
 
     std::shared_ptr<faiss::gpu::StandardGpuResources> faiss_res;
     int64_t id;
+    std::mutex mutex;
 };
 using ResPtr = std::shared_ptr<Resource>;
 using ResWPtr = std::weak_ptr<Resource>;
 
 class FaissGpuResourceMgr {
  public:
+    using ResBQ = zilliz::milvus::server::BlockingQueue<ResPtr>;
+
     struct DeviceParams {
         int64_t temp_mem_size = 0;
         int64_t pinned_mem_size = 0;
@@ -55,11 +59,8 @@ class FaissGpuResourceMgr {
 
     // allocate gpu memory before search
     // this func will return True if the device is idle and exists an idle resource.
-    bool
-    GetRes(const int64_t& device_id, ResPtr &res, const int64_t& alloc_size = 0);
-
-    void
-    MoveToInuse(const int64_t &device_id, const ResPtr& res);
+    //bool
+    //GetRes(const int64_t& device_id, ResPtr &res, const int64_t& alloc_size = 0);
 
     void
     MoveToIdle(const int64_t &device_id, const ResPtr& res);
@@ -67,33 +68,34 @@ class FaissGpuResourceMgr {
     void
     Dump();
 
- protected:
-    void
-    RemoveResource(const int64_t& device_id, const ResPtr& res, std::map<int64_t, std::vector<ResPtr>>& resource_pool);
-
  protected:
     bool is_init = false;
 
-    std::mutex mutex_;
     std::map<int64_t, DeviceParams> devices_params_;
-    std::map<int64_t, std::vector<ResPtr>> in_use_;
-    std::map<int64_t, std::vector<ResPtr>> idle_;
+    std::map<int64_t, ResBQ> idle_map;
 };
 
 class ResScope {
  public:
-    ResScope(const int64_t device_id,  ResPtr &res) : resource(res), device_id(device_id) {
-        FaissGpuResourceMgr::GetInstance().MoveToInuse(device_id, resource);
+    ResScope(const int64_t device_id,  ResPtr &res) : resource(res), device_id(device_id), move(true) {
+        res->mutex.lock();
+    }
+
+    ResScope(ResPtr &res) : resource(res), device_id(-1), move(false) {
+        res->mutex.lock();
     }
 
     ~ResScope() {
-        //resource->faiss_res->noTempMemory();
-        FaissGpuResourceMgr::GetInstance().MoveToIdle(device_id, resource);
+        if (move) {
+            FaissGpuResourceMgr::GetInstance().MoveToIdle(device_id, resource);
+        }
+        resource->mutex.unlock();
     }
 
  private:
     ResPtr resource;
     int64_t device_id;
+    bool move = true;
 };
 
 class GPUIndex {

cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp

@@ -130,19 +130,17 @@ void GPUIVF::search_impl(int64_t n,
                          float *distances,
                          int64_t *labels,
                          const Config &cfg) {
-    // TODO(linxj): allocate mem
-    auto temp_res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_);
-    if (temp_res) {
-        ResScope rs(gpu_id_, temp_res);
-        if (auto device_index = std::static_pointer_cast<faiss::gpu::GpuIndexIVF>(index_)) {
-            auto nprobe = cfg.get_with_default("nprobe", size_t(1));
-
-            std::lock_guard<std::mutex> lk(mutex_);
-            device_index->setNumProbes(nprobe);
+    std::lock_guard<std::mutex> lk(mutex_);
+
+    if (auto device_index = std::static_pointer_cast<faiss::gpu::GpuIndexIVF>(index_)) {
+        auto nprobe = cfg.get_with_default("nprobe", size_t(1));
+        device_index->setNumProbes(nprobe);
+
+        {
+            // TODO(linxj): allocate mem
+            ResScope rs(res_);
             device_index->search(n, (float *) data, k, distances, labels);
         }
-    } else {
-        KNOWHERE_THROW_MSG("search can't get gpu resource");
     }
 }
 
@@ -282,120 +280,75 @@ void FaissGpuResourceMgr::InitResource() {
 
     is_init = true;
 
+    std::cout << "InitResource" << std::endl;
     for(auto& device : devices_params_) {
-        auto& resource_vec = idle_[device.first];
+        auto& device_id = device.first;
+        std::cout << "Device Id: " << device_id << std::endl;
+        auto& device_param = device.second;
+        auto& bq = idle_map[device_id];
 
-        for (int64_t i = 0; i < device.second.resource_num; ++i) {
-            auto res = std::make_shared<faiss::gpu::StandardGpuResources>();
+        for (int64_t i = 0; i < device_param.resource_num; ++i) {
+            std::cout << "Resource Id: " << i << std::endl;
+            auto raw_resource = std::make_shared<faiss::gpu::StandardGpuResources>();
 
             // TODO(linxj): enable set pinned memory
-            //res->noTempMemory();
-            auto res_wrapper = std::make_shared<Resource>(res);
-            AllocateTempMem(res_wrapper, device.first, 0);
+            auto res_wrapper = std::make_shared<Resource>(raw_resource);
+            AllocateTempMem(res_wrapper, device_id, 0);
 
-            resource_vec.emplace_back(res_wrapper);
+            bq.Put(res_wrapper);
         }
     }
+    std::cout << "End initResource" << std::endl;
 }
 
 ResPtr FaissGpuResourceMgr::GetRes(const int64_t &device_id,
                                    const int64_t &alloc_size) {
-    std::lock_guard<std::mutex> lk(mutex_);
-
-    if (!is_init) {
-        InitResource();
-        is_init = true;
-    }
-
-    auto search = idle_.find(device_id);
-    if (search != idle_.end()) {
-        auto res = search->second.back();
-        //AllocateTempMem(res, device_id, alloc_size);
-
-        search->second.pop_back();
-        return res;
+    InitResource();
+
+    auto finder = idle_map.find(device_id);
+    if (finder != idle_map.end()) {
+        auto& bq = finder->second;
+        auto&& resource = bq.Take();
+        AllocateTempMem(resource, device_id, alloc_size);
+        return resource;
     }
     return nullptr;
 }
 
-bool FaissGpuResourceMgr::GetRes(const int64_t &device_id,
-                                 ResPtr &res,
-                                 const int64_t &alloc_size) {
-    std::lock_guard<std::mutex> lk(mutex_);
-
-    if (!is_init) {
-        InitResource();
-        is_init = true;
-    }
-
-    auto search = idle_.find(device_id);
-    if (search != idle_.end()) {
-        auto &res_vec = search->second;
-        for (auto it = res_vec.cbegin(); it != res_vec.cend(); ++it) {
-            if ((*it)->id == res->id) {
-                //AllocateTempMem(res, device_id, alloc_size);
-                res_vec.erase(it);
-                return true;
-            }
-        }
-    }
-    // else
-    return false;
-}
-
-void FaissGpuResourceMgr::MoveToInuse(const int64_t &device_id, const ResPtr &res) {
-    std::lock_guard<std::mutex> lk(mutex_);
-    RemoveResource(device_id, res, idle_);
-    in_use_[device_id].push_back(res);
-}
+//bool FaissGpuResourceMgr::GetRes(const int64_t &device_id,
+//                                 ResPtr &res,
+//                                 const int64_t &alloc_size) {
+//    InitResource();
+//
+//    std::lock_guard<std::mutex> lk(res->mutex);
+//    AllocateTempMem(res, device_id, alloc_size);
+//    return true;
+//}
 
 void FaissGpuResourceMgr::MoveToIdle(const int64_t &device_id, const ResPtr &res) {
-    std::lock_guard<std::mutex> lk(mutex_);
-    RemoveResource(device_id, res, in_use_);
-    auto it = idle_[device_id].begin();
-    idle_[device_id].insert(it, res);
-}
-
-void
-FaissGpuResourceMgr::RemoveResource(const int64_t &device_id,
-                                    const ResPtr &res,
-                                    std::map<int64_t, std::vector<ResPtr>> &resource_pool) {
-    if (resource_pool.find(device_id) != resource_pool.end()) {
-        std::vector<ResPtr> &res_array = resource_pool[device_id];
-        res_array.erase(std::remove_if(res_array.begin(), res_array.end(),
-                                       [&](ResPtr &ptr) { return ptr->id == res->id; }),
-                                       res_array.end());
+    auto finder = idle_map.find(device_id);
+    if (finder != idle_map.end()) {
+        auto& bq = finder->second;
+        bq.Put(res);
     }
 }
 
 void FaissGpuResourceMgr::Free() {
-    for (auto &item : in_use_) {
-        auto& res_vec = item.second;
-        res_vec.clear();
-    }
-    for (auto &item : idle_) {
-        auto& res_vec = item.second;
-        res_vec.clear();
+    for (auto &item : idle_map) {
+        auto& bq = item.second;
+        while (!bq.Empty()) {
+            bq.Take();
+        }
     }
     is_init = false;
 }
 
 void
 FaissGpuResourceMgr::Dump() {
-    std::cout << "In used resource" << std::endl;
-    for(auto& item: in_use_) {
-        std::cout << "device_id: " << item.first << std::endl;
-        for(auto& elem : item.second) {
-            std::cout << "resource_id: " << elem->id << std::endl;
-        }
-    }
-
-    std::cout << "Idle resource" << std::endl;
-    for(auto& item: idle_) {
-        std::cout << "device_id: " << item.first << std::endl;
-        for(auto& elem : item.second) {
-            std::cout << "resource_id: " << elem->id << std::endl;
-        }
+    for (auto &item : idle_map) {
+        auto& bq = item.second;
+        std::cout << "device_id: " << item.first
+                  << ", resource count:" << bq.Size();
     }
 }
 

cpp/src/core/test/test_ivf.cpp

@@ -386,7 +386,7 @@ class GPURESTEST
     int64_t elems = 0;
 };
 
-const int search_count = 10;
+const int search_count = 18;
 const int load_count = 3;
 
 TEST_F(GPURESTEST, gpu_ivf_resource_test) {

cpp/src/db/Utils.cpp

@@ -135,7 +135,7 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab
         }
     }
 
-    std::string msg = "Table file doesn't exist: " + table_file.file_id_;
+    std::string msg = "Table file doesn't exist: " + file_path;
     ENGINE_LOG_ERROR << msg;
     return Status(DB_ERROR, msg);
 }

cpp/src/db/meta/MySQLMetaImpl.cpp

@@ -814,7 +814,6 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
         file_schema.engine_type_ = table_schema.engine_type_;
         file_schema.nlist_ = table_schema.nlist_;
         file_schema.metric_type_ = table_schema.metric_type_;
-        utils::GetTableFilePath(options_, file_schema);
 
         std::string id = "NULL"; //auto-increment
         std::string table_id = file_schema.table_id_;
@@ -924,7 +923,10 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) {
             table_file.nlist_ = groups[table_file.table_id_].nlist_;
             table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
 
-            utils::GetTableFilePath(options_, table_file);
+            auto status = utils::GetTableFilePath(options_, table_file);
+            if(!status.ok()) {
+                return status;
+            }
 
             files.push_back(table_file);
         }
@@ -1027,7 +1029,10 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
 
             table_file.dimension_ = table_schema.dimension_;
 
-            utils::GetTableFilePath(options_, table_file);
+            auto status = utils::GetTableFilePath(options_, table_file);
+            if(!status.ok()) {
+                return status;
+            }
 
             auto dateItr = files.find(table_file.date_);
             if (dateItr == files.end()) {
@@ -1113,7 +1118,10 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
 
             table_file.dimension_ = table_schema.dimension_;
 
-            utils::GetTableFilePath(options_, table_file);
+            auto status = utils::GetTableFilePath(options_, table_file);
+            if(!status.ok()) {
+                return status;
+            }
 
             auto dateItr = files.find(table_file.date_);
             if (dateItr == files.end()) {
@@ -1203,7 +1211,10 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id,
 
             file_schema.dimension_ = table_schema.dimension_;
 
-            utils::GetTableFilePath(options_, file_schema);
+            auto status = utils::GetTableFilePath(options_, file_schema);
+            if(!status.ok()) {
+                return status;
+            }
 
             table_files.emplace_back(file_schema);
         }

cpp/src/db/meta/SqliteMetaImpl.cpp

@@ -614,7 +614,10 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
             table_file.engine_type_ = std::get<7>(file);
             table_file.created_on_ = std::get<8>(file);
 
-            utils::GetTableFilePath(options_, table_file);
+            auto status = utils::GetTableFilePath(options_, table_file);
+            if(!status.ok()) {
+                return status;
+            }
             auto groupItr = groups.find(table_file.table_id_);
             if (groupItr == groups.end()) {
                 TableSchema table_schema;
@@ -707,7 +710,11 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
             table_file.nlist_ = table_schema.nlist_;
             table_file.metric_type_ = table_schema.metric_type_;
 
-            utils::GetTableFilePath(options_, table_file);
+            auto status = utils::GetTableFilePath(options_, table_file);
+            if(!status.ok()) {
+                return status;
+            }
+
             auto dateItr = files.find(table_file.date_);
             if (dateItr == files.end()) {
                 files[table_file.date_] = TableFilesSchema();
@@ -773,7 +780,11 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id,
             table_file.nlist_ = table_schema.nlist_;
             table_file.metric_type_ = table_schema.metric_type_;
 
-            utils::GetTableFilePath(options_, table_file);
+            auto status = utils::GetTableFilePath(options_, table_file);
+            if(!status.ok()) {
+                return status;
+            }
+
             auto dateItr = files.find(table_file.date_);
             if (dateItr == files.end()) {
                 files[table_file.date_] = TableFilesSchema();
@@ -827,7 +838,10 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
             file_schema.nlist_ = table_schema.nlist_;
             file_schema.metric_type_ = table_schema.metric_type_;
 
-            utils::GetTableFilePath(options_, file_schema);
+            auto status = utils::GetTableFilePath(options_, file_schema);
+            if(!status.ok()) {
+                return status;
+            }
 
             table_files.emplace_back(file_schema);
         }

cpp/src/db/scheduler/TaskScheduler.cpp

@@ -132,4 +132,4 @@ TaskScheduler::TaskWorker() {
 
 }
 }
-}
\ No newline at end of file
+}

cpp/src/db/scheduler/TaskScheduler.h

@@ -10,6 +10,8 @@
 #include "TaskDispatchQueue.h"
 #include "utils/BlockingQueue.h"
 
+#include <thread>
+
 namespace zilliz {
 namespace milvus {
 namespace engine {

cpp/src/server/ServerConfig.h

@@ -52,9 +52,7 @@ static const char* CONFIG_OMP_THREAD_NUM = "omp_thread_num";
 static const char* CONFIG_RESOURCE = "resource_config";
 static const char* CONFIG_RESOURCES = "resources";
 static const char* CONFIG_RESOURCE_TYPE = "type";
-static const char* CONFIG_RESOURCE_MEMORY = "memory";
 static const char* CONFIG_RESOURCE_DEVICE_ID = "device_id";
-static const char* CONFIG_RESOURCE_ENABLE_LOADER = "enable_loader";
 static const char* CONFIG_RESOURCE_ENABLE_EXECUTOR = "enable_executor";
 static const char* CONFIG_RESOURCE_NUM = "gpu_resource_num";
 static const char* CONFIG_RESOURCE_PIN_MEMORY = "pinned_memory";
@@ -69,11 +67,19 @@ class ServerConfig {
     static ServerConfig &GetInstance();
 
     ErrorCode LoadConfigFile(const std::string& config_filename);
-    ErrorCode ValidateConfig() const;
+    ErrorCode ValidateConfig();
     void PrintAll() const;
 
     ConfigNode GetConfig(const std::string& name) const;
     ConfigNode& GetConfig(const std::string& name);
+
+ private:
+    ErrorCode CheckServerConfig();
+    ErrorCode CheckDBConfig();
+    ErrorCode CheckMetricConfig();
+    ErrorCode CheckCacheConfig();
+    ErrorCode CheckEngineConfig();
+    ErrorCode CheckResourceConfig();
 };
 
 }

cpp/src/utils/BlockingQueue.inl

 #pragma once
 
-#include "Log.h"
+//#include "Log.h"
 #include "Error.h"
 
 namespace zilliz {
@@ -17,7 +17,7 @@ BlockingQueue<T>::Put(const T &task) {
         std::string error_msg =
                 "blocking queue is full, capacity: " + std::to_string(capacity_) + " queue_size: " +
                 std::to_string(queue_.size());
-        SERVER_LOG_ERROR << error_msg;
+        //SERVER_LOG_ERROR << error_msg;
         throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg);
     }
 
@@ -33,7 +33,7 @@ BlockingQueue<T>::Take() {
 
     if (queue_.empty()) {
         std::string error_msg = "blocking queue empty";
-        SERVER_LOG_ERROR << error_msg;
+        //SERVER_LOG_ERROR << error_msg;
         throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg);
     }
 
@@ -57,7 +57,7 @@ BlockingQueue<T>::Front() {
     empty_.wait(lock, [this] { return !queue_.empty(); });
     if (queue_.empty()) {
         std::string error_msg = "blocking queue empty";
-        SERVER_LOG_ERROR << error_msg;
+        //SERVER_LOG_ERROR << error_msg;
         throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg);
     }
     T front(queue_.front());
@@ -72,7 +72,7 @@ BlockingQueue<T>::Back() {
 
     if (queue_.empty()) {
         std::string error_msg = "blocking queue empty";
-        SERVER_LOG_ERROR << error_msg;
+        //SERVER_LOG_ERROR << error_msg;
         throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg);
     }
 

cpp/src/utils/LogUtil.cpp

@@ -9,11 +9,70 @@
 #include <easylogging++.h>
 #include <ctype.h>
 
+#include <string>
+#include <libgen.h>
+
+
 namespace zilliz {
 namespace milvus {
 namespace server {
 
-int32_t InitLog(const std::string& log_config_file) {
+namespace {
+static int global_idx = 0;
+static int debug_idx = 0;
+static int warning_idx = 0;
+static int trace_idx = 0;
+static int error_idx = 0;
+static int fatal_idx = 0;
+}
+
+// TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename
+void rolloutHandler(const char *filename, std::size_t size) {
+    char *dirc = strdup(filename);
+    char *basec = strdup(filename);
+    char *dir = dirname(dirc);
+    char *base = basename(basec);
+
+    std::string s(base);
+    std::stringstream ss;
+    std::string
+        list[] = {"\\", " ", "\'", "\"", "*", "\?", "{", "}", ";", "<", ">", "|", "^", "&", "$", "#", "!", "`", "~"};
+    std::string::size_type position;
+    for (auto substr : list) {
+        position = 0;
+        while ((position = s.find_first_of(substr, position)) != std::string::npos) {
+            s.insert(position, "\\");
+            position += 2;
+        }
+    }
+    int ret;
+    std::string m(std::string(dir) + "/" + s);
+    s = m;
+    if ((position = s.find("global")) != std::string::npos) {
+        s.append("." + std::to_string(++global_idx));
+        ret = rename(m.c_str(), s.c_str());
+    } else if ((position = s.find("debug")) != std::string::npos) {
+        s.append("." + std::to_string(++debug_idx));
+        ret = rename(m.c_str(), s.c_str());
+    } else if ((position = s.find("warning")) != std::string::npos) {
+        s.append("." + std::to_string(++warning_idx));
+        ret = rename(m.c_str(), s.c_str());
+    } else if ((position = s.find("trace")) != std::string::npos) {
+        s.append("." + std::to_string(++trace_idx));
+        ret = rename(m.c_str(), s.c_str());
+    } else if ((position = s.find("error")) != std::string::npos) {
+        s.append("." + std::to_string(++error_idx));
+        ret = rename(m.c_str(), s.c_str());
+    } else if ((position = s.find("fatal")) != std::string::npos) {
+        s.append("." + std::to_string(++fatal_idx));
+        ret = rename(m.c_str(), s.c_str());
+    } else {
+        s.append("." + std::to_string(++global_idx));
+        ret = rename(m.c_str(), s.c_str());
+    }
+}
+
+int32_t InitLog(const std::string &log_config_file) {
 #if 0
     ServerConfig &config = ServerConfig::GetInstance();
     ConfigNode log_config = config.GetConfig(CONFIG_LOG);
@@ -50,8 +109,10 @@ int32_t InitLog(const std::string& log_config_file) {
 #else
     el::Configurations conf(log_config_file);
 #endif
-
     el::Loggers::reconfigureAllLoggers(conf);
+
+    el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck);
+    el::Helpers::installPreRollOutCallback(rolloutHandler);
     return 0;
 }
 

cpp/src/utils/LogUtil.h

@@ -11,9 +11,7 @@
 namespace zilliz {
 namespace milvus {
 namespace server {
-
 int32_t InitLog(const std::string& log_config_file);
-
 inline std::string GetFileName(std::string filename) {
     int pos = filename.find_last_of('/');
     return filename.substr(pos + 1);

cpp/src/utils/ValidationUtil.cpp

@@ -4,6 +4,12 @@
 
 #include <cuda_runtime.h>
 
+#include <arpa/inet.h>
+
+#include <regex>
+#include <algorithm>
+
+
 namespace zilliz {
 namespace milvus {
 namespace server {
@@ -51,15 +57,16 @@ ValidationUtil::ValidateTableDimension(int64_t dimension) {
     if (dimension <= 0 || dimension > TABLE_DIMENSION_LIMIT) {
         SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << TABLE_DIMENSION_LIMIT;
         return SERVER_INVALID_VECTOR_DIMENSION;
-    } else {
+    }
+    else {
         return SERVER_SUCCESS;
     }
 }
 
 ErrorCode
 ValidationUtil::ValidateTableIndexType(int32_t index_type) {
-    int engine_type = (int)engine::EngineType(index_type);
-    if(engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) {
+    int engine_type = (int) engine::EngineType(index_type);
+    if (engine_type <= 0 || engine_type > (int) engine::EngineType::MAX_VALUE) {
         return SERVER_INVALID_INDEX_TYPE;
     }
 
@@ -68,7 +75,7 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) {
 
 ErrorCode
 ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
-    if(nlist <= 0) {
+    if (nlist <= 0) {
         return SERVER_INVALID_INDEX_NLIST;
     }
 
@@ -77,7 +84,7 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
 
 ErrorCode
 ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
-    if(index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) {
+    if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) {
         return SERVER_INVALID_INDEX_FILE_SIZE;
     }
 
@@ -86,14 +93,14 @@ ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
 
 ErrorCode
 ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
-    if(metric_type != (int32_t)engine::MetricType::L2 && metric_type != (int32_t)engine::MetricType::IP) {
+    if (metric_type != (int32_t) engine::MetricType::L2 && metric_type != (int32_t) engine::MetricType::IP) {
         return SERVER_INVALID_INDEX_METRIC_TYPE;
     }
     return SERVER_SUCCESS;
 }
 
 ErrorCode
-ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) {
+ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema &table_schema) {
     if (top_k <= 0 || top_k > 2048) {
         return SERVER_INVALID_TOPK;
     }
@@ -102,7 +109,7 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchem
 }
 
 ErrorCode
-ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema) {
+ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema &table_schema) {
     if (nprobe <= 0 || nprobe > table_schema.nlist_) {
         return SERVER_INVALID_NPROBE;
     }
@@ -119,7 +126,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
         return SERVER_UNEXPECTED_ERROR;
     }
 
-    if(gpu_index >= num_devices) {
+    if (gpu_index >= num_devices) {
         return SERVER_INVALID_ARGUMENT;
     }
 
@@ -127,7 +134,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
 }
 
 ErrorCode
-ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) {
+ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t &memory) {
     cudaDeviceProp deviceProp;
     auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);
     if (cuda_err) {
@@ -139,6 +146,108 @@ ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) {
     return SERVER_SUCCESS;
 }
 
+ErrorCode
+ValidationUtil::ValidateIpAddress(const std::string &ip_address) {
+
+    struct in_addr address;
+
+    int result = inet_pton(AF_INET, ip_address.c_str(), &address);
+
+    switch (result) {
+        case 1:return SERVER_SUCCESS;
+        case 0:SERVER_LOG_ERROR << "Invalid IP address: " << ip_address;
+            return SERVER_INVALID_ARGUMENT;
+        default:SERVER_LOG_ERROR << "inet_pton conversion error";
+            return SERVER_UNEXPECTED_ERROR;
+    }
+}
+
+ErrorCode
+ValidationUtil::ValidateStringIsNumber(const std::string &string) {
+    if (!string.empty() && std::all_of(string.begin(), string.end(), ::isdigit)) {
+        return SERVER_SUCCESS;
+    }
+    else {
+        return SERVER_INVALID_ARGUMENT;
+    }
+}
+
+ErrorCode
+ValidationUtil::ValidateStringIsBool(std::string &str) {
+    std::transform(str.begin(), str.end(), str.begin(), ::tolower);
+    if (str == "true" || str == "on" || str == "yes" || str == "1" ||
+        str == "false" || str == "off" || str == "no" || str == "0" ||
+        str.empty()) {
+        return SERVER_SUCCESS;
+    }
+    else {
+        return SERVER_INVALID_ARGUMENT;
+    }
+}
+
+ErrorCode
+ValidationUtil::ValidateStringIsDouble(const std::string &str, double &val) {
+    char *end = nullptr;
+    val = std::strtod(str.c_str(), &end);
+    if (end != str.c_str() && *end == '\0' && val != HUGE_VAL) {
+        return SERVER_SUCCESS;
+    }
+    else {
+        return SERVER_INVALID_ARGUMENT;
+    }
+}
+
+ErrorCode
+ValidationUtil::ValidateDbURI(const std::string &uri) {
+    std::string dialectRegex = "(.*)";
+    std::string usernameRegex = "(.*)";
+    std::string passwordRegex = "(.*)";
+    std::string hostRegex = "(.*)";
+    std::string portRegex = "(.*)";
+    std::string dbNameRegex = "(.*)";
+    std::string uriRegexStr = dialectRegex + "\\:\\/\\/" +
+        usernameRegex + "\\:" +
+        passwordRegex + "\\@" +
+        hostRegex + "\\:" +
+        portRegex + "\\/" +
+        dbNameRegex;
+    std::regex uriRegex(uriRegexStr);
+    std::smatch pieces_match;
+
+    bool okay = true;
+
+    if (std::regex_match(uri, pieces_match, uriRegex)) {
+        std::string dialect = pieces_match[1].str();
+        std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower);
+        if (dialect.find("mysql") == std::string::npos && dialect.find("sqlite") == std::string::npos) {
+            SERVER_LOG_ERROR << "Invalid dialect in URI: dialect = " << dialect;
+            okay = false;
+        }
+
+        std::string host = pieces_match[4].str();
+        if (!host.empty() && host != "localhost") {
+            if (ValidateIpAddress(host) != SERVER_SUCCESS) {
+                SERVER_LOG_ERROR << "Invalid host ip address in uri = " << host;
+                okay = false;
+            }
+        }
+
+        std::string port = pieces_match[5].str();
+        if (!port.empty()) {
+            if (ValidateStringIsNumber(port) != SERVER_SUCCESS) {
+                SERVER_LOG_ERROR << "Invalid port in uri = " << port;
+                okay = false;
+            }
+        }
+    }
+    else {
+        SERVER_LOG_ERROR << "Wrong URI format: URI = " << uri;
+        okay = false;
+    }
+
+    return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT);
+}
+
 }
 }
 }
\ No newline at end of file

cpp/src/utils/ValidationUtil.h

@@ -40,7 +40,19 @@ public:
     GetGpuMemory(uint32_t gpu_index, size_t &memory);
 
     static ErrorCode
-    ValidateConfig();
+    ValidateIpAddress(const std::string &ip_address);
+
+    static ErrorCode
+    ValidateStringIsNumber(const std::string &str);
+
+    static ErrorCode
+    ValidateStringIsBool(std::string &str);
+
+    static ErrorCode
+    ValidateStringIsDouble(const std::string &str, double &val);
+
+    static ErrorCode
+    ValidateDbURI(const std::string &uri);
 };
 
 }

cpp/unittest/db/db_tests.cpp

@@ -315,6 +315,79 @@ TEST_F(DBTest, PRELOADTABLE_TEST) {
 
 }
 
+TEST_F(DBTest, SHUTDOWN_TEST) {
+    db_->Stop();
+
+    engine::meta::TableSchema table_info = BuildTableSchema();
+    engine::Status stat = db_->CreateTable(table_info);
+    ASSERT_FALSE(stat.ok());
+
+    stat = db_->DescribeTable(table_info);
+    ASSERT_FALSE(stat.ok());
+
+    bool has_table = false;
+    stat = db_->HasTable(table_info.table_id_, has_table);
+    ASSERT_FALSE(stat.ok());
+
+    engine::IDNumbers ids;
+    stat = db_->InsertVectors(table_info.table_id_, 0, nullptr, ids);
+    ASSERT_FALSE(stat.ok());
+
+    stat = db_->PreloadTable(table_info.table_id_);
+    ASSERT_FALSE(stat.ok());
+
+    uint64_t row_count = 0;
+    stat = db_->GetTableRowCount(table_info.table_id_, row_count);
+    ASSERT_FALSE(stat.ok());
+
+    engine::TableIndex index;
+    stat = db_->CreateIndex(table_info.table_id_, index);
+    ASSERT_FALSE(stat.ok());
+
+    stat = db_->DescribeIndex(table_info.table_id_, index);
+    ASSERT_FALSE(stat.ok());
+
+    engine::meta::DatesT dates;
+    engine::QueryResults results;
+    stat = db_->Query(table_info.table_id_, 1, 1, 1, nullptr, dates, results);
+    ASSERT_FALSE(stat.ok());
+    std::vector<std::string> file_ids;
+    stat = db_->Query(table_info.table_id_, file_ids, 1, 1, 1, nullptr, dates, results);
+    ASSERT_FALSE(stat.ok());
+
+    stat = db_->DeleteTable(table_info.table_id_, dates);
+    ASSERT_FALSE(stat.ok());
+}
+
+TEST_F(DBTest, INDEX_TEST) {
+    engine::meta::TableSchema table_info = BuildTableSchema();
+    engine::Status stat = db_->CreateTable(table_info);
+
+    int64_t nb = VECTOR_COUNT;
+    std::vector<float> xb;
+    BuildVectors(nb, xb);
+
+    engine::IDNumbers vector_ids;
+    db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
+    ASSERT_EQ(vector_ids.size(), nb);
+
+    engine::TableIndex index;
+    index.engine_type_ = (int)engine::EngineType::FAISS_IVFSQ8;
+    index.metric_type_ = (int)engine::MetricType::IP;
+    stat = db_->CreateIndex(table_info.table_id_, index);
+    ASSERT_TRUE(stat.ok());
+
+    engine::TableIndex index_out;
+    stat = db_->DescribeIndex(table_info.table_id_, index_out);
+    ASSERT_TRUE(stat.ok());
+    ASSERT_EQ(index.engine_type_, index_out.engine_type_);
+    ASSERT_EQ(index.nlist_, index_out.nlist_);
+    ASSERT_EQ(table_info.metric_type_, index_out.metric_type_);
+
+    stat = db_->DropIndex(table_info.table_id_);
+    ASSERT_TRUE(stat.ok());
+}
+
 TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
 
     engine::meta::TableSchema table_info = BuildTableSchema();

cpp/unittest/db/utils.cpp

@@ -13,6 +13,7 @@
 #include "db/Factories.h"
 #include "db/Options.h"
 #include "server/ServerConfig.h"
+#include "knowhere/index/vector_index/gpu_ivf.h"
 
 INITIALIZE_EASYLOGGINGPP
 
@@ -59,6 +60,8 @@ engine::Options BaseTest::GetOptions() {
 void DBTest::SetUp() {
     BaseTest::SetUp();
 
+    zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(0, 1024*1024*200, 1024*1024*300, 2);
+
     server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
     config.AddSequenceItem(server::CONFIG_GPU_IDS, "0");
 
@@ -84,6 +87,8 @@ void DBTest::TearDown() {
     db_->DropAll();
     delete db_;
 
+    zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free();
+
     engine::ResMgrInst::GetInstance()->Stop();
     engine::SchedInst::GetInstance()->Stop();
 

cpp/unittest/scheduler/resource_test.cpp

@@ -86,9 +86,7 @@ TEST_F(ResourceBaseTest, dump) {
     ASSERT_FALSE(only_executor_->Dump().empty());
     ASSERT_FALSE(both_enable_->Dump().empty());
     ASSERT_FALSE(both_disable_->Dump().empty());
-    std::stringstream ss;
-    ss << only_loader_ << only_executor_ << both_enable_ << both_disable_;
-    ASSERT_FALSE(ss.str().empty());
+    std::cout << *only_loader_ << *only_executor_ << *both_enable_ << *both_disable_;
 }
 
 /************ ResourceAdvanceTest ************/

cpp/unittest/server/util_test.cpp

@@ -5,6 +5,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 #include <gtest/gtest.h>
 #include <thread>
+#include <easylogging++.h>
 
 #include "utils/CommonUtil.h"
 #include "utils/Error.h"