Merge remote-tracking branch 'source/0.6.0' into ongoing

CHANGELOG.md

@@ -21,6 +21,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version
 - \#458 - Index data is not compatible between 0.5 and 0.6
 - \#465 - Server hang caused by searching with nsg index
+- \#485 - Increase code coverage rate
 - \#486 - gpu no usage during index building
 - \#497 - CPU-version search performance decreased
 - \#504 - The code coverage rate of core/src/scheduler/optimizer is too low
@@ -37,6 +38,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#561 - Milvus server should report exception/error message or terminate on mysql metadata backend error
 - \#596 - Frequently insert operation cost too much disk space
 - \#599 - Build index log is incorrect
+- \#602 - Optimizer specify wrong gpu_id
 
 ## Feature
 - \#12 - Pure CPU version for Milvus
@@ -48,6 +50,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#420 - Update shards merge part to match v0.5.3
 - \#488 - Add log in scheduler/optimizer
 - \#502 - C++ SDK support IVFPQ and SPTAG
+- \#560 - Add version in server config file
 
 ## Improvement
 - \#255 - Add ivfsq8 test report detailed version

core/conf/server_cpu_config.template

 # Default values are used when you make no changes to the following parameters.
 
+version: 0.1                        # config version
+
 server_config:
   address: 0.0.0.0                  # milvus server ip address (IPv4)
   port: 19530                       # milvus server port, must in range [1025, 65534]

core/conf/server_gpu_config.template

 # Default values are used when you make no changes to the following parameters.
 
+version: 0.1                        # config version
+
 server_config:
   address: 0.0.0.0                  # milvus server ip address (IPv4)
   port: 19530                       # milvus server port, must in range [1025, 65534]

core/src/db/engine/ExecutionEngineImpl.cpp

@@ -112,10 +112,12 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
             index = GetVecIndexFactory(IndexType::NSG_MIX);
             break;
         }
+#ifdef CUSTOMIZATION
         case EngineType::FAISS_IVFSQ8H: {
             index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_HYBRID);
             break;
         }
+#endif
         case EngineType::FAISS_PQ: {
 #ifdef MILVUS_CPU_VERSION
             index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_CPU);

core/src/scheduler/optimizer/FaissFlatPass.cpp

@@ -60,7 +60,7 @@ FaissFlatPass::Run(const TaskPtr& task) {
         auto best_device_id = count_ % gpus.size();
         SERVER_LOG_DEBUG << "FaissFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id << " to search!";
         count_++;
-        res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);
+        res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpus[best_device_id]);
     }
     auto label = std::make_shared<SpecResLabel>(res_ptr);
     task->label() = label;

core/src/scheduler/optimizer/FaissIVFFlatPass.cpp

@@ -63,7 +63,7 @@ FaissIVFFlatPass::Run(const TaskPtr& task) {
         SERVER_LOG_DEBUG << "FaissIVFFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id
                          << " to search!";
         count_++;
-        res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);
+        res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpus[best_device_id]);
     }
     auto label = std::make_shared<SpecResLabel>(res_ptr);
     task->label() = label;

core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp

@@ -61,7 +61,7 @@ FaissIVFSQ8HPass::Run(const TaskPtr& task) {
         SERVER_LOG_DEBUG << "FaissIVFSQ8HPass: nq > gpu_search_threshold, specify gpu" << best_device_id
                          << " to search!";
         count_++;
-        res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);
+        res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpus[best_device_id]);
     }
     auto label = std::make_shared<SpecResLabel>(res_ptr);
     task->label() = label;

core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp

@@ -63,7 +63,7 @@ FaissIVFSQ8Pass::Run(const TaskPtr& task) {
         SERVER_LOG_DEBUG << "FaissIVFSQ8Pass: nq > gpu_search_threshold, specify gpu" << best_device_id
                          << " to search!";
         count_++;
-        res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);
+        res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpus[best_device_id]);
     }
     auto label = std::make_shared<SpecResLabel>(res_ptr);
     task->label() = label;

core/src/sdk/include/MilvusApi.h

@@ -85,18 +85,18 @@ struct RowRecord {
  * @brief TopK query result
  */
 struct QueryResult {
-    std::vector<int64_t> ids;
-    std::vector<float> distances;
+    std::vector<int64_t> ids;      ///< Query ids result
+    std::vector<float> distances;  ///< Query distances result
 };
-using TopKQueryResult = std::vector<QueryResult>;
+using TopKQueryResult = std::vector<QueryResult>;  ///< Topk query result
 
 /**
  * @brief index parameters
  */
 struct IndexParam {
-    std::string table_name;
-    IndexType index_type;
-    int32_t nlist;
+    std::string table_name;  ///< Table name for create index
+    IndexType index_type;    ///< Create index type
+    int32_t nlist;           ///< Index nlist
 };
 
 /**
@@ -142,8 +142,8 @@ class Connection {
     /**
      * @brief Connect
      *
-     * Connect function should be called before any operations
-     * Server will be connected after Connect return OK
+     * This method is used to connect server.
+     * Connect function should be called before any operations.
      *
      * @param param, use to provide server information
      *
@@ -156,10 +156,10 @@ class Connection {
     /**
      * @brief Connect
      *
-     * Connect function should be called before any operations
-     * Server will be connected after Connect return OK
+     * This method is used to connect server.
+     * Connect function should be called before any operations.
      *
-     * @param uri, use to provide server information, example: milvus://ipaddress:port
+     * @param uri, use to provide server uri, example: milvus://ipaddress:port
      *
      * @return Indicate if connect is successful
      */
@@ -169,7 +169,7 @@ class Connection {
     /**
      * @brief connected
      *
-     * Connection status.
+     * This method is used to test whether server is connected.
      *
      * @return Indicate if connection status
      */
@@ -179,7 +179,7 @@ class Connection {
     /**
      * @brief Disconnect
      *
-     * Server will be disconnected after Disconnect return OK
+     * This method is used to disconnect server.
      *
      * @return Indicate if disconnect is successful
      */
@@ -189,7 +189,7 @@ class Connection {
     /**
      * @brief Create table method
      *
-     * This method is used to create table
+     * This method is used to create table.
      *
      * @param param, use to provide table information to be created.
      *
@@ -201,7 +201,7 @@ class Connection {
     /**
      * @brief Test table existence method
      *
-     * This method is used to create table
+     * This method is used to create table.
      *
      * @param table_name, target table's name.
      *
@@ -211,13 +211,13 @@ class Connection {
     HasTable(const std::string& table_name) = 0;
 
     /**
-     * @brief Delete table method
+     * @brief Drop table method
      *
-     * This method is used to delete table(and its partitions).
+     * This method is used to drop table(and its partitions).
      *
      * @param table_name, target table's name.
      *
-     * @return Indicate if table is delete successfully.
+     * @return Indicate if table is drop successfully.
      */
     virtual Status
     DropTable(const std::string& table_name) = 0;
@@ -239,14 +239,17 @@ class Connection {
     CreateIndex(const IndexParam& index_param) = 0;
 
     /**
-     * @brief Add vector to table
+     * @brief Insert vector to table
      *
-     * This method is used to add vector array to table.
+     * This method is used to insert vector array to table.
      *
      * @param table_name, target table's name.
      * @param partition_tag, target partition's tag, keep empty if no partition.
      * @param record_array, vector array is inserted.
-     * @param id_array, after inserted every vector is given a id.
+     * @param id_array,
+     *  specify id for each vector,
+     *  if this array is empty, milvus will generate unique id for each vector,
+     *  and return all ids by this parameter.
      *
      * @return Indicate if vector array are inserted successfully
      */
@@ -259,11 +262,12 @@ class Connection {
      *
      * This method is used to query vector in table.
      *
-     * @param table_name, target table's name, keep empty if no partition.
-     * @param partition_tags, target partitions.
+     * @param table_name, target table's name.
+     * @param partition_tags, target partitions, keep empty if no partition.
      * @param query_record_array, all vector are going to be queried.
-     * @param query_range_array, time ranges, if not specified, will search in whole table
+     * @param query_range_array, [deprecated] time ranges, if not specified, will search in whole table
      * @param topk, how many similarity vectors will be searched.
+     * @param nprobe, the number of centroids choose to search.
      * @param topk_query_result_array, result array.
      *
      * @return Indicate if query is successful.
@@ -304,7 +308,7 @@ class Connection {
      *
      * This method is used to list all tables.
      *
-     * @param table_array, all tables are push into the array.
+     * @param table_array, all tables in database.
      *
      * @return Indicate if this operation is successful.
      */
@@ -346,12 +350,13 @@ class Connection {
      *
      * This method is internal used.
      *
-     * @return Server status.
+     * @return Task information in tasktables.
      */
     virtual std::string
     DumpTaskTables() const = 0;
 
     /**
+     * [deprecated]
      * @brief delete tables by date range
      *
      * This method is used to delete table data by date range.

core/src/server/Config.cpp

@@ -20,6 +20,7 @@
 #include <iostream>
 #include <regex>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 #include "config/YamlConfigMgr.h"
@@ -33,6 +34,8 @@ namespace server {
 
 constexpr uint64_t GB = 1UL << 30;
 
+static const std::unordered_map<std::string, std::string> milvus_config_version_map({{"0.6.0", "0.1"}});
+
 Config&
 Config::GetInstance() {
     static Config config_inst;
@@ -69,6 +72,12 @@ Status
 Config::ValidateConfig() {
     Status s;
 
+    std::string config_version;
+    s = GetConfigVersion(config_version);
+    if (!s.ok()) {
+        return s;
+    }
+
     /* server config */
     std::string server_addr;
     s = GetServerConfigAddress(server_addr);
@@ -383,6 +392,16 @@ Config::PrintAll() {
 }
 
 ////////////////////////////////////////////////////////////////////////////////
+Status
+Config::CheckConfigVersion(const std::string& value) {
+    if (milvus_config_version_map.at(MILVUS_VERSION) != value) {
+        std::string msg = "Invalid config version: " + value +
+                          ". Expected config version: " + milvus_config_version_map.at(MILVUS_VERSION);
+        return Status(SERVER_INVALID_ARGUMENT, msg);
+    }
+    return Status::OK();
+}
+
 Status
 Config::CheckServerConfigAddress(const std::string& value) {
     if (!ValidationUtil::ValidateIpAddress(value).ok()) {
@@ -766,10 +785,14 @@ Config::CheckGpuResourceConfigBuildIndexResources(const std::vector<std::string>
 
 ////////////////////////////////////////////////////////////////////////////////
 ConfigNode&
-Config::GetConfigNode(const std::string& name) {
+Config::GetConfigRoot() {
     ConfigMgr* mgr = YamlConfigMgr::GetInstance();
-    ConfigNode& root_node = mgr->GetRootNode();
-    return root_node.GetChild(name);
+    return mgr->GetRootNode();
+}
+
+ConfigNode&
+Config::GetConfigNode(const std::string& name) {
+    return GetConfigRoot().GetChild(name);
 }
 
 Status
@@ -816,6 +839,12 @@ Config::GetConfigSequenceStr(const std::string& parent_key, const std::string& c
     return value;
 }
 
+Status
+Config::GetConfigVersion(std::string& value) {
+    value = GetConfigRoot().GetValue(CONFIG_VERSION);
+    return CheckConfigVersion(value);
+}
+
 Status
 Config::GetServerConfigAddress(std::string& value) {
     value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT);

core/src/server/Config.h

@@ -28,6 +28,8 @@
 namespace milvus {
 namespace server {
 
+static const char* CONFIG_VERSION = "version";
+
 /* server config */
 static const char* CONFIG_SERVER = "server_config";
 static const char* CONFIG_SERVER_ADDRESS = "address";
@@ -115,6 +117,8 @@ class Config {
     PrintAll();
 
  private:
+    ConfigNode&
+    GetConfigRoot();
     ConfigNode&
     GetConfigNode(const std::string& name);
     Status
@@ -125,6 +129,9 @@ class Config {
     PrintConfigSection(const std::string& config_node_name);
 
     ///////////////////////////////////////////////////////////////////////////
+    Status
+    CheckConfigVersion(const std::string& value);
+
     /* server config */
     Status
     CheckServerConfigAddress(const std::string& value);
@@ -193,6 +200,8 @@ class Config {
     std::string
     GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim = ",",
                          const std::string& default_value = "");
+    Status
+    GetConfigVersion(std::string& value);
 
  public:
     /* server config */

core/unittest/db/test_engine.cpp

@@ -59,6 +59,29 @@ TEST_F(EngineTest, FACTORY_TEST) {
 
         ASSERT_TRUE(engine_ptr != nullptr);
     }
+
+    {
+        auto engine_ptr = milvus::engine::EngineFactory::Build(
+                512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_PQ, milvus::engine::MetricType::IP, 1024);
+
+        ASSERT_TRUE(engine_ptr != nullptr);
+    }
+
+    {
+        auto engine_ptr = milvus::engine::EngineFactory::Build(
+                512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
+                milvus::engine::MetricType::L2, 1024);
+
+        ASSERT_TRUE(engine_ptr != nullptr);
+    }
+
+    {
+        auto engine_ptr = milvus::engine::EngineFactory::Build(
+                512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
+                milvus::engine::MetricType::L2, 1024);
+
+        ASSERT_TRUE(engine_ptr != nullptr);
+    }
 }
 
 TEST_F(EngineTest, ENGINE_IMPL_TEST) {
@@ -69,7 +92,7 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
 
     std::vector<float> data;
     std::vector<int64_t> ids;
-    const int row_count = 10000;
+    const int row_count = 500;
     data.reserve(row_count * dimension);
     ids.reserve(row_count);
     for (int64_t i = 0; i < row_count; i++) {
@@ -95,5 +118,8 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
     // ASSERT_TRUE(status.ok());
 
     auto engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IVFSQ8);
+    engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_3", milvus::engine::EngineType::FAISS_PQ);
+    engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_4", milvus::engine::EngineType::SPTAG_KDT);
+    engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_5", milvus::engine::EngineType::SPTAG_BKT);
     // ASSERT_TRUE(status.ok());
 }

core/unittest/server/utils.cpp

@@ -28,6 +28,8 @@ namespace {
 static const char* VALID_CONFIG_STR =
     "# Default values are used when you make no changes to the following parameters.\n"
     "\n"
+    "version: 0.1"
+    "\n"
     "server_config:\n"
     "  address: 0.0.0.0                  # milvus server ip address (IPv4)\n"
     "  port: 19530                       # port range: 1025 ~ 65534\n"