Merge branch 'branch-0.4.0' into 'branch-0.4.0'

Branch 0.4.0

See merge request megasearch/milvus!553

Former-commit-id: 6e296f3d7d10786dc22d3f5a8139d94235b72c34

cpp/conf/server_config.template

@@ -56,12 +56,12 @@ resource_config:
     cpu:
       type: CPU
       device_id: 0
-      enable_executor: false
+      enable_executor: true
 
     gpu0:
       type: GPU
       device_id: 0
-      enable_executor: true
+      enable_executor: false
       gpu_resource_num: 2
       pinned_memory: 300
       temp_memory: 300

cpp/src/db/scheduler/context/SearchContext.cpp

@@ -50,6 +50,7 @@ void
 SearchContext::WaitResult() {
     std::unique_lock <std::mutex> lock(mtx_);
     done_cond_.wait(lock, [this] { return map_index_files_.empty(); });
+    SERVER_LOG_DEBUG << "SearchContext " << identity_ << " all done";
 }
 
 }

cpp/src/scheduler/task/SearchTask.cpp

@@ -152,10 +152,10 @@ XSearchTask::Execute() {
         return;
     }
 
-    ENGINE_LOG_DEBUG << "Searching in file id:" << index_id_ << " with "
+    ENGINE_LOG_DEBUG << "Searching in file id " << index_id_ << " with "
                      << search_contexts_.size() << " tasks";
 
-    server::TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_));
+    server::TimeRecorder rc("DoSearch file id " + std::to_string(index_id_));
 
     server::CollectDurationMetrics metrics(index_type_);
 
@@ -163,32 +163,37 @@ XSearchTask::Execute() {
     std::vector<float> output_distance;
     for (auto &context : search_contexts_) {
         //step 1: allocate memory
-        auto inner_k = context->topk();
+        auto nq = context->nq();
+        auto topk = context->topk();
         auto nprobe = context->nprobe();
-        output_ids.resize(inner_k * context->nq());
-        output_distance.resize(inner_k * context->nq());
+        auto vectors = context->vectors();
+
+        output_ids.resize(topk * nq);
+        output_distance.resize(topk * nq);
+        std::string hdr = "context " + context->Identity() +
+                          " nq " + std::to_string(nq) +
+                          " topk " + std::to_string(topk);
 
         try {
             //step 2: search
-            index_engine_->Search(context->nq(), context->vectors(), inner_k, nprobe, output_distance.data(),
-                                  output_ids.data());
+            index_engine_->Search(nq, vectors, topk, nprobe, output_distance.data(), output_ids.data());
 
-            double span = rc.RecordSection("do search for context:" + context->Identity());
+            double span = rc.RecordSection(hdr + ", do search");
             context->AccumSearchCost(span);
 
 
             //step 3: cluster result
             SearchContext::ResultSet result_set;
-            auto spec_k = index_engine_->Count() < context->topk() ? index_engine_->Count() : context->topk();
-            XSearchTask::ClusterResult(output_ids, output_distance, context->nq(), spec_k, result_set);
+            auto spec_k = index_engine_->Count() < topk ? index_engine_->Count() : topk;
+            XSearchTask::ClusterResult(output_ids, output_distance, nq, spec_k, result_set);
 
-            span = rc.RecordSection("cluster result for context:" + context->Identity());
+            span = rc.RecordSection(hdr + ", cluster result");
             context->AccumReduceCost(span);
 
             // step 4: pick up topk result
-            XSearchTask::TopkResult(result_set, inner_k, metric_l2, context->GetResult());
+            XSearchTask::TopkResult(result_set, topk, metric_l2, context->GetResult());
 
-            span = rc.RecordSection("reduce topk for context:" + context->Identity());
+            span = rc.RecordSection(hdr + ", reduce topk");
             context->AccumReduceCost(span);
         } catch (std::exception &ex) {
             ENGINE_LOG_ERROR << "SearchTask encounter exception: " << ex.what();

cpp/src/server/ServerConfig.cpp

@@ -86,11 +86,12 @@ ErrorCode ServerConfig::ValidateConfig() {
 ErrorCode
 ServerConfig::CheckServerConfig() {
 /*
-    server_config:
-    address: 0.0.0.0            # milvus server ip address
-    port: 19530                 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
-    gpu_index: 0                # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1
-    mode: single                # milvus deployment type: single, cluster, read_only
+  server_config:
+  address: 0.0.0.0            # milvus server ip address (IPv4)
+  port: 19530                 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
+  mode: single                # milvus deployment type: single, cluster, read_only
+  time_zone: UTC+8            # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time
+
 */
     bool okay = true;
     ConfigNode server_config = GetConfig(CONFIG_SERVER);
@@ -144,20 +145,20 @@ ServerConfig::CheckServerConfig() {
 ErrorCode
 ServerConfig::CheckDBConfig() {
 /*
-    db_config:
-    db_path: @MILVUS_DB_PATH@             # milvus data storage path
-    db_slave_path:                        # secondry data storage path, split by semicolon
-    parallel_reduce: false                # use multi-threads to reduce topk result
-
-    # URI format: dialect://username:password@host:port/database
-    # All parts except dialect are optional, but you MUST include the delimiters
-    # Currently dialect supports mysql or sqlite
-    db_backend_url: sqlite://:@:/
-
-    archive_disk_threshold: 0        # triger archive action if storage size exceed this value, 0 means no limit, unit: GB
-    archive_days_threshold: 0        # files older than x days will be archived, 0 means no limit, unit: day
-    insert_buffer_size: 4            # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.
-    # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB
+  db_config:
+  db_path: @MILVUS_DB_PATH@             # milvus data storage path
+  db_slave_path:                        # secondry data storage path, split by semicolon
+
+  # URI format: dialect://username:password@host:port/database
+  # All parts except dialect are optional, but you MUST include the delimiters
+  # Currently dialect supports mysql or sqlite
+  db_backend_url: sqlite://:@:/
+
+  archive_disk_threshold: 0        # triger archive action if storage size exceed this value, 0 means no limit, unit: GB
+  archive_days_threshold: 0        # files older than x days will be archived, 0 means no limit, unit: day
+  insert_buffer_size: 4            # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.
+                                   # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB
+  build_index_gpu: 0               # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1
 */
     bool okay = true;
     ConfigNode db_config = GetConfig(CONFIG_DB);
@@ -249,15 +250,13 @@ ServerConfig::CheckMetricConfig() {
 ErrorCode
 ServerConfig::CheckCacheConfig() {
 /*
-    cache_config:
-    cpu_cache_capacity: 16            # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
-    cpu_cache_free_percent: 0.85      # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
-    insert_cache_immediately: false   # insert data will be load into cache immediately for hot query
-    gpu_cache_capacity: 5             # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory
-    gpu_cache_free_percent: 0.85      # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
-    gpu_ids:                          # gpu id
-    - 0
-    - 1
+  cache_config:
+  cpu_cache_capacity: 16            # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
+  cpu_cache_free_percent: 0.85      # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
+  insert_cache_immediately: false   # insert data will be load into cache immediately for hot query
+  gpu_cache_capacity: 5             # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory
+  gpu_cache_free_percent: 0.85      # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
+
 */
     bool okay = true;
     ConfigNode cache_config = GetConfig(CONFIG_CACHE);
@@ -379,48 +378,47 @@ ServerConfig::CheckEngineConfig() {
 ErrorCode
 ServerConfig::CheckResourceConfig() {
 /*
-
-    resource_config:
-    # resource list, length: 0~N
-    # please set a DISK resource and a CPU resource least, or system will not return query result.
-    #
-      # example:
-    # resource_name:               # resource name, just using in connections below
-    #   type: DISK                 # resource type, optional: DISK/CPU/GPU
-    #   device_id: 0
-    #   enable_executor: false     # if is enable executor, optional: true, false
-
-    resources:
+  resource_config:
+  # resource list, length: 0~N
+  # please set a DISK resource and a CPU resource least, or system will not return query result.
+  #
+  # example:
+  # resource_name:               # resource name, just using in connections below
+  #   type: DISK                 # resource type, optional: DISK/CPU/GPU
+  #   device_id: 0
+  #   enable_executor: false     # if is enable executor, optional: true, false
+
+  resources:
     ssda:
-    type: DISK
-    device_id: 0
-    enable_executor: false
+      type: DISK
+      device_id: 0
+      enable_executor: false
 
     cpu:
-    type: CPU
-    device_id: 0
-    enable_executor: false
+      type: CPU
+      device_id: 0
+      enable_executor: true
 
     gpu0:
-    type: GPU
-    device_id: 0
-    enable_executor: true
-    gpu_resource_num: 2
-    pinned_memory: 300
-    temp_memory: 300
-
-    # connection list, length: 0~N
-    # example:
-    # connection_name:
-    #   speed: 100                                        # unit: MS/s
-    #   endpoint: ${resource_name}===${resource_name}
-    connections:
+      type: GPU
+      device_id: 0
+      enable_executor: false
+      gpu_resource_num: 2
+      pinned_memory: 300
+      temp_memory: 300
+
+  # connection list, length: 0~N
+  # example:
+  # connection_name:
+  #   speed: 100                                        # unit: MS/s
+  #   endpoint: ${resource_name}===${resource_name}
+  connections:
     io:
-    speed: 500
-    endpoint: ssda===cpu
+      speed: 500
+      endpoint: ssda===cpu
     pcie0:
-    speed: 11000
-    endpoint: cpu===gpu0
+      speed: 11000
+      endpoint: cpu===gpu0
 */
     bool okay = true;
     server::ConfigNode resource_config = GetConfig(CONFIG_RESOURCE);

cpp/src/server/grpc_impl/GrpcRequestTask.cpp

@@ -572,7 +572,11 @@ SearchTask::Create(const ::milvus::grpc::SearchParam *search_vector_infos,
 ErrorCode
 SearchTask::OnExecute() {
     try {
-        TimeRecorder rc("SearchTask");
+        int64_t top_k = search_param_->topk();
+        int64_t nprobe = search_param_->nprobe();
+
+        std::string hdr = "SearchTask(k=" + std::to_string(top_k) + ", nprob=" + std::to_string(nprobe) + ")";
+        TimeRecorder rc(hdr);
 
         //step 1: check table name
         std::string table_name_ = search_param_->table_name();
@@ -594,13 +598,11 @@ SearchTask::OnExecute() {
         }
 
         //step 3: check search parameter
-        int64_t top_k = search_param_->topk();
         res = ValidationUtil::ValidateSearchTopk(top_k, table_info);
         if (res != SERVER_SUCCESS) {
             return SetError(res, "Invalid topk: " + std::to_string(top_k));
         }
 
-        int64_t nprobe = search_param_->nprobe();
         res = ValidationUtil::ValidateSearchNprobe(nprobe, table_info);
         if (res != SERVER_SUCCESS) {
             return SetError(res, "Invalid nprobe: " + std::to_string(nprobe));

cpp/src/utils/ValidationUtil.cpp

@@ -224,6 +224,9 @@ ValidationUtil::ValidateDbURI(const std::string &uri) {
             okay = false;
         }
 
+/*
+ *      Could be DNS, skip checking
+ *
         std::string host = pieces_match[4].str();
         if (!host.empty() && host != "localhost") {
             if (ValidateIpAddress(host) != SERVER_SUCCESS) {
@@ -231,6 +234,7 @@ ValidationUtil::ValidateDbURI(const std::string &uri) {
                 okay = false;
             }
         }
+*/
 
         std::string port = pieces_match[5].str();
         if (!port.empty()) {