Merge pull request #492 from fishpenguin/0.6.0-yk-refactor-scheduler

Add log in scheduler/optimizer and gpu no usage during index build

CHANGELOG.md

@@ -20,6 +20,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#440 - Query API in customization still uses old version
 - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version
 - \#458 - Index data is not compatible between 0.5 and 0.6
+- \#486 - gpu no usage during index building
 
 ## Feature
 - \#12 - Pure CPU version for Milvus
@@ -28,6 +29,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#226 - Experimental shards middleware for Milvus
 - \#227 - Support new index types SPTAG-KDT and SPTAG-BKT
 - \#346 - Support build index with multiple gpu
+- \#488 - Add log in scheduler/optimizer
 
 ## Improvement
 - \#255 - Add ivfsq8 test report detailed version

core/src/scheduler/SchedInst.h

@@ -106,6 +106,25 @@ class OptimizerInst {
                 server::Config& config = server::Config::GetInstance();
                 config.GetGpuResourceConfigEnable(enable_gpu);
                 if (enable_gpu) {
+                    std::vector<int64_t> build_gpus;
+                    std::vector<int64_t> search_gpus;
+                    int64_t gpu_search_threshold;
+                    config.GetGpuResourceConfigBuildIndexResources(build_gpus);
+                    config.GetGpuResourceConfigSearchResources(search_gpus);
+                    config.GetEngineConfigGpuSearchThreshold(gpu_search_threshold);
+                    std::string build_msg = "Build index gpu:";
+                    for (auto build_id : build_gpus) {
+                        build_msg.append(" gpu" + std::to_string(build_id));
+                    }
+                    SERVER_LOG_DEBUG << build_msg;
+
+                    std::string search_msg = "Search gpu:";
+                    for (auto search_id : search_gpus) {
+                        search_msg.append(" gpu" + std::to_string(search_id));
+                    }
+                    search_msg.append(". gpu_search_threshold:" + std::to_string(gpu_search_threshold));
+                    SERVER_LOG_DEBUG << search_msg;
+
                     pass_list.push_back(std::make_shared<BuildIndexPass>());
                     pass_list.push_back(std::make_shared<FaissFlatPass>());
                     pass_list.push_back(std::make_shared<FaissIVFFlatPass>());

core/src/scheduler/optimizer/BuildIndexPass.cpp

@@ -26,8 +26,7 @@ namespace scheduler {
 void
 BuildIndexPass::Init() {
     server::Config& config = server::Config::GetInstance();
-    std::vector<int64_t> build_resources;
-    Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources);
+    Status s = config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids_);
     if (!s.ok()) {
         throw;
     }
@@ -38,13 +37,16 @@ BuildIndexPass::Run(const TaskPtr& task) {
     if (task->Type() != TaskType::BuildIndexTask)
         return false;
 
-    if (build_gpu_ids_.empty())
+    if (build_gpu_ids_.empty()) {
+        SERVER_LOG_WARNING << "BuildIndexPass cannot get build index gpu!";
         return false;
+    }
 
     ResourcePtr res_ptr;
     res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_gpu_ids_[specified_gpu_id_]);
     auto label = std::make_shared<SpecResLabel>(std::weak_ptr<Resource>(res_ptr));
     task->label() = label;
+    SERVER_LOG_DEBUG << "Specify gpu" << specified_gpu_id_ << " to build index!";
 
     specified_gpu_id_ = (specified_gpu_id_ + 1) % build_gpu_ids_.size();
     return true;

core/src/scheduler/optimizer/BuildIndexPass.h

@@ -45,7 +45,7 @@ class BuildIndexPass : public Pass {
 
  private:
     uint64_t specified_gpu_id_ = 0;
-    std::vector<int32_t> build_gpu_ids_;
+    std::vector<int64_t> build_gpu_ids_;
 };
 
 using BuildIndexPassPtr = std::shared_ptr<BuildIndexPass>;

core/src/scheduler/optimizer/FaissFlatPass.cpp

@@ -54,9 +54,11 @@ FaissFlatPass::Run(const TaskPtr& task) {
     auto search_job = std::static_pointer_cast<SearchJob>(search_task->job_.lock());
     ResourcePtr res_ptr;
     if (search_job->nq() < threshold_) {
+        SERVER_LOG_DEBUG << "FaissFlatPass: nq < gpu_search_threshold, specify cpu to search!";
         res_ptr = ResMgrInst::GetInstance()->GetResource("cpu");
     } else {
         auto best_device_id = count_ % gpus.size();
+        SERVER_LOG_DEBUG << "FaissFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id << " to search!";
         count_++;
         res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);
     }

core/src/scheduler/optimizer/FaissIVFFlatPass.cpp

@@ -54,9 +54,12 @@ FaissIVFFlatPass::Run(const TaskPtr& task) {
     auto search_job = std::static_pointer_cast<SearchJob>(search_task->job_.lock());
     ResourcePtr res_ptr;
     if (search_job->nq() < threshold_) {
+        SERVER_LOG_DEBUG << "FaissIVFFlatPass: nq < gpu_search_threshold, specify cpu to search!";
         res_ptr = ResMgrInst::GetInstance()->GetResource("cpu");
     } else {
         auto best_device_id = count_ % gpus.size();
+        SERVER_LOG_DEBUG << "FaissIVFFlatPass: nq > gpu_search_threshold, specify gpu" << best_device_id
+                         << " to search!";
         count_++;
         res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);
     }

core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp

@@ -51,9 +51,12 @@ FaissIVFSQ8HPass::Run(const TaskPtr& task) {
     auto search_job = std::static_pointer_cast<SearchJob>(search_task->job_.lock());
     ResourcePtr res_ptr;
     if (search_job->nq() < threshold_) {
+        SERVER_LOG_DEBUG << "FaissIVFSQ8HPass: nq < gpu_search_threshold, specify cpu to search!";
         res_ptr = ResMgrInst::GetInstance()->GetResource("cpu");
     } else {
         auto best_device_id = count_ % gpus.size();
+        SERVER_LOG_DEBUG << "FaissIVFSQ8HPass: nq > gpu_search_threshold, specify gpu" << best_device_id
+                         << " to search!";
         count_++;
         res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);
     }

core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp

@@ -54,9 +54,12 @@ FaissIVFSQ8Pass::Run(const TaskPtr& task) {
     auto search_job = std::static_pointer_cast<SearchJob>(search_task->job_.lock());
     ResourcePtr res_ptr;
     if (search_job->nq() < threshold_) {
+        SERVER_LOG_DEBUG << "FaissIVFSQ8Pass: nq < gpu_search_threshold, specify cpu to search!";
         res_ptr = ResMgrInst::GetInstance()->GetResource("cpu");
     } else {
         auto best_device_id = count_ % gpus.size();
+        SERVER_LOG_DEBUG << "FaissIVFSQ8Pass: nq > gpu_search_threshold, specify gpu" << best_device_id
+                         << " to search!";
         count_++;
         res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);
     }

core/src/scheduler/optimizer/FallbackPass.cpp

@@ -33,6 +33,7 @@ FallbackPass::Run(const TaskPtr& task) {
         return false;
     }
     // NEVER be empty
+    SERVER_LOG_DEBUG << "FallbackPass!";
     auto cpu = ResMgrInst::GetInstance()->GetCpuResources()[0];
     auto label = std::make_shared<SpecResLabel>(cpu);
     task->label() = label;

core/src/scheduler/task/BuildIndexTask.cpp

@@ -85,7 +85,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) {
 
         size_t file_size = to_index_engine_->PhysicalSize();
 
-        std::string info = "Load file id:" + std::to_string(file_->id_) +
+        std::string info = "Load file id:" + std::to_string(file_->id_) + " " + type_str +
                            " file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) +
                            " bytes from location: " + file_->location_ + " totally cost";
         double span = rc.ElapseFromBegin(info);