Merge branch 'master' into 0.6.0

core/src/db/DBImpl.cpp

@@ -84,12 +84,12 @@ DBImpl::Start() {
         return Status::OK();
     }
 
-    ENGINE_LOG_TRACE << "DB service start";
+    // ENGINE_LOG_TRACE << "DB service start";
     shutting_down_.store(false, std::memory_order_release);
 
     // for distribute version, some nodes are read only
     if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) {
-        ENGINE_LOG_TRACE << "StartTimerTasks";
+        // ENGINE_LOG_TRACE << "StartTimerTasks";
         bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this);
     }
 
@@ -114,7 +114,7 @@ DBImpl::Stop() {
         meta_ptr_->CleanUp();
     }
 
-    ENGINE_LOG_TRACE << "DB service stop";
+    // ENGINE_LOG_TRACE << "DB service stop";
     return Status::OK();
 }
 
@@ -558,7 +558,7 @@ DBImpl::StartMetricTask() {
         return;
     }
 
-    ENGINE_LOG_TRACE << "Start metric task";
+    // ENGINE_LOG_TRACE << "Start metric task";
 
     server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL);
     int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
@@ -584,7 +584,7 @@ DBImpl::StartMetricTask() {
     server::Metrics::GetInstance().GPUTemperature();
     server::Metrics::GetInstance().CPUTemperature();
 
-    ENGINE_LOG_TRACE << "Metric task finished";
+    // ENGINE_LOG_TRACE << "Metric task finished";
 }
 
 Status
@@ -756,7 +756,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) {
 
 void
 DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
-    ENGINE_LOG_TRACE << "Background compaction thread start";
+    // ENGINE_LOG_TRACE << " Background compaction thread start";
 
     Status status;
     for (auto& table_id : table_ids) {
@@ -779,7 +779,7 @@ DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
     }
     meta_ptr_->CleanUpFilesWithTTL(ttl);
 
-    ENGINE_LOG_TRACE << "Background compaction thread exit";
+    // ENGINE_LOG_TRACE << " Background compaction thread exit";
 }
 
 void
@@ -812,7 +812,7 @@ DBImpl::StartBuildIndexTask(bool force) {
 
 void
 DBImpl::BackgroundBuildIndex() {
-    ENGINE_LOG_TRACE << "Background build index thread start";
+    // ENGINE_LOG_TRACE << "Background build index thread start";
 
     std::unique_lock<std::mutex> lock(build_index_mutex_);
     meta::TableFilesSchema to_index_files;
@@ -835,7 +835,7 @@ DBImpl::BackgroundBuildIndex() {
         }
     }
 
-    ENGINE_LOG_TRACE << "Background build index thread exit";
+    // ENGINE_LOG_TRACE << "Background build index thread exit";
 }
 
 Status

core/src/scheduler/task/BuildIndexTask.cpp

@@ -146,8 +146,7 @@ XBuildIndexTask::Execute() {
             status = meta_ptr->UpdateTableFile(table_file);
             ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
 
-            std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough"
-                      << std::endl;
+            ENGINE_LOG_ERROR << "Failed to build index, index file is too large or gpu memory is not enough";
 
             build_index_job->BuildIndexDone(to_index_id_);
             build_index_job->GetStatus() = Status(DB_ERROR, msg);
@@ -179,8 +178,8 @@ XBuildIndexTask::Execute() {
             status = meta_ptr->UpdateTableFile(table_file);
             ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
 
-            std::cout << "ERROR: failed to persist index file: " << table_file.location_
-                      << ", possible out of disk space" << std::endl;
+            ENGINE_LOG_ERROR << "Failed to persist index file: " << table_file.location_
+                             << ", possible out of disk space";
 
             build_index_job->BuildIndexDone(to_index_id_);
             build_index_job->GetStatus() = Status(DB_ERROR, msg);

core/src/wrapper/VecImpl.cpp

@@ -31,7 +31,7 @@
 
 /*
  * no parameter check in this layer.
- * only responible for index combination
+ * only responsible for index combination
  */
 
 namespace milvus {
@@ -246,5 +246,130 @@ BFIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const
     return Status::OK();
 }
 
+// TODO(linxj): add lock here.
+Status
+IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt,
+                      const float* xt) {
+    try {
+        dim = cfg->d;
+        auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
+
+        auto preprocessor = index_->BuildPreprocessor(dataset, cfg);
+        index_->set_preprocessor(preprocessor);
+        auto model = index_->Train(dataset, cfg);
+        index_->set_index_model(model);
+        index_->Add(dataset, cfg);
+
+        if (auto device_index = std::dynamic_pointer_cast<knowhere::GPUIndex>(index_)) {
+            auto host_index = device_index->CopyGpuToCpu(Config());
+            index_ = host_index;
+            type = ConvertToCpuIndexType(type);
+        } else {
+            WRAPPER_LOG_ERROR << "Build IVFMIXIndex Failed";
+            return Status(KNOWHERE_ERROR, "Build IVFMIXIndex Failed");
+        }
+    } catch (knowhere::KnowhereException& e) {
+        WRAPPER_LOG_ERROR << e.what();
+        return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
+    } catch (std::exception& e) {
+        WRAPPER_LOG_ERROR << e.what();
+        return Status(KNOWHERE_ERROR, e.what());
+    }
+    return Status::OK();
+}
+
+Status
+IVFMixIndex::Load(const knowhere::BinarySet& index_binary) {
+    index_->Load(index_binary);
+    dim = Dimension();
+    return Status::OK();
+}
+
+knowhere::QuantizerPtr
+IVFHybridIndex::LoadQuantizer(const Config& conf) {
+    // TODO(linxj): Hardcode here
+    if (auto new_idx = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_)) {
+        return new_idx->LoadQuantizer(conf);
+    } else {
+        WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type);
+    }
+}
+
+Status
+IVFHybridIndex::SetQuantizer(const knowhere::QuantizerPtr& q) {
+    try {
+        // TODO(linxj): Hardcode here
+        if (auto new_idx = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_)) {
+            new_idx->SetQuantizer(q);
+        } else {
+            WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type);
+            return Status(KNOWHERE_ERROR, "not supported");
+        }
+    } catch (knowhere::KnowhereException& e) {
+        WRAPPER_LOG_ERROR << e.what();
+        return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
+    } catch (std::exception& e) {
+        WRAPPER_LOG_ERROR << e.what();
+        return Status(KNOWHERE_ERROR, e.what());
+    }
+    return Status::OK();
+}
+
+Status
+IVFHybridIndex::UnsetQuantizer() {
+    try {
+        // TODO(linxj): Hardcode here
+        if (auto new_idx = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_)) {
+            new_idx->UnsetQuantizer();
+        } else {
+            WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type);
+            return Status(KNOWHERE_ERROR, "not supported");
+        }
+    } catch (knowhere::KnowhereException& e) {
+        WRAPPER_LOG_ERROR << e.what();
+        return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
+    } catch (std::exception& e) {
+        WRAPPER_LOG_ERROR << e.what();
+        return Status(KNOWHERE_ERROR, e.what());
+    }
+    return Status::OK();
+}
+
+VecIndexPtr
+IVFHybridIndex::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) {
+    try {
+        // TODO(linxj): Hardcode here
+        if (auto new_idx = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_)) {
+            return std::make_shared<IVFHybridIndex>(new_idx->LoadData(q, conf), type);
+        } else {
+            WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type);
+        }
+    } catch (knowhere::KnowhereException& e) {
+        WRAPPER_LOG_ERROR << e.what();
+    } catch (std::exception& e) {
+        WRAPPER_LOG_ERROR << e.what();
+    }
+    return nullptr;
+}
+
+std::pair<VecIndexPtr, knowhere::QuantizerPtr>
+IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& cfg) {
+    try {
+        // TODO(linxj): Hardcode here
+        if (auto hybrid_idx = std::dynamic_pointer_cast<knowhere::IVFSQHybrid>(index_)) {
+            auto pair = hybrid_idx->CopyCpuToGpuWithQuantizer(device_id, cfg);
+            auto new_idx = std::make_shared<IVFHybridIndex>(pair.first, type);
+            return std::make_pair(new_idx, pair.second);
+        } else {
+            WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type);
+        }
+    } catch (knowhere::KnowhereException& e) {
+        WRAPPER_LOG_ERROR << e.what();
+    } catch (std::exception& e) {
+        WRAPPER_LOG_ERROR << e.what();
+    }
+    return std::make_pair(nullptr, nullptr);
+}
+
 }  // namespace engine
 }  // namespace milvus