diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 747a61ddbea1656d822aa7e49ea2dcb32644fabc..f5560a547684992693a78c08aa0d57ccf7293cff 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -898,34 +898,33 @@ DBImpl::BackgroundBuildIndex() { meta_ptr_->FilesToIndex(to_index_files); Status status; -// scheduler::BuildIndexJobPtr -// job = std::make_shared(0, meta_ptr_, options_); + scheduler::BuildIndexJobPtr + job = std::make_shared(0, meta_ptr_, options_); // step 2: put build index task to scheduler -// for (auto &file : to_index_files) { -// scheduler::TableFileSchemaPtr file_ptr = std::make_shared(file); -// job->AddToIndexFiles(file_ptr); -// } -// scheduler::JobMgrInst::GetInstance()->Put(job); -// job->WaitBuildIndexFinish(); -// if (!job->GetStatus().ok()) { -// Status status = job->GetStatus(); -// ENGINE_LOG_ERROR << "Building index failed: " << status.ToString(); -// } - for (auto &file : to_index_files) { - std::cout << "get to index file" << std::endl; - status = BuildIndex(file); - if (!status.ok()) { - ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); - } - - if (shutting_down_.load(std::memory_order_acquire)) { - ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action"; - break; - } + scheduler::TableFileSchemaPtr file_ptr = std::make_shared(file); + job->AddToIndexFiles(file_ptr); + } + scheduler::JobMgrInst::GetInstance()->Put(job); + job->WaitBuildIndexFinish(); + if (!job->GetStatus().ok()) { + Status status = job->GetStatus(); + ENGINE_LOG_ERROR << "Building index failed: " << status.ToString(); } +// for (auto &file : to_index_files) { +// status = BuildIndex(file); +// if (!status.ok()) { +// ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); +// } +// +// if (shutting_down_.load(std::memory_order_acquire)) { +// ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action"; +// break; +// } +// } + ENGINE_LOG_TRACE << "Background build index thread exit"; } diff --git a/cpp/src/db/engine/ExecutionEngineImpl.h b/cpp/src/db/engine/ExecutionEngineImpl.h index 5793763fdd77ac1285713811cae91258b2f8978e..56a584999439b87a962a1f762ac497f62621454b 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.h +++ b/cpp/src/db/engine/ExecutionEngineImpl.h @@ -59,7 +59,7 @@ class ExecutionEngineImpl : public ExecutionEngine { CopyToGpu(uint64_t device_id) override; Status - CopyToIndexFileToGpu(uint64_t device_id) override; + CopyToIndexFileToGpu(uint64_t device_id) override; Status CopyToCpu() override; diff --git a/cpp/src/scheduler/job/BuildIndexJob.cpp b/cpp/src/scheduler/job/BuildIndexJob.cpp index 9dc21f9bec90730c125a68e0f579d01441205cb5..f3efe294b223645e4635ede0ff86ca3edb20eafb 100644 --- a/cpp/src/scheduler/job/BuildIndexJob.cpp +++ b/cpp/src/scheduler/job/BuildIndexJob.cpp @@ -15,16 +15,14 @@ // specific language governing permissions and limitations // under the License. -#include "BuildIndexJob.h" #include "utils/Log.h" - +#include "BuildIndexJob.h" namespace milvus { namespace scheduler { BuildIndexJob::BuildIndexJob(JobId id, engine::meta::MetaPtr meta_ptr, engine::DBOptions options) : Job(id, JobType::BUILD), meta_ptr_(std::move(meta_ptr)), options_(std::move(options)) { - } bool @@ -54,6 +52,5 @@ BuildIndexJob::BuildIndexDone(size_t to_index_id) { SERVER_LOG_DEBUG << "BuildIndexJob " << id() << " finish index file: " << to_index_id; } - -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/job/BuildIndexJob.h b/cpp/src/scheduler/job/BuildIndexJob.h index e4fa7b1fd8c25128f891dbcb611a096d82da7358..cc50d70fc90dcc361a2cda72c142723aa258e2f3 100644 --- a/cpp/src/scheduler/job/BuildIndexJob.h +++ b/cpp/src/scheduler/job/BuildIndexJob.h @@ -87,5 +87,5 @@ class BuildIndexJob : public Job { using BuildIndexJobPtr = std::shared_ptr; -} -} +} // namespace scheduler +} // namespace milvus diff --git a/cpp/src/scheduler/task/BuildIndexTask.cpp b/cpp/src/scheduler/task/BuildIndexTask.cpp index 2794c1b45d8255edc7dc3564e4471699ca0c3707..2332e652b7a044cb43f703416778bc55b6646f00 100644 --- a/cpp/src/scheduler/task/BuildIndexTask.cpp +++ b/cpp/src/scheduler/task/BuildIndexTask.cpp @@ -26,6 +26,7 @@ #include #include + namespace milvus { namespace scheduler { @@ -61,7 +62,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) { error_msg = "Wrong load type"; stat = Status(SERVER_UNEXPECTED_ERROR, error_msg); } - } catch (std::exception& ex) { + } catch (std::exception &ex) { // typical error: out of disk space or permition denied error_msg = "Failed to load to_index file: " + std::string(ex.what()); stat = Status(SERVER_UNEXPECTED_ERROR, error_msg); @@ -69,7 +70,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) { if (!stat.ok()) { Status s; - if(stat.ToString().find("out of memory") != std::string::npos) { + if (stat.ToString().find("out of memory") != std::string::npos) { error_msg = "out of memory: " + type_str; s = Status(SERVER_UNEXPECTED_ERROR, error_msg); } else { @@ -108,7 +109,7 @@ XBuildIndexTask::Execute() { if (auto job = job_.lock()) { auto build_index_job = std::static_pointer_cast(job); std::string location = file_->location_; - EngineType engine_type = (EngineType)file_->engine_type_; + EngineType engine_type = (EngineType) file_->engine_type_; std::shared_ptr index; // step 2: create table file @@ -116,7 +117,7 @@ XBuildIndexTask::Execute() { table_file.table_id_ = file_->table_id_; table_file.date_ = file_->date_; table_file.file_type_ = - engine::meta::TableFileSchema::NEW_INDEX; // for multi-db-path, distribute index file averagely to each path + engine::meta::TableFileSchema::NEW_INDEX; engine::meta::MetaPtr meta_ptr = build_index_job->meta(); Status status = build_index_job->meta()->CreateTableFile(table_file); @@ -204,7 +205,8 @@ XBuildIndexTask::Execute() { table_file.file_type_ = engine::meta::TableFileSchema::TO_DELETE; status = meta_ptr->UpdateTableFile(table_file); - ENGINE_LOG_DEBUG << "Failed to up date file to index, mark file: " << table_file.file_id_ << " to to_delete"; + ENGINE_LOG_DEBUG << "Failed to up date file to index, mark file: " << table_file.file_id_ + << " to to_delete"; } build_index_job->BuildIndexDone(to_index_id_); diff --git a/cpp/src/scheduler/task/BuildIndexTask.h b/cpp/src/scheduler/task/BuildIndexTask.h index 770295a042971753d33be121ee2dbc60555e2fe7..a601f760ae83edddaafb5537b69dda163a8ec9c4 100644 --- a/cpp/src/scheduler/task/BuildIndexTask.h +++ b/cpp/src/scheduler/task/BuildIndexTask.h @@ -41,7 +41,6 @@ class XBuildIndexTask : public Task { size_t to_index_id_ = 0; int to_index_type_ = 0; ExecutionEnginePtr to_index_engine_ = nullptr; - }; } // namespace scheduler diff --git a/cpp/unittest/metrics/utils.cpp b/cpp/unittest/metrics/utils.cpp index c2a53babc3e881775885adb365710eb1989aaf2c..e345923b7b59ef094e23e101fd102c920f7ea894 100644 --- a/cpp/unittest/metrics/utils.cpp +++ b/cpp/unittest/metrics/utils.cpp @@ -66,6 +66,7 @@ ms::engine::DBOptions MetricTest::GetOptions() { } void MetricTest::SetUp() { + boost::filesystem::remove_all("/tmp/milvus_test"); InitLog(); auto options = GetOptions(); db_ = ms::engine::DBFactory::Build(options);