From 56d7c1523dbb4d323742c91712be936e85a77167 Mon Sep 17 00:00:00 2001 From: wxyu Date: Tue, 8 Oct 2019 19:58:37 +0800 Subject: [PATCH] MS-619 Add optimizer class in scheduler Former-commit-id: d0f8950c8ca6e823df307a324d3bd56828b80801 --- cpp/CHANGELOG.md | 1 + cpp/src/CMakeLists.txt | 2 + cpp/src/scheduler/optimizer/HybridPass.cpp | 35 +++++++++++++++ cpp/src/scheduler/optimizer/HybridPass.h | 48 ++++++++++++++++++++ cpp/src/scheduler/optimizer/Optimizer.cpp | 43 ++++++++++++++++++ cpp/src/scheduler/optimizer/Optimizer.h | 51 ++++++++++++++++++++++ cpp/src/scheduler/optimizer/Pass.h | 46 +++++++++++++++++++ cpp/src/scheduler/task/Task.h | 1 - 8 files changed, 226 insertions(+), 1 deletion(-) create mode 100644 cpp/src/scheduler/optimizer/HybridPass.cpp create mode 100644 cpp/src/scheduler/optimizer/HybridPass.h create mode 100644 cpp/src/scheduler/optimizer/Optimizer.cpp create mode 100644 cpp/src/scheduler/optimizer/Optimizer.h create mode 100644 cpp/src/scheduler/optimizer/Pass.h diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 5ff949e7..10ab94f2 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -26,6 +26,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-608 - Update TODO names - MS-609 - Update task construct function - MS-611 - Add resources validity check in ResourceMgr +- MS-619 - Add optimizer class in scheduler ## New Feature diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 9810c6b7..0005edba 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -50,6 +50,7 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/job scheduler_job_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/optimizer scheduler_optimizer_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_files) set(scheduler_files @@ -57,6 +58,7 @@ set(scheduler_files ${scheduler_action_files} ${scheduler_event_files} ${scheduler_job_files} + ${scheduler_optimizer_files} ${scheduler_resource_files} ${scheduler_task_files} ) diff --git a/cpp/src/scheduler/optimizer/HybridPass.cpp b/cpp/src/scheduler/optimizer/HybridPass.cpp new file mode 100644 index 00000000..1b0ab7c4 --- /dev/null +++ b/cpp/src/scheduler/optimizer/HybridPass.cpp @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "HybridPass.h" +#include "scheduler/task/SearchTask.h" + +namespace milvus { +namespace scheduler { + +bool +HybridPass::Run(const TaskPtr& task) { + // TODO: Index::IVFSQ8Hybrid, if nq < threshold set cpu, else set gpu + if (task->Type() != TaskType::SearchTask) return false; + auto search_task = std::static_pointer_cast(task); + // if (search_task->file_->engine_type_ == engine::EngineType::FAISS_IVFSQ8Hybrid) + return false; +} + +} +} + diff --git a/cpp/src/scheduler/optimizer/HybridPass.h b/cpp/src/scheduler/optimizer/HybridPass.h new file mode 100644 index 00000000..e043bf8a --- /dev/null +++ b/cpp/src/scheduler/optimizer/HybridPass.h @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class HybridPass : public Pass { + public: + HybridPass() = default; + + public: + bool + Run(const TaskPtr& task) override; +}; + +using HybridPassPtr = std::shared_ptr; + +} +} + diff --git a/cpp/src/scheduler/optimizer/Optimizer.cpp b/cpp/src/scheduler/optimizer/Optimizer.cpp new file mode 100644 index 00000000..517cd85f --- /dev/null +++ b/cpp/src/scheduler/optimizer/Optimizer.cpp @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "Optimizer.h" + +namespace milvus { +namespace scheduler { + +void +Optimizer::Init() { + for (auto& pass : pass_list_) { + pass->Init(); + } +} + +bool +Optimizer::Run(const TaskPtr& task) { + for (auto& pass : pass_list_) { + if (pass->Run(task)) { + return true; + } + } + + return false; +} + +} +} + diff --git a/cpp/src/scheduler/optimizer/Optimizer.h b/cpp/src/scheduler/optimizer/Optimizer.h new file mode 100644 index 00000000..cbb13a8f --- /dev/null +++ b/cpp/src/scheduler/optimizer/Optimizer.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class Optimizer { + public: + Optimizer() = default; + + void + Init(); + + bool + Run(const TaskPtr &task); + + private: + std::vector pass_list_; +}; + +} +} + diff --git a/cpp/src/scheduler/optimizer/Pass.h b/cpp/src/scheduler/optimizer/Pass.h new file mode 100644 index 00000000..1c8def41 --- /dev/null +++ b/cpp/src/scheduler/optimizer/Pass.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "scheduler/task/Task.h" + +namespace milvus { +namespace scheduler { + +class Pass { + public: + virtual void + Init() {} + + virtual bool + Run(const TaskPtr& task) = 0; +}; +using PassPtr = std::shared_ptr; + +} +} diff --git a/cpp/src/scheduler/task/Task.h b/cpp/src/scheduler/task/Task.h index 493adb41..da012153 100644 --- a/cpp/src/scheduler/task/Task.h +++ b/cpp/src/scheduler/task/Task.h @@ -84,7 +84,6 @@ class Task { public: Path task_path_; - // std::vector search_contexts_; scheduler::JobWPtr job_; TaskType type_; TaskLabelPtr label_ = nullptr; -- GitLab