From 8d76c708df6fd5746456cd1b4670c386df216d9f Mon Sep 17 00:00:00 2001 From: dinghao Date: Sun, 21 Jun 2020 14:37:27 +0800 Subject: [PATCH] init serving --- CMakeLists.txt | 4 + build.sh | 13 +- include/inference.h | 2 + mindspore/ccsrc/CMakeLists.txt | 2 +- mindspore/ccsrc/session/session.cc | 25 +- serving/CMakeLists.txt | 69 +++++ serving/README.en.md | 36 +++ serving/README.md | 37 +++ serving/core/server.cc | 277 ++++++++++++++++++ serving/core/server.h | 56 ++++ serving/core/util/file_system_operation.cc | 102 +++++++ serving/core/util/file_system_operation.h | 32 ++ serving/core/util/option_parser.cc | 243 +++++++++++++++ serving/core/util/option_parser.h | 84 ++++++ serving/core/util/status.h | 25 ++ serving/core/version_control/model.cc | 33 +++ serving/core/version_control/model.h | 47 +++ .../version_control/version_controller.cc | 134 +++++++++ .../core/version_control/version_controller.h | 71 +++++ serving/cpp_example/CMakeLists.txt | 72 +++++ serving/cpp_example/ms_client.cc | 105 +++++++ serving/cpp_example/ms_server.cc | 67 +++++ serving/main.cc | 29 ++ serving/ms_service.proto | 48 +++ serving/python_example/ms_client.py | 57 ++++ serving/python_example/ms_client_test_call.py | 46 +++ serving/python_example/ms_server.py | 55 ++++ serving/python_example/ms_service_pb2_grpc.py | 96 ++++++ serving/python_example/test_cpu_lenet.py | 91 ++++++ serving/scripts/format_source_code.sh | 105 +++++++ 30 files changed, 2059 insertions(+), 4 deletions(-) create mode 100644 serving/CMakeLists.txt create mode 100644 serving/README.en.md create mode 100644 serving/README.md create mode 100644 serving/core/server.cc create mode 100644 serving/core/server.h create mode 100644 serving/core/util/file_system_operation.cc create mode 100644 serving/core/util/file_system_operation.h create mode 100644 serving/core/util/option_parser.cc create mode 100644 serving/core/util/option_parser.h create mode 100644 serving/core/util/status.h create mode 100644 serving/core/version_control/model.cc create mode 100644 serving/core/version_control/model.h create mode 100644 serving/core/version_control/version_controller.cc create mode 100644 serving/core/version_control/version_controller.h create mode 100644 serving/cpp_example/CMakeLists.txt create mode 100644 serving/cpp_example/ms_client.cc create mode 100644 serving/cpp_example/ms_server.cc create mode 100644 serving/main.cc create mode 100644 serving/ms_service.proto create mode 100644 serving/python_example/ms_client.py create mode 100644 serving/python_example/ms_client_test_call.py create mode 100644 serving/python_example/ms_server.py create mode 100644 serving/python_example/ms_service_pb2_grpc.py create mode 100644 serving/python_example/test_cpu_lenet.py create mode 100755 serving/scripts/format_source_code.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 34521d22d..37c3288f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,4 +96,8 @@ if (ENABLE_TESTCASES) add_subdirectory(tests) endif() +if (ENABLE_SERVING) + add_subdirectory(serving) +endif() + include(cmake/package.cmake) diff --git a/build.sh b/build.sh index 7676665be..04bbe633a 100755 --- a/build.sh +++ b/build.sh @@ -53,6 +53,7 @@ usage() echo " -V Specify the minimum required cuda version, default CUDA 9.2" echo " -I Compile predict, default off" echo " -K Compile with AKG, default off" + echo " -s Enable serving module, default off" } # check value of input is 'on' or 'off' @@ -92,9 +93,9 @@ checkopts() USE_GLOG="on" PREDICT_PLATFORM="" ENABLE_AKG="off" - + ENABLE_SERVING="off" # Process the options - while getopts 'drvj:c:t:hsb:a:g:p:ie:m:I:LRP:Q:D:zM:V:K' opt + while getopts 'drvj:c:t:hsb:a:g:p:ie:m:I:LRP:Q:D:zM:V:K:s' opt do OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]') case "${opt}" in @@ -235,6 +236,10 @@ checkopts() ENABLE_AKG="on" echo "enable compile with akg" ;; + s) + ENABLE_SERVING="on" + echo "enable serving" + ;; *) echo "Unknown option ${opt}!" usage @@ -314,6 +319,10 @@ build_mindspore() if [[ "X$ENABLE_AKG" = "Xon" ]] && [[ "X$ENABLE_D" = "Xon" ]]; then CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_AKG=ON" fi + if [[ "X$ENABLE_SERVING" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_SERVING=ON" + fi + echo "${CMAKE_ARGS}" if [[ "X$INC_BUILD" = "Xoff" ]]; then cmake ${CMAKE_ARGS} ../.. diff --git a/include/inference.h b/include/inference.h index bb4cff7ec..7e5ee27d4 100644 --- a/include/inference.h +++ b/include/inference.h @@ -37,6 +37,8 @@ class MS_API MSSession { }; std::shared_ptr MS_API LoadModel(const char *model_buf, size_t size, const std::string &device); + +void MS_API ExitInference(); } // namespace inference } // namespace mindspore #endif // MINDSPORE_INCLUDE_MS_SESSION_H diff --git a/mindspore/ccsrc/CMakeLists.txt b/mindspore/ccsrc/CMakeLists.txt index c9e224080..80f82fd7e 100644 --- a/mindspore/ccsrc/CMakeLists.txt +++ b/mindspore/ccsrc/CMakeLists.txt @@ -247,7 +247,7 @@ add_library(inference SHARED ${CMAKE_CURRENT_SOURCE_DIR}/session/session.cc ${LOAD_ONNX_SRC} ) -target_link_libraries(inference PRIVATE ${PYTHON_LIB} ${SECUREC_LIBRARY} +target_link_libraries(inference PRIVATE ${PYTHON_LIBRARY} ${SECUREC_LIBRARY} -Wl,--whole-archive mindspore -Wl,--no-whole-archive mindspore_gvar mindspore::protobuf) if (ENABLE_CPU) diff --git a/mindspore/ccsrc/session/session.cc b/mindspore/ccsrc/session/session.cc index bd883dc84..90e02b37f 100644 --- a/mindspore/ccsrc/session/session.cc +++ b/mindspore/ccsrc/session/session.cc @@ -38,6 +38,18 @@ std::shared_ptr LoadModel(const char *model_buf, size_t size, const s return anf_graph; } +void ExitInference() { + auto ms_context = MsContext::GetInstance(); + if (ms_context == nullptr) { + MS_LOG(ERROR) << "Get Context failed!"; + return; + } + if (!ms_context->CloseTsd()) { + MS_LOG(ERROR) << "Inference CloseTsd failed!"; + return; + } +} + std::shared_ptr MSSession::CreateSession(const std::string &device, uint32_t device_id) { auto session = std::make_shared(); auto ret = session->Init(device, device_id); @@ -101,11 +113,14 @@ void Session::RegAllOp() { uint32_t Session::CompileGraph(std::shared_ptr funcGraphPtr) { MS_ASSERT(session_impl_ != nullptr); - return session_impl_->CompileGraph(NOT_NULL(funcGraphPtr)); + auto graph_id = session_impl_->CompileGraph(NOT_NULL(funcGraphPtr)); + py::gil_scoped_release gil_release; + return graph_id; } MultiTensor Session::RunGraph(uint32_t graph_id, const std::vector> &inputs) { std::vector inTensors; + inTensors.resize(inputs.size()); bool has_error = false; std::transform(inputs.begin(), inputs.end(), inTensors.begin(), [&has_error](const std::shared_ptr &tensor_ptr) -> tensor::TensorPtr { @@ -144,6 +159,14 @@ int Session::Init(const std::string &device, uint32_t device_id) { return -1; } session_impl_->Init(device_id); + if (ms_context == nullptr) { + MS_LOG(ERROR) << "Get Context failed!"; + return -1; + } + if (!ms_context->OpenTsd()) { + MS_LOG(ERROR) << "Session init OpenTsd failed!"; + return -1; + } return 0; } diff --git a/serving/CMakeLists.txt b/serving/CMakeLists.txt new file mode 100644 index 000000000..3c1c08ece --- /dev/null +++ b/serving/CMakeLists.txt @@ -0,0 +1,69 @@ +find_package(Threads REQUIRED) + +# This branch assumes that gRPC and all its dependencies are already installed +# on this system, so they can be located by find_package(). + +# Find Protobuf installation +# Looks for protobuf-config.cmake file installed by Protobuf's cmake installation. + +#set(protobuf_MODULE_COMPATIBLE TRUE) +#find_package(Protobuf CONFIG REQUIRED) +#message(STATUS "Using protobuf ${protobuf_VERSION}") +add_library(protobuf::libprotobuf ALIAS protobuf::protobuf) +add_executable(protobuf::libprotoc ALIAS protobuf::protoc) + +set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf) +set(_REFLECTION gRPC::grpc++_reflection) +if(CMAKE_CROSSCOMPILING) + find_program(_PROTOBUF_PROTOC protoc) +else() + set(_PROTOBUF_PROTOC $) +endif() + +# Find gRPC installation +# Looks for gRPCConfig.cmake file installed by gRPC's cmake installation. +find_package(gRPC CONFIG REQUIRED) +message(STATUS "Using gRPC ${gRPC_VERSION}") + +set(_GRPC_GRPCPP gRPC::grpc++) +if(CMAKE_CROSSCOMPILING) + find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin) +else() + set(_GRPC_CPP_PLUGIN_EXECUTABLE $) +endif() + +# Proto file +get_filename_component(hw_proto "ms_service.proto" ABSOLUTE) +get_filename_component(hw_proto_path "${hw_proto}" PATH) + +# Generated sources +set(hw_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.pb.cc") +set(hw_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.pb.h") +set(hw_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.grpc.pb.cc") +set(hw_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.grpc.pb.h") +add_custom_command( + OUTPUT "${hw_proto_srcs}" "${hw_proto_hdrs}" "${hw_grpc_srcs}" "${hw_grpc_hdrs}" + COMMAND ${_PROTOBUF_PROTOC} + ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" + --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" + -I "${hw_proto_path}" + --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}" + "${hw_proto}" + DEPENDS "${hw_proto}") + +# Include generated *.pb.h files +include_directories("${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/core" + "${PROJECT_SOURCE_DIR}/mindspore/ccsrc") +file(GLOB_RECURSE CORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "core/*.cc" "core/util/*.cc" "core/version_control/*.cc") + +list(APPEND SERVING_SRC "main.cc" ${hw_proto_srcs} ${hw_grpc_srcs} ${CORE_SRC_LIST}) + +include_directories(${CMAKE_BINARY_DIR}) +add_executable(ms_serving ${SERVING_SRC}) +target_link_libraries(ms_serving inference mindspore_gvar) +target_link_libraries(ms_serving ${_REFLECTION} ${_GRPC_GRPCPP} ${_PROTOBUF_LIBPROTOBUF} pthread) +if (ENABLE_D) + add_compile_definitions(ENABLE_D) + target_link_libraries(ms_serving ${RUNTIME_LIB}) +endif() diff --git a/serving/README.en.md b/serving/README.en.md new file mode 100644 index 000000000..830b94537 --- /dev/null +++ b/serving/README.en.md @@ -0,0 +1,36 @@ +# serving + +#### Description +A flexible, high-performance serving system for deep learning models + +#### Software Architecture +Software architecture description + +#### Installation + +1. xxxx +2. xxxx +3. xxxx + +#### Instructions + +1. xxxx +2. xxxx +3. xxxx + +#### Contribution + +1. Fork the repository +2. Create Feat_xxx branch +3. Commit your code +4. Create Pull Request + + +#### Gitee Feature + +1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md +2. Gitee blog [blog.gitee.com](https://blog.gitee.com) +3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore) +4. The most valuable open source project [GVP](https://gitee.com/gvp) +5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help) +6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) diff --git a/serving/README.md b/serving/README.md new file mode 100644 index 000000000..b26b9a688 --- /dev/null +++ b/serving/README.md @@ -0,0 +1,37 @@ +# serving + +#### 介绍 +A flexible, high-performance serving system for deep learning models + +#### 软件架构 +软件架构说明 + + +#### 安装教程 + +1. xxxx +2. xxxx +3. xxxx + +#### 使用说明 + +1. xxxx +2. xxxx +3. xxxx + +#### 参与贡献 + +1. Fork 本仓库 +2. 新建 Feat_xxx 分支 +3. 提交代码 +4. 新建 Pull Request + + +#### 码云特技 + +1. 使用 Readme\_XXX.md 来支持不同的语言，例如 Readme\_en.md, Readme\_zh.md +2. 码云官方博客 [blog.gitee.com](https://blog.gitee.com) +3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解码云上的优秀开源项目 +4. [GVP](https://gitee.com/gvp) 全称是码云最有价值开源项目，是码云综合评定出的优秀开源项目 +5. 码云官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help) +6. 码云封面人物是一档用来展示码云会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) diff --git a/serving/core/server.cc b/serving/core/server.cc new file mode 100644 index 000000000..add9d16be --- /dev/null +++ b/serving/core/server.cc @@ -0,0 +1,277 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "core/server.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mindspore/ccsrc/utils/log_adapter.h" +#include "serving/ms_service.grpc.pb.h" +#include "core/util/option_parser.h" +#include "core/version_control/version_controller.h" +#include "mindspore/ccsrc/utils/context/ms_context.h" +#include "core/util/file_system_operation.h" +#include "graphengine/third_party/fwkacllib/inc/runtime/context.h" + +using ms_serving::MSService; +using ms_serving::PredictReply; +using ms_serving::PredictRequest; + +namespace mindspore { +namespace serving { +using MSTensorPtr = std::shared_ptr; + +Status Session::CreatDeviceSession(const std::string &device, uint32_t device_id) { + session_ = inference::MSSession::CreateSession(device + "Inference", device_id); + if (session_ == nullptr) { + MS_LOG(ERROR) << "Creat Session Failed"; + return FAILED; + } + device_type_ = device; + return SUCCESS; +} + +Session &Session::Instance() { + static Session instance; + return instance; +} + +Status Session::Predict(const std::vector &inputs, inference::MultiTensor *outputs) { + if (last_graph_ == nullptr) { + MS_LOG(ERROR) << "the model has not loaded"; + return FAILED; + } + if (session_ == nullptr) { + MS_LOG(ERROR) << "the inference session has not be initialized"; + return FAILED; + } + std::lock_guard lock(mutex_); + MS_LOG(INFO) << "run Predict"; + + *outputs = session_->RunGraph(graph_id_, inputs); + return SUCCESS; +} + +Status Session::Warmup(const MindSporeModelPtr model) { + if (session_ == nullptr) { + MS_LOG(ERROR) << "The CreatDeviceSession should be called, before warmup"; + return FAILED; + } + std::lock_guard lock(mutex_); + size_t size = 0; + std::string file_name = model->GetModelPath() + '/' + model->GetModelName(); + char *graphBuf = ReadFile(file_name.c_str(), &size); + if (graphBuf == nullptr) { + MS_LOG(ERROR) << "Load graph model failed, file name is " << file_name.c_str(); + return FAILED; + } + last_graph_ = inference::LoadModel(graphBuf, size, device_type_); + graph_id_ = session_->CompileGraph(last_graph_); + MS_LOG(INFO) << "Session Warmup"; + return SUCCESS; +} + +Status Session::Clear() { + session_ = nullptr; + return SUCCESS; +} + +namespace { +const std::map type2id_map{ + {ms_serving::MS_UNKNOWN, TypeId::kNumberTypeBegin}, {ms_serving::MS_BOOL, TypeId::kNumberTypeBool}, + {ms_serving::MS_INT8, TypeId::kNumberTypeInt8}, {ms_serving::MS_UINT8, TypeId::kNumberTypeUInt8}, + {ms_serving::MS_INT16, TypeId::kNumberTypeInt16}, {ms_serving::MS_UINT16, TypeId::kNumberTypeUInt16}, + {ms_serving::MS_INT32, TypeId::kNumberTypeInt32}, {ms_serving::MS_UINT32, TypeId::kNumberTypeUInt32}, + {ms_serving::MS_INT64, TypeId::kNumberTypeInt64}, {ms_serving::MS_UINT64, TypeId::kNumberTypeUInt64}, + {ms_serving::MS_FLOAT16, TypeId::kNumberTypeFloat16}, {ms_serving::MS_FLOAT32, TypeId::kNumberTypeFloat32}, + {ms_serving::MS_FLOAT64, TypeId::kNumberTypeFloat64}, +}; + +const std::map id2type_map{ + {TypeId::kNumberTypeBegin, ms_serving::MS_UNKNOWN}, {TypeId::kNumberTypeBool, ms_serving::MS_BOOL}, + {TypeId::kNumberTypeInt8, ms_serving::MS_INT8}, {TypeId::kNumberTypeUInt8, ms_serving::MS_UINT8}, + {TypeId::kNumberTypeInt16, ms_serving::MS_INT16}, {TypeId::kNumberTypeUInt16, ms_serving::MS_UINT16}, + {TypeId::kNumberTypeInt32, ms_serving::MS_INT32}, {TypeId::kNumberTypeUInt32, ms_serving::MS_UINT32}, + {TypeId::kNumberTypeInt64, ms_serving::MS_INT64}, {TypeId::kNumberTypeUInt64, ms_serving::MS_UINT64}, + {TypeId::kNumberTypeFloat16, ms_serving::MS_FLOAT16}, {TypeId::kNumberTypeFloat32, ms_serving::MS_FLOAT32}, + {TypeId::kNumberTypeFloat64, ms_serving::MS_FLOAT64}, +}; +const std::map length_map{ + {ms_serving::MS_UNKNOWN, 0}, + {ms_serving::MS_BOOL, sizeof(bool)}, + {ms_serving::MS_INT8, sizeof(int8_t)}, + {ms_serving::MS_UINT8, sizeof(uint8_t)}, + {ms_serving::MS_INT16, sizeof(int16_t)}, + {ms_serving::MS_UINT16, sizeof(uint16_t)}, + {ms_serving::MS_INT32, sizeof(int32_t)}, + {ms_serving::MS_UINT32, sizeof(uint32_t)}, + {ms_serving::MS_INT64, sizeof(int64_t)}, + {ms_serving::MS_UINT64, sizeof(uint64_t)}, + {ms_serving::MS_FLOAT16, 2}, + {ms_serving::MS_FLOAT32, 4}, + {ms_serving::MS_FLOAT64, 8}, +}; +MSTensorPtr ServingTensor2MSTensor(const ms_serving::Tensor &tensor) { + std::vector shape; + for (auto dim : tensor.tensor_shape().dims()) { + shape.push_back(static_cast(dim)); + } + auto iter = type2id_map.find(tensor.tensor_type()); + if (iter == type2id_map.end()) { + MS_LOG(ERROR) << "input tensor type is wrong, type is " << tensor.tensor_type(); + return nullptr; + } + TypeId type = iter->second; + auto ms_tensor = std::shared_ptr(inference::MSTensor::CreateTensor(type, shape)); + memcpy_s(ms_tensor->MutableData(), tensor.data().size(), tensor.data().data(), tensor.data().size()); + return ms_tensor; +} + +ms_serving::Tensor MSTensor2ServingTensor(MSTensorPtr ms_tensor) { + ms_serving::Tensor tensor; + ms_serving::TensorShape shape; + for (auto dim : ms_tensor->shape()) { + shape.add_dims(dim); + } + *tensor.mutable_tensor_shape() = shape; + auto iter = id2type_map.find(ms_tensor->data_type()); + if (iter == id2type_map.end()) { + MS_LOG(ERROR) << "input tensor type is wrong, type is " << tensor.tensor_type(); + return tensor; + } + tensor.set_tensor_type(iter->second); + tensor.set_data(ms_tensor->MutableData(), ms_tensor->Size()); + return tensor; +} + +void ClearEnv() { + Session::Instance().Clear(); + inference::ExitInference(); +} +void HandleSignal(int sig) { + ClearEnv(); + exit(0); +} + +#ifdef ENABLE_D +static rtContext_t g_ctx = nullptr; +#endif +} // namespace + +// Service Implement +class MSServiceImpl final : public MSService::Service { + grpc::Status Predict(grpc::ServerContext *context, const PredictRequest *request, PredictReply *reply) override { + std::lock_guard lock(mutex_); +#ifdef ENABLE_D + if (g_ctx == nullptr) { + MS_LOG(ERROR) << "rtCtx is nullptr"; + return grpc::Status::CANCELLED; + } + rtError_t rt_ret = rtCtxSetCurrent(g_ctx); + if (rt_ret != RT_ERROR_NONE) { + MS_LOG(ERROR) << "set Ascend rtCtx failed"; + } +#endif + std::vector inputs; + inference::MultiTensor outputs; + for (int i = 0; i < request->data_size(); i++) { + auto input = ServingTensor2MSTensor(request->data(i)); + if (input == nullptr) { + MS_LOG(ERROR) << "Tensor convert failed"; + return grpc::Status::CANCELLED; + } + inputs.push_back(input); + } + auto res = Session::Instance().Predict(inputs, &outputs); + if (res != SUCCESS) { + return grpc::Status::CANCELLED; + } + for (const auto &tensor : outputs) { + *reply->add_result() = MSTensor2ServingTensor(tensor); + } + MS_LOG(INFO) << "Finish call service Eval"; + return grpc::Status::OK; + } + + grpc::Status Test(grpc::ServerContext *context, const PredictRequest *request, PredictReply *reply) override { + MS_LOG(INFO) << "TestService call"; + return grpc::Status::OK; + } + std::mutex mutex_; +}; + +Status Server::BuildAndStart() { + // handle exit signal + signal(SIGINT, HandleSignal); + Status res; + auto option_args = Options::Instance().GetArgs(); + std::string server_address = "0.0.0.0:" + std::to_string(option_args->grpc_port); + std::string model_path = option_args->model_path; + std::string model_name = option_args->model_name; + std::string device_type = option_args->device_type; + auto device_id = option_args->device_id; + res = Session::Instance().CreatDeviceSession(device_type, device_id); + if (res != SUCCESS) { + MS_LOG(ERROR) << "creat session failed"; + ClearEnv(); + return res; + } + VersionController version_controller(option_args->poll_model_wait_seconds, model_path, model_name); + res = version_controller.Run(); + if (res != SUCCESS) { + MS_LOG(ERROR) << "load model failed"; + ClearEnv(); + return res; + } +#ifdef ENABLE_D + // set d context + rtContext_t ctx = nullptr; + rtError_t rt_ret = rtCtxGetCurrent(&ctx); + if (rt_ret != RT_ERROR_NONE || ctx == nullptr) { + MS_LOG(ERROR) << "the ascend device context is null"; + return FAILED; + } + g_ctx = ctx; +#endif + MSServiceImpl service; + grpc::EnableDefaultHealthCheckService(true); + grpc::reflection::InitProtoReflectionServerBuilderPlugin(); + // Set the port is not reuseable + auto option = grpc::MakeChannelArgumentOption(GRPC_ARG_ALLOW_REUSEPORT, 0); + grpc::ServerBuilder builder; + builder.SetOption(std::move(option)); + // Listen on the given address without any authentication mechanism. + builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); + // Register "service" as the instance through which we'll communicate with + // clients. In this case it corresponds to an *synchronous* service. + builder.RegisterService(&service); + // Finally assemble the server. + std::unique_ptr server(builder.BuildAndStart()); + MS_LOG(INFO) << "Server listening on " << server_address << std::endl; + + // Wait for the server to shutdown. Note that some other thread must be + // responsible for shutting down the server for this call to ever return. + server->Wait(); + return SUCCESS; +} + +} // namespace serving +} // namespace mindspore diff --git a/serving/core/server.h b/serving/core/server.h new file mode 100644 index 000000000..f1927e994 --- /dev/null +++ b/serving/core/server.h @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_SERVER_H +#define MINDSPORE_SERVER_H + +#include +#include +#include +#include +#include "util/status.h" +#include "version_control/model.h" +#include "include/inference.h" +#include "mindspore/ccsrc/debug/info.h" +namespace mindspore { +namespace serving { +class Session { + public: + static Session &Instance(); + Status CreatDeviceSession(const std::string &device, uint32_t device_id); + Status Predict(const std::vector> &inputs, inference::MultiTensor *output); + Status Warmup(const MindSporeModelPtr model); + Status Clear(); + + private: + Session() = default; + ~Session() = default; + int sesseion_id_{0}; + std::shared_ptr session_{nullptr}; + FuncGraphPtr last_graph_{nullptr}; + uint32_t graph_id_{0}; + std::mutex mutex_; + std::string device_type_; +}; + +class Server { + public: + Server() = default; + ~Server() = default; + Status BuildAndStart(); +}; +} // namespace serving +} // namespace mindspore +#endif // MINDSPORE_SERVER_H diff --git a/serving/core/util/file_system_operation.cc b/serving/core/util/file_system_operation.cc new file mode 100644 index 000000000..a5143995d --- /dev/null +++ b/serving/core/util/file_system_operation.cc @@ -0,0 +1,102 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "core/util/file_system_operation.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mindspore/ccsrc/utils/log_adapter.h" + +namespace mindspore { +namespace serving { + +char *ReadFile(const char *file, size_t *size) { + if (file == nullptr) { + MS_LOG(ERROR) << "file is nullptr"; + return nullptr; + } + MS_ASSERT(size != nullptr); + std::string realPath = file; + std::ifstream ifs(realPath); + if (!ifs.good()) { + MS_LOG(ERROR) << "file: " << realPath << " is not exist"; + return nullptr; + } + + if (!ifs.is_open()) { + MS_LOG(ERROR) << "file: " << realPath << "open failed"; + return nullptr; + } + + ifs.seekg(0, std::ios::end); + *size = ifs.tellg(); + std::unique_ptr buf(new (std::nothrow) char[*size]); + if (buf == nullptr) { + MS_LOG(ERROR) << "malloc buf failed, file: " << realPath; + ifs.close(); + return nullptr; + } + + ifs.seekg(0, std::ios::beg); + ifs.read(buf.get(), *size); + ifs.close(); + + return buf.release(); +} + +bool DirOrFileExist(const std::string &file_path) { + int ret = access(file_path.c_str(), 0); + return (ret == -1) ? false : true; +} + +std::vector GetAllSubDirs(const std::string &dir_path) { + DIR *dir; + struct dirent *ptr; + std::vector SubDirs; + + if ((dir = opendir(dir_path.c_str())) == NULL) { + MS_LOG(ERROR) << "Open " << dir_path << " error!"; + return std::vector(); + } + + while ((ptr = readdir(dir)) != NULL) { + std::string name = ptr->d_name; + if (name == "." || name == "..") { + continue; + } + if (ptr->d_type == DT_DIR) { + SubDirs.push_back(dir_path + "/" + name); + } + } + closedir(dir); + std::sort(SubDirs.begin(), SubDirs.end()); + return SubDirs; +} + +time_t GetModifyTime(const std::string &file_path) { + struct stat info; + (void)stat(file_path.c_str(), &info); + return info.st_mtime; +} +} // namespace serving +} // namespace mindspore diff --git a/serving/core/util/file_system_operation.h b/serving/core/util/file_system_operation.h new file mode 100644 index 000000000..e03883b81 --- /dev/null +++ b/serving/core/util/file_system_operation.h @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_SERVING_FILE_SYSTEM_OPERATION_H_ +#define MINDSPORE_SERVING_FILE_SYSTEM_OPERATION_H_ + +#include +#include +#include + +namespace mindspore { +namespace serving { +char *ReadFile(const char *file, size_t *size); +bool DirOrFileExist(const std::string &file_path); +std::vector GetAllSubDirs(const std::string &dir_path); +time_t GetModifyTime(const std::string &file_path); +} // namespace serving +} // namespace mindspore + +#endif // !MINDSPORE_SERVING_FILE_SYSTEM_OPERATION_H_ diff --git a/serving/core/util/option_parser.cc b/serving/core/util/option_parser.cc new file mode 100644 index 000000000..9cbd7eaee --- /dev/null +++ b/serving/core/util/option_parser.cc @@ -0,0 +1,243 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "core/util/option_parser.h" +#include +#include +#include +#include +#include +#include "mindspore/ccsrc/utils/log_adapter.h" + +namespace mindspore { +namespace serving { +bool StartWith(const std::string &str, const std::string &expected) { + return expected.empty() || + (str.size() >= expected.size() && memcmp(str.data(), expected.data(), expected.size()) == 0); +} + +bool RemovePrefix(std::string *str, const std::string &prefix) { + if (!StartWith(*str, prefix)) return false; + str->replace(str->begin(), str->begin() + prefix.size(), ""); + return true; +} + +bool Option::ParseInt32(std::string *arg) { + if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) { + char extra; + int32_t parsed_value; + if (sscanf(arg->data(), "%d%c", &parsed_value, &extra) != 1) { + std::cout << "Parse " << name_ << "Error for option " << *arg << std::endl; + return false; + } else { + *int32_default_ = parsed_value; + } + return true; + } + + return false; +} + +bool Option::ParseBool(std::string *arg) { + if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) { + if (*arg == "true") { + *bool_default_ = true; + } else if (*arg == "false") { + *bool_default_ = false; + } else { + std::cout << "Parse " << name_ << " Error for option " << *arg << std::endl; + return false; + } + return true; + } + + return false; +} + +bool Option::ParseString(std::string *arg) { + if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) { + *string_default_ = *arg; + return true; + } + return false; +} + +bool Option::ParseFloat(std::string *arg) { + if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) { + char extra; + float parsed_value; + if (sscanf(arg->data(), "%f%c", &parsed_value, &extra) != 1) { + std::cout << "Parse " << name_ << "Error for option " << *arg << std::endl; + return false; + } else { + *float_default_ = parsed_value; + } + return true; + } + + return false; +} + +Option::Option(const std::string &name, int32_t *default_point, const std::string &usage) + : name_(name), + type_(MS_TYPE_INT32), + int32_default_(default_point), + bool_default_(nullptr), + string_default_(nullptr), + float_default_(nullptr), + usage_(usage) {} + +Option::Option(const std::string &name, bool *default_point, const std::string &usage) + : name_(name), + type_(MS_TYPE_BOOL), + int32_default_(nullptr), + bool_default_(default_point), + string_default_(nullptr), + float_default_(nullptr), + usage_(usage) {} + +Option::Option(const std::string &name, std::string *default_point, const std::string &usage) + : name_(name), + type_(MS_TYPE_STRING), + int32_default_(nullptr), + bool_default_(nullptr), + string_default_(default_point), + float_default_(nullptr), + usage_(usage) {} + +Option::Option(const std::string &name, float *default_point, const std::string &usage) + : name_(name), + type_(MS_TYPE_FLOAT), + int32_default_(nullptr), + bool_default_(nullptr), + string_default_(nullptr), + float_default_(default_point), + usage_(usage) {} + +bool Option::Parse(std::string *arg) { + bool result = false; + switch (type_) { + case MS_TYPE_BOOL: + result = ParseBool(arg); + break; + case MS_TYPE_FLOAT: + result = ParseFloat(arg); + break; + case MS_TYPE_INT32: + result = ParseInt32(arg); + break; + case MS_TYPE_STRING: + result = ParseString(arg); + break; + default: + break; + } + return result; +} + +std::shared_ptr Options::inst_ = nullptr; + +Options &Options::Instance() { + static Options instance; + return instance; +} + +Options::Options() : args_(nullptr) { CreateOptions(); } + +void Options::CreateOptions() { + args_ = std::make_shared(); + std::vector