diff --git a/CMakeLists.txt b/CMakeLists.txt index cad0bb5bc638e08bd05a573fe548c7a81323435c..11fd0f09a6aa5adfa5971580ba8babe55d12d550 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -104,6 +104,7 @@ if (SERVER OR CLIENT) include(external/brpc) include(external/gflags) include(external/glog) + include(external/utf8proc) if (WITH_PYTHON) include(external/pybind11) include(external/python) diff --git a/cmake/external/boost.cmake b/cmake/external/boost.cmake index 0ab248f8c8a0bca9fa6f97f4520a5a9781c9b239..5eca6c5ba88ca50909e78a3d3b67c60c228c8207 100644 --- a/cmake/external/boost.cmake +++ b/cmake/external/boost.cmake @@ -25,7 +25,7 @@ set(BOOST_PROJECT "extern_boost") set(BOOST_VER "1.74.0") set(BOOST_TAR "boost_1_74_0" CACHE STRING "" FORCE) -set(BOOST_URL "http://paddlepaddledeps.cdn.bcebos.com/${BOOST_TAR}.tar.gz" CACHE STRING "" FORCE) +set(BOOST_URL "http://paddlepaddledeps.bj.bcebos.com/${BOOST_TAR}.tar.gz" CACHE STRING "" FORCE) MESSAGE(STATUS "BOOST_TAR: ${BOOST_TAR}, BOOST_URL: ${BOOST_URL}") diff --git a/cmake/external/utf8proc.cmake b/cmake/external/utf8proc.cmake new file mode 100644 index 0000000000000000000000000000000000000000..3478e77fa520e75d68a83160b04f91163dbe2e2a --- /dev/null +++ b/cmake/external/utf8proc.cmake @@ -0,0 +1,51 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +INCLUDE(ExternalProject) +SET(GIT_URL https://github.com) +SET(UTF8PROC_PREFIX_DIR ${THIRD_PARTY_PATH}/utf8proc) +SET(UTF8PROC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/utf8proc) +# As we add extra features for utf8proc, we use the non-official repo +SET(UTF8PROC_REPOSITORY ${GIT_URL}/JuliaStrings/utf8proc.git) +SET(UTF8PROC_TAG v2.6.1) + +IF(WIN32) + SET(UTF8PROC_LIBRARIES "${UTF8PROC_INSTALL_DIR}/lib/utf8proc_static.lib") + add_definitions(-DUTF8PROC_STATIC) +ELSE(WIN32) + SET(UTF8PROC_LIBRARIES "${UTF8PROC_INSTALL_DIR}/lib/libutf8proc.a") +ENDIF(WIN32) + +INCLUDE_DIRECTORIES(${UTF8PROC_INSTALL_DIR}/include) + +ExternalProject_Add( + extern_utf8proc + ${EXTERNAL_PROJECT_LOG_ARGS} + ${SHALLOW_CLONE} + GIT_REPOSITORY ${UTF8PROC_REPOSITORY} + GIT_TAG ${UTF8PROC_TAG} + PREFIX ${UTF8PROC_PREFIX_DIR} + UPDATE_COMMAND "" + CMAKE_ARGS -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DBUILD_SHARED=ON + -DBUILD_STATIC=ON + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} + -DCMAKE_INSTALL_PREFIX:PATH=${UTF8PROC_INSTALL_DIR} + -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} + BUILD_BYPRODUCTS ${UTF8PROC_LIBRARIES} +) + +ADD_LIBRARY(utf8proc STATIC IMPORTED GLOBAL) +SET_PROPERTY(TARGET utf8proc PROPERTY IMPORTED_LOCATION ${UTF8PROC_LIBRARIES}) +ADD_DEPENDENCIES(utf8proc extern_utf8proc) diff --git a/cmake/paddlepaddle.cmake b/cmake/paddlepaddle.cmake index ef1f0bb9d05b0b9c23c4e98dabfdb335df2a1e4e..6c0e3261e361bbf8d3208fea21eb01ba57381719 100644 --- a/cmake/paddlepaddle.cmake +++ b/cmake/paddlepaddle.cmake @@ -30,7 +30,7 @@ message( "WITH_GPU = ${WITH_GPU}") # Paddle Version should be one of: # latest: latest develop build # version number like 1.5.2 -SET(PADDLE_VERSION "2.2.0-rc0") +SET(PADDLE_VERSION "2.2.0") if (WITH_GPU) message("CUDA: ${CUDA_VERSION}, CUDNN_MAJOR_VERSION: ${CUDNN_MAJOR_VERSION}") # cuda 11.0 is not supported, 11.2 would be added. @@ -52,14 +52,19 @@ if (WITH_GPU) else() set(WITH_TRT OFF) endif() - if (WITH_GPU) SET(PADDLE_LIB_VERSION "${PADDLE_VERSION}/cxx_c/Linux/GPU/${CUDA_SUFFIX}") elseif (WITH_LITE) + message("cpu arch: ${CMAKE_SYSTEM_PROCESSOR}") if (WITH_XPU) - SET(PADDLE_LIB_VERSION "arm64_gcc7.3_openblas") + if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + SET(PADDLE_LIB_VERSION "x86-64_gcc8.2_avx_mkl") + elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") + SET(PADDLE_LIB_VERSION "arm64_gcc7.3_openblas") + endif() else() - SET(PADDLE_LIB_VERSION "${PADDLE_VERSION}-${CMAKE_SYSTEM_PROCESSOR}") + MESSAGE("paddle lite lib is unknown.") + SET(PADDLE_LIB_VERSION "paddle-lite-unknown") endif() else() if (WITH_AVX) diff --git a/core/general-server/CMakeLists.txt b/core/general-server/CMakeLists.txt index 9319d9ee5646570dfcc8b10b4bea1f4eeb290ef3..7875d42dc848a29b455a5c0681ab3ba60c741791 100644 --- a/core/general-server/CMakeLists.txt +++ b/core/general-server/CMakeLists.txt @@ -3,7 +3,7 @@ include(op/CMakeLists.txt) include(proto/CMakeLists.txt) add_executable(serving ${serving_srcs}) -add_dependencies(serving pdcodegen paddle_inference_engine pdserving paddle_inference cube-api utils) +add_dependencies(serving pdcodegen paddle_inference_engine pdserving paddle_inference cube-api utils utf8proc) if (WITH_GPU) add_dependencies(serving paddle_inference_engine) @@ -30,7 +30,7 @@ target_link_libraries(serving protobuf) target_link_libraries(serving pdserving) target_link_libraries(serving cube-api) target_link_libraries(serving utils) - +target_link_libraries(serving utf8proc) if(WITH_GPU) target_link_libraries(serving ${CUDA_LIBRARIES}) diff --git a/paddle_inference/paddle/include/paddle_engine.h b/paddle_inference/paddle/include/paddle_engine.h index 7cc8120f4eb818905c303b22a0b00d6b205bddb4..c76147b6842b9f01b3b4f65785102766d3940aef 100644 --- a/paddle_inference/paddle/include/paddle_engine.h +++ b/paddle_inference/paddle/include/paddle_engine.h @@ -266,6 +266,7 @@ class PaddleInferenceEngine : public EngineCore { if (engine_conf.has_use_xpu() && engine_conf.use_xpu()) { // 2 MB l3 cache config.EnableXpu(2 * 1024 * 1024); + config.SetXpuDeviceId(gpu_id); } if (engine_conf.has_enable_memory_optimization() && diff --git a/python/examples/pipeline/bert/config.yml b/python/examples/pipeline/bert/config.yml index a2b39264dd78ccb8f2936c7bd603d1c3d57b2574..5f1226646bb1a14fee3460bc98e25321b6aaa27a 100644 --- a/python/examples/pipeline/bert/config.yml +++ b/python/examples/pipeline/bert/config.yml @@ -1,17 +1,32 @@ +#worker_num, 最大并发数。当build_dag_each_worker=True时, 框架会创建worker_num个进程，每个进程内构建grpcSever和DAG +##当build_dag_each_worker=False时，框架会设置主线程grpc线程池的max_workers=worker_num worker_num: 20 +#build_dag_each_worker, False，框架在进程内创建一条DAG；True，框架会每个进程内创建多个独立的DAG +build_dag_each_worker: false + dag: + #op资源类型, True, 为线程模型；False，为进程模型 is_thread_op: false + #使用性能分析, True，生成Timeline性能数据，对性能有一定影响；False为不使用 tracer: interval_s: 10 +#http端口, rpc_port和http_port不允许同时为空。当rpc_port可用且http_port为空时，不自动生成http_port http_port: 18082 +#rpc端口, rpc_port和http_port不允许同时为空。当rpc_port为空且http_port不为空时，会自动将rpc_port设置为http_port+1 rpc_port: 9998 op: bert: + #并发数，is_thread_op=True时，为线程并发；否则为进程并发 concurrency: 2 - + #当op配置没有server_endpoints时，从local_service_conf读取本地服务配置 local_service_conf: + #client类型，包括brpc, grpc和local_predictor.local_predictor不启动Serving服务，进程内预测 client_type: local_predictor + # device_type, 0=cpu, 1=gpu, 2=tensorRT, 3=arm cpu, 4=kunlun xpu device_type: 1 + #计算硬件ID，当devices为""或不写时为CPU预测；当devices为"0", "0,1,2"时为GPU预测，表示使用的GPU卡 devices: '2' + #Fetch结果列表，以bert_seq128_model中fetch_var的alias_name为准, 如果没有设置则全部返回 fetch_list: + #bert模型路径 model_config: bert_seq128_model/ diff --git a/python/examples/pipeline/ocr/config.yml b/python/examples/pipeline/ocr/config.yml index 58e3ed54d5d286290ff4846364c2393af427bd9d..18b960b09adf7647d35b9358fdc5c6dc3aee872a 100644 --- a/python/examples/pipeline/ocr/config.yml +++ b/python/examples/pipeline/ocr/config.yml @@ -71,6 +71,8 @@ op: #Fetch结果列表，以client_config中fetch_var的alias_name为准 fetch_list: ["ctc_greedy_decoder_0.tmp_0", "softmax_0.tmp_0"] + # device_type, 0=cpu, 1=gpu, 2=tensorRT, 3=arm cpu, 4=kunlun xpu + device_type: 1 #计算硬件ID，当devices为""或不写时为CPU预测；当devices为"0", "0,1,2"时为GPU预测，表示使用的GPU卡 devices: "" diff --git a/python/paddle_serving_app/local_predict.py b/python/paddle_serving_app/local_predict.py index afe4ba62d69850482e82ba97d43ac747e0f69aaf..7de419530462b59f733f6ecc81e8b2fd9ce61b80 100644 --- a/python/paddle_serving_app/local_predict.py +++ b/python/paddle_serving_app/local_predict.py @@ -219,6 +219,7 @@ class LocalPredictor(object): if use_xpu: # 2MB l3 cache config.enable_xpu(8 * 1024 * 1024) + config.set_xpu_device_id(gpu_id) # set cpu low precision if not use_gpu and not use_lite: if precision_type == paddle_infer.PrecisionType.Int8: diff --git a/python/paddle_serving_server/env.py b/python/paddle_serving_server/env.py new file mode 100644 index 0000000000000000000000000000000000000000..56c0180c177ca90f6061230422adda2645161c73 --- /dev/null +++ b/python/paddle_serving_server/env.py @@ -0,0 +1,52 @@ +# coding:utf-8 +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License" +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +''' +This module is used to store environmental variables in PaddleServing. + + +SERVING_HOME --> the root directory for storing Paddleserving related data. Default to the current directory of starting PaddleServing . Users can + change the default value through the SERVING_HOME environment variable. +CONF_HOME --> Store the default configuration files. + +''' + +import os +import sys + +def _get_user_home(): + return os.path.expanduser(os.getcwd()) + + +def _get_serving_home(): + if 'SERVING_HOME' in os.environ: + home_path = os.environ['SERVING_HOME'] + if os.path.exists(home_path): + if os.path.isdir(home_path): + return home_path + else: + raise RuntimeError('The environment variable SERVING_HOME {} is not a directory.'.format(home_path)) + else: + return home_path + return os.path.join(_get_user_home()) + + +def _get_sub_home(directory): + home = os.path.join(_get_serving_home(), directory) + if not os.path.exists(home): + os.makedirs(home) + return home + +SERVING_HOME = _get_serving_home() +CONF_HOME = _get_sub_home("") diff --git a/python/paddle_serving_server/serve.py b/python/paddle_serving_server/serve.py index 8531e83fc1bb3a330d276e0c0d72616a810eea72..f0a9d699600c3a1993514c70e4667b0d6c4e5a05 100755 --- a/python/paddle_serving_server/serve.py +++ b/python/paddle_serving_server/serve.py @@ -31,6 +31,9 @@ elif sys.version_info.major == 3: from contextlib import closing import socket +from paddle_serving_server.env import CONF_HOME +import signal +from paddle_serving_server.util import * # web_service.py is still used by Pipeline. @@ -106,6 +109,7 @@ def is_gpu_mode(unformatted_gpus): def serve_args(): parser = argparse.ArgumentParser("serve") + parser.add_argument("server", type=str, default="start",nargs="?", help="stop or start PaddleServing") parser.add_argument( "--thread", type=int, @@ -366,17 +370,83 @@ class MainService(BaseHTTPRequestHandler): self.wfile.write(json.dumps(response).encode()) +def stop_serving(command : str, port : int = None): + ''' + Stop PaddleServing by port. + + Args: + command(str): stop->SIGINT, kill->SIGKILL + port(int): Default to None, kill all processes in ProcessInfo.json. + Not None, kill the specific process relating to port + + Returns: + True if stop serving successfully. + False if error occured + + Examples: + .. code-block:: python + + stop_serving("stop", 9494) + ''' + filepath = os.path.join(CONF_HOME, "ProcessInfo.json") + infoList = load_pid_file(filepath) + if infoList is False: + return False + lastInfo = infoList[-1] + for info in infoList: + storedPort = info["port"] + pid = info["pid"] + model = info["model"] + start_time = info["start_time"] + if port is not None: + if port in storedPort: + kill_stop_process_by_pid(command ,pid) + infoList.remove(info) + if len(infoList): + with open(filepath, "w") as fp: + json.dump(infoList, fp) + else: + os.remove(filepath) + return True + else: + if lastInfo == info: + raise ValueError( + "Please confirm the port [%s] you specified is correct." % + port) + else: + pass + else: + kill_stop_process_by_pid(command ,pid) + if lastInfo == info: + os.remove(filepath) + return True + if __name__ == "__main__": # args.device is not used at all. # just keep the interface. # so --device should not be recommended at the HomePage. args = serve_args() + if args.server == "stop" or args.server == "kill": + result = 0 + if "--port" in sys.argv: + result = stop_serving(args.server, args.port) + else: + result = stop_serving(args.server) + if result == 0: + os._exit(0) + else: + os._exit(-1) + for single_model_config in args.model: if os.path.isdir(single_model_config): pass elif os.path.isfile(single_model_config): raise ValueError("The input of --model should be a dir not file.") + if port_is_available(args.port): + portList = [args.port] + dump_pid_file(portList, args.model) + if args.use_encryption_model: p_flag = False p = None diff --git a/python/paddle_serving_server/util.py b/python/paddle_serving_server/util.py new file mode 100644 index 0000000000000000000000000000000000000000..64a36d5150846e3270030621ca68bf093b2cb00e --- /dev/null +++ b/python/paddle_serving_server/util.py @@ -0,0 +1,125 @@ +# coding:utf-8 +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import signal +import os +import time +import json +from paddle_serving_server.env import CONF_HOME + + +def pid_is_exist(pid: int): + ''' + Try to kill process by PID. + + Args: + pid(int): PID of process to be killed. + + Returns: + True if PID will be killed. + + Examples: + .. code-block:: python + + pid_is_exist(pid=8866) + ''' + try: + os.kill(pid, 0) + except: + return False + else: + return True + +def kill_stop_process_by_pid(command : str, pid : int): + ''' + using different signals to kill process group by PID . + + Args: + command(str): stop->SIGINT, kill->SIGKILL + pid(int): PID of process to be killed. + + Returns: + None + Examples: + .. code-block:: python + + kill_stop_process_by_pid("stop", 9494) + ''' + if not pid_is_exist(pid): + print("Process [%s] has been stopped."%pid) + return + try: + if command == "stop": + os.killpg(pid, signal.SIGINT) + elif command == "kill": + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + if command == "stop": + os.kill(pid, signal.SIGINT) + elif command == "kill": + os.kill(pid, signal.SIGKILL) + +def dump_pid_file(portList, model): + ''' + Write PID info to file. + + Args: + portList(List): PiplineServing includes http_port and rpc_port + PaddleServing include one port + model(str): 'Pipline' for PiplineServing + Specific model list for ServingModel + + Returns: + None + Examples: + .. code-block:: python + + dump_pid_file([9494, 10082], 'serve') + ''' + pid = os.getpid() + pidInfoList = [] + filepath = os.path.join(CONF_HOME, "ProcessInfo.json") + if os.path.exists(filepath): + if os.path.getsize(filepath): + with open(filepath, "r") as fp: + pidInfoList = json.load(fp) + # delete old pid data when new port number is same as old's + for info in pidInfoList: + storedPort = list(info["port"]) + interList = list(set(portList)&set(storedPort)) + if interList: + pidInfoList.remove(info) + + with open(filepath, "w") as fp: + info ={"pid": pid, "port" : portList, "model" : str(model), "start_time" : time.time()} + pidInfoList.append(info) + json.dump(pidInfoList, fp) + +def load_pid_file(filepath: str): + ''' + Read PID info from file. + ''' + if not os.path.exists(filepath): + raise ValueError( + "ProcessInfo.json file is not exists, All processes of PaddleServing has been stopped.") + return False + + if os.path.getsize(filepath): + with open(filepath, "r") as fp: + infoList = json.load(fp) + return infoList + else: + os.remove(filepath) + print("ProcessInfo.json file is empty, All processes of PaddleServing has been stopped.") + return False diff --git a/python/pipeline/local_service_handler.py b/python/pipeline/local_service_handler.py index d04b96547e9fb2f7fa35d0983b6cb046f505e698..d9df5e3091053a62c98fd108a5985a1e518a7767 100644 --- a/python/pipeline/local_service_handler.py +++ b/python/pipeline/local_service_handler.py @@ -280,6 +280,10 @@ class LocalServiceHandler(object): server.set_gpuid(gpuid) # TODO: support arm or arm + xpu later server.set_device(self._device_name) + if self._use_xpu: + server.set_xpu() + if self._use_lite: + server.set_lite() server.set_op_sequence(op_seq_maker.get_op_sequence()) server.set_num_threads(thread_num) diff --git a/python/pipeline/pipeline_server.py b/python/pipeline/pipeline_server.py index 5d3fa3540149412186b9335741964910a7ed56d2..f3da47137a40ffe81095038fd8b6acf838698c88 100644 --- a/python/pipeline/pipeline_server.py +++ b/python/pipeline/pipeline_server.py @@ -23,12 +23,15 @@ import multiprocessing import yaml import io import time +import os from .proto import pipeline_service_pb2_grpc, pipeline_service_pb2 from . import operator from . import dag from . import util from . import channel +from paddle_serving_server.env import CONF_HOME +from paddle_serving_server.util import dump_pid_file _LOGGER = logging.getLogger(__name__) @@ -78,7 +81,6 @@ def _reserve_port(port): finally: sock.close() - class PipelineServer(object): """ Pipeline Server : grpc gateway + grpc server. @@ -198,7 +200,14 @@ class PipelineServer(object): self._http_port): raise SystemExit("Failed to prepare_server: http_port({}) " "is already used".format(self._http_port)) - + # write the port info into ProcessInfo.json + portList = [] + if self._http_port is not None: + portList.append(self._rpc_port) + if self._rpc_port is not None: + portList.append(self._http_port) + if len(portList): + dump_pid_file(portList, "pipline") self._worker_num = conf["worker_num"] self._build_dag_each_worker = conf["build_dag_each_worker"] self._init_ops(conf["op"])