Merge branch 'develop' of https://github.com/bjjwwang/serving into develop

CMakeLists.txt

@@ -104,6 +104,7 @@ if (SERVER OR CLIENT)
     include(external/brpc)
     include(external/gflags)
     include(external/glog)
+    include(external/utf8proc)
     if (WITH_PYTHON)
         include(external/pybind11)
         include(external/python)

cmake/external/boost.cmake

@@ -25,7 +25,7 @@ set(BOOST_PROJECT       "extern_boost")
 
 set(BOOST_VER           "1.74.0")
 set(BOOST_TAR "boost_1_74_0" CACHE STRING "" FORCE)
-set(BOOST_URL "http://paddlepaddledeps.cdn.bcebos.com/${BOOST_TAR}.tar.gz" CACHE STRING "" FORCE)
+set(BOOST_URL "http://paddlepaddledeps.bj.bcebos.com/${BOOST_TAR}.tar.gz" CACHE STRING "" FORCE)
 
 MESSAGE(STATUS "BOOST_TAR: ${BOOST_TAR}, BOOST_URL: ${BOOST_URL}")
 

cmake/external/utf8proc.cmake

+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+INCLUDE(ExternalProject)
+SET(GIT_URL https://github.com)
+SET(UTF8PROC_PREFIX_DIR    ${THIRD_PARTY_PATH}/utf8proc)
+SET(UTF8PROC_INSTALL_DIR   ${THIRD_PARTY_PATH}/install/utf8proc)
+# As we add extra features for utf8proc, we use the non-official repo
+SET(UTF8PROC_REPOSITORY    ${GIT_URL}/JuliaStrings/utf8proc.git)
+SET(UTF8PROC_TAG           v2.6.1)
+
+IF(WIN32)
+  SET(UTF8PROC_LIBRARIES     "${UTF8PROC_INSTALL_DIR}/lib/utf8proc_static.lib")
+  add_definitions(-DUTF8PROC_STATIC)
+ELSE(WIN32)
+  SET(UTF8PROC_LIBRARIES     "${UTF8PROC_INSTALL_DIR}/lib/libutf8proc.a")
+ENDIF(WIN32)
+
+INCLUDE_DIRECTORIES(${UTF8PROC_INSTALL_DIR}/include)
+
+ExternalProject_Add(
+  extern_utf8proc
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  ${SHALLOW_CLONE}
+  GIT_REPOSITORY        ${UTF8PROC_REPOSITORY}
+  GIT_TAG               ${UTF8PROC_TAG}
+  PREFIX                ${UTF8PROC_PREFIX_DIR}
+  UPDATE_COMMAND        ""
+  CMAKE_ARGS            -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
+                        -DBUILD_SHARED=ON
+                        -DBUILD_STATIC=ON
+                        -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+                        -DCMAKE_INSTALL_PREFIX:PATH=${UTF8PROC_INSTALL_DIR}
+                        -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
+  BUILD_BYPRODUCTS     ${UTF8PROC_LIBRARIES}
+)
+
+ADD_LIBRARY(utf8proc STATIC IMPORTED GLOBAL)
+SET_PROPERTY(TARGET utf8proc PROPERTY IMPORTED_LOCATION ${UTF8PROC_LIBRARIES})
+ADD_DEPENDENCIES(utf8proc extern_utf8proc)

cmake/paddlepaddle.cmake

@@ -30,7 +30,7 @@ message( "WITH_GPU = ${WITH_GPU}")
 # Paddle Version should be one of:
 # latest: latest develop build
 # version number like 1.5.2
-SET(PADDLE_VERSION "2.2.0-rc0")
+SET(PADDLE_VERSION "2.2.0")
 if (WITH_GPU)
     message("CUDA: ${CUDA_VERSION}, CUDNN_MAJOR_VERSION: ${CUDNN_MAJOR_VERSION}")
     # cuda 11.0 is not supported, 11.2 would be added.
@@ -52,14 +52,19 @@ if (WITH_GPU)
 else()
     set(WITH_TRT OFF)
 endif()  
-
 if (WITH_GPU)
     SET(PADDLE_LIB_VERSION "${PADDLE_VERSION}/cxx_c/Linux/GPU/${CUDA_SUFFIX}")
 elseif (WITH_LITE)
+    message("cpu arch: ${CMAKE_SYSTEM_PROCESSOR}")
     if (WITH_XPU)
-        SET(PADDLE_LIB_VERSION "arm64_gcc7.3_openblas")
+        if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+            SET(PADDLE_LIB_VERSION "x86-64_gcc8.2_avx_mkl")
+        elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
+            SET(PADDLE_LIB_VERSION "arm64_gcc7.3_openblas")
+        endif()
     else()
-        SET(PADDLE_LIB_VERSION "${PADDLE_VERSION}-${CMAKE_SYSTEM_PROCESSOR}")
+        MESSAGE("paddle lite lib is unknown.")
+        SET(PADDLE_LIB_VERSION "paddle-lite-unknown")
     endif()
 else()
     if (WITH_AVX)

core/general-server/CMakeLists.txt

@@ -3,7 +3,7 @@ include(op/CMakeLists.txt)
 include(proto/CMakeLists.txt)
 
 add_executable(serving ${serving_srcs})
-add_dependencies(serving pdcodegen paddle_inference_engine pdserving paddle_inference cube-api utils)
+add_dependencies(serving pdcodegen paddle_inference_engine pdserving paddle_inference cube-api utils utf8proc)
 
 if (WITH_GPU)
     add_dependencies(serving paddle_inference_engine)
@@ -30,7 +30,7 @@ target_link_libraries(serving protobuf)
 target_link_libraries(serving pdserving)
 target_link_libraries(serving cube-api)
 target_link_libraries(serving utils)
-
+target_link_libraries(serving utf8proc)
 
 if(WITH_GPU)
     target_link_libraries(serving ${CUDA_LIBRARIES})

paddle_inference/paddle/include/paddle_engine.h

@@ -266,6 +266,7 @@ class PaddleInferenceEngine : public EngineCore {
     if (engine_conf.has_use_xpu() && engine_conf.use_xpu()) {
       // 2 MB l3 cache
       config.EnableXpu(2 * 1024 * 1024);
+      config.SetXpuDeviceId(gpu_id);
     }
 
     if (engine_conf.has_enable_memory_optimization() &&

python/examples/pipeline/bert/config.yml

+#worker_num, 最大并发数。当build_dag_each_worker=True时, 框架会创建worker_num个进程，每个进程内构建grpcSever和DAG
+##当build_dag_each_worker=False时，框架会设置主线程grpc线程池的max_workers=worker_num
 worker_num: 20
+#build_dag_each_worker, False，框架在进程内创建一条DAG；True，框架会每个进程内创建多个独立的DAG
+build_dag_each_worker: false
+
 dag:
+  #op资源类型, True, 为线程模型；False，为进程模型
   is_thread_op: false
+  #使用性能分析, True，生成Timeline性能数据，对性能有一定影响；False为不使用
   tracer:
     interval_s: 10
+#http端口, rpc_port和http_port不允许同时为空。当rpc_port可用且http_port为空时，不自动生成http_port
 http_port: 18082
+#rpc端口, rpc_port和http_port不允许同时为空。当rpc_port为空且http_port不为空时，会自动将rpc_port设置为http_port+1
 rpc_port: 9998
 op:
   bert:
+    #并发数，is_thread_op=True时，为线程并发；否则为进程并发
     concurrency: 2
-
+    #当op配置没有server_endpoints时，从local_service_conf读取本地服务配置
     local_service_conf:
+      #client类型，包括brpc, grpc和local_predictor.local_predictor不启动Serving服务，进程内预测
       client_type: local_predictor
+      # device_type, 0=cpu, 1=gpu, 2=tensorRT, 3=arm cpu, 4=kunlun xpu
       device_type: 1
+      #计算硬件ID，当devices为""或不写时为CPU预测；当devices为"0", "0,1,2"时为GPU预测，表示使用的GPU卡
       devices: '2'
+      #Fetch结果列表，以bert_seq128_model中fetch_var的alias_name为准, 如果没有设置则全部返回
       fetch_list:
+      #bert模型路径
       model_config: bert_seq128_model/

python/examples/pipeline/ocr/config.yml

@@ -71,6 +71,8 @@ op:
 
             #Fetch结果列表，以client_config中fetch_var的alias_name为准
             fetch_list: ["ctc_greedy_decoder_0.tmp_0", "softmax_0.tmp_0"] 
+            # device_type, 0=cpu, 1=gpu, 2=tensorRT, 3=arm cpu, 4=kunlun xpu
+            device_type: 1
 
             #计算硬件ID，当devices为""或不写时为CPU预测；当devices为"0", "0,1,2"时为GPU预测，表示使用的GPU卡
             devices: ""

python/paddle_serving_app/local_predict.py

@@ -219,6 +219,7 @@ class LocalPredictor(object):
         if use_xpu:
             # 2MB l3 cache
             config.enable_xpu(8 * 1024 * 1024)
+            config.set_xpu_device_id(gpu_id)
         # set cpu low precision
         if not use_gpu and not use_lite:
             if precision_type == paddle_infer.PrecisionType.Int8:

python/paddle_serving_server/env.py

+# coding:utf-8
+# Copyright (c) 2020  PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''
+This module is used to store environmental variables in PaddleServing.
+
+
+SERVING_HOME              -->  the root directory for storing Paddleserving related data. Default to the current directory of starting PaddleServing . Users can
+                               change the default value through the SERVING_HOME environment variable.
+CONF_HOME                 -->  Store the default configuration files.
+
+'''
+
+import os
+import sys
+
+def _get_user_home():
+    return os.path.expanduser(os.getcwd())
+
+
+def _get_serving_home():
+    if 'SERVING_HOME' in os.environ:
+        home_path = os.environ['SERVING_HOME']
+        if os.path.exists(home_path):
+            if os.path.isdir(home_path):
+                return home_path
+            else:
+                raise RuntimeError('The environment variable SERVING_HOME {} is not a directory.'.format(home_path))
+        else:
+            return home_path
+    return os.path.join(_get_user_home())
+
+
+def _get_sub_home(directory):
+    home = os.path.join(_get_serving_home(), directory)
+    if not os.path.exists(home):
+        os.makedirs(home)
+    return home
+
+SERVING_HOME = _get_serving_home()
+CONF_HOME = _get_sub_home("")

python/paddle_serving_server/serve.py

@@ -31,6 +31,9 @@ elif sys.version_info.major == 3:
 
 from contextlib import closing
 import socket
+from paddle_serving_server.env import CONF_HOME
+import signal
+from paddle_serving_server.util import *
 
 
 # web_service.py is still used by Pipeline.
@@ -106,6 +109,7 @@ def is_gpu_mode(unformatted_gpus):
 
 def serve_args():
     parser = argparse.ArgumentParser("serve")
+    parser.add_argument("server", type=str, default="start",nargs="?", help="stop or start PaddleServing")
     parser.add_argument(
         "--thread",
         type=int,
@@ -366,17 +370,83 @@ class MainService(BaseHTTPRequestHandler):
         self.wfile.write(json.dumps(response).encode())
 
 
+def stop_serving(command : str, port : int = None):
+    '''
+    Stop PaddleServing by port.
+
+    Args:
+        command(str): stop->SIGINT, kill->SIGKILL
+        port(int): Default to None, kill all processes in ProcessInfo.json.
+                   Not None, kill the specific process relating to port
+
+    Returns:
+         True if stop serving successfully.
+         False if error occured
+
+    Examples:
+    ..  code-block:: python
+
+        stop_serving("stop", 9494)
+    '''
+    filepath = os.path.join(CONF_HOME, "ProcessInfo.json")
+    infoList = load_pid_file(filepath)
+    if infoList is False:
+        return False
+    lastInfo = infoList[-1]
+    for info in infoList:
+        storedPort = info["port"]
+        pid = info["pid"]
+        model = info["model"]
+        start_time = info["start_time"]
+        if port is not None:
+            if port in storedPort:
+                kill_stop_process_by_pid(command ,pid)
+                infoList.remove(info)
+                if len(infoList):
+                    with open(filepath, "w") as fp:
+                        json.dump(infoList, fp)
+                else:
+                    os.remove(filepath)
+                return True
+            else:
+                if lastInfo == info:
+                     raise ValueError(
+                         "Please confirm the port [%s] you specified is correct." %
+                         port)
+                else:
+                    pass
+        else:
+            kill_stop_process_by_pid(command ,pid)
+            if lastInfo == info:
+                os.remove(filepath)
+    return True
+
 if __name__ == "__main__":
     # args.device is not used at all.
     # just keep the interface.
     # so --device should not be recommended at the HomePage.
     args = serve_args()
+    if args.server == "stop" or args.server == "kill":
+        result = 0
+        if "--port" in sys.argv:
+            result = stop_serving(args.server, args.port)
+        else:
+            result = stop_serving(args.server)
+        if result == 0:
+            os._exit(0)
+        else:
+            os._exit(-1)
+    
     for single_model_config in args.model:
         if os.path.isdir(single_model_config):
             pass
         elif os.path.isfile(single_model_config):
             raise ValueError("The input of --model should be a dir not file.")
 
+    if port_is_available(args.port):
+        portList = [args.port]
+        dump_pid_file(portList, args.model)
+
     if args.use_encryption_model:
         p_flag = False
         p = None

python/paddle_serving_server/util.py

+# coding:utf-8
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import signal
+import os
+import time
+import json
+from paddle_serving_server.env import CONF_HOME
+
+
+def pid_is_exist(pid: int):
+    '''
+    Try to kill process by PID.
+
+    Args:
+        pid(int): PID of process to be killed.
+
+    Returns:
+         True if PID will be killed.
+
+    Examples:
+    .. code-block:: python
+
+        pid_is_exist(pid=8866)
+    '''
+    try:
+        os.kill(pid, 0)
+    except:
+        return False
+    else:
+        return True
+
+def kill_stop_process_by_pid(command : str, pid : int):
+    '''
+    using different signals to kill process group by PID .
+
+    Args:
+        command(str): stop->SIGINT, kill->SIGKILL
+        pid(int): PID of process to be killed.
+
+    Returns:
+       None
+    Examples:
+    .. code-block:: python
+
+       kill_stop_process_by_pid("stop", 9494)
+    '''
+    if not pid_is_exist(pid):
+        print("Process [%s]  has been stopped."%pid)
+        return
+    try:
+         if command == "stop":
+             os.killpg(pid, signal.SIGINT)
+         elif command == "kill":
+             os.killpg(pid, signal.SIGKILL)
+    except ProcessLookupError:
+         if command == "stop":
+             os.kill(pid, signal.SIGINT)
+         elif command == "kill":
+             os.kill(pid, signal.SIGKILL)
+
+def dump_pid_file(portList, model):
+    '''
+    Write PID info to file.
+
+    Args:
+        portList(List): PiplineServing includes http_port and rpc_port
+                        PaddleServing include one port
+        model(str): 'Pipline' for PiplineServing
+                    Specific model list for ServingModel 
+
+    Returns:
+       None
+    Examples:
+    .. code-block:: python
+
+       dump_pid_file([9494, 10082], 'serve')
+    '''
+    pid = os.getpid()
+    pidInfoList = []
+    filepath = os.path.join(CONF_HOME, "ProcessInfo.json")
+    if os.path.exists(filepath):
+        if os.path.getsize(filepath):
+            with open(filepath, "r") as fp:
+                pidInfoList = json.load(fp)
+                # delete old pid data when new port number is same as old's
+                for info in pidInfoList:
+                    storedPort = list(info["port"])
+                    interList = list(set(portList)&set(storedPort))
+                    if interList:
+                        pidInfoList.remove(info)
+
+    with open(filepath, "w") as fp:
+        info ={"pid": pid, "port" : portList, "model" : str(model), "start_time" : time.time()}
+        pidInfoList.append(info)
+        json.dump(pidInfoList, fp)
+
+def load_pid_file(filepath: str):
+    '''
+    Read PID info from file.
+    '''
+    if not os.path.exists(filepath):
+        raise ValueError(
+            "ProcessInfo.json file is not exists, All processes of PaddleServing has been stopped.")
+        return False
+    
+    if os.path.getsize(filepath):
+        with open(filepath, "r") as fp:
+            infoList = json.load(fp)
+            return infoList
+    else:
+        os.remove(filepath)
+        print("ProcessInfo.json file is empty, All processes of PaddleServing has been stopped.")
+        return False

python/pipeline/local_service_handler.py

@@ -280,6 +280,10 @@ class LocalServiceHandler(object):
                 server.set_gpuid(gpuid)
             # TODO: support arm or arm + xpu later
             server.set_device(self._device_name)
+            if self._use_xpu:
+                server.set_xpu()
+            if self._use_lite:
+                server.set_lite()
 
         server.set_op_sequence(op_seq_maker.get_op_sequence())
         server.set_num_threads(thread_num)

python/pipeline/pipeline_server.py

@@ -23,12 +23,15 @@ import multiprocessing
 import yaml
 import io
 import time
+import os
 
 from .proto import pipeline_service_pb2_grpc, pipeline_service_pb2
 from . import operator
 from . import dag
 from . import util
 from . import channel
+from paddle_serving_server.env import CONF_HOME
+from paddle_serving_server.util import dump_pid_file
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -78,7 +81,6 @@ def _reserve_port(port):
     finally:
         sock.close()
 
-
 class PipelineServer(object):
     """
     Pipeline Server : grpc gateway + grpc server.
@@ -198,7 +200,14 @@ class PipelineServer(object):
                         self._http_port):
                     raise SystemExit("Failed to prepare_server: http_port({}) "
                                      "is already used".format(self._http_port))
-
+        # write the port info into ProcessInfo.json
+        portList = []
+        if self._http_port is not None:
+            portList.append(self._rpc_port)
+        if self._rpc_port is not None:
+            portList.append(self._http_port)
+        if len(portList):
+            dump_pid_file(portList, "pipline")
         self._worker_num = conf["worker_num"]
         self._build_dag_each_worker = conf["build_dag_each_worker"]
         self._init_ops(conf["op"])