Merge pull request #1082 from zhangjun/py-merge

merge paddle_serving_server_gpu and paddle_serving_server

core/configure/CMakeLists.txt

 if (SERVER OR CLIENT)
-LIST(APPEND protofiles
+  LIST(APPEND protofiles
           ${CMAKE_CURRENT_LIST_DIR}/proto/server_configure.proto
           ${CMAKE_CURRENT_LIST_DIR}/proto/sdk_configure.proto
           ${CMAKE_CURRENT_LIST_DIR}/proto/inferencer_configure.proto
   	${CMAKE_CURRENT_LIST_DIR}/proto/general_model_config.proto
-)
+  )
   
-PROTOBUF_GENERATE_CPP(configure_proto_srcs configure_proto_hdrs ${protofiles})
-list(APPEND configure_srcs ${configure_proto_srcs})
+  PROTOBUF_GENERATE_CPP(configure_proto_srcs configure_proto_hdrs ${protofiles})
+  list(APPEND configure_srcs ${configure_proto_srcs})
   
-list(APPEND configure_srcs ${CMAKE_CURRENT_LIST_DIR}/src/configure_parser.cpp)
+  list(APPEND configure_srcs ${CMAKE_CURRENT_LIST_DIR}/src/configure_parser.cpp)
   
-add_library(configure ${configure_srcs})
-add_dependencies(configure brpc)
+  add_library(configure ${configure_srcs})
+  add_dependencies(configure brpc)
   
-install(TARGETS configure 
+  install(TARGETS configure 
           ARCHIVE DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/lib
           )
   
-install(FILES ${CMAKE_CURRENT_LIST_DIR}/include/configure_parser.h
+  install(FILES ${CMAKE_CURRENT_LIST_DIR}/include/configure_parser.h
           DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/include/configure/include)
   
-FILE(GLOB inc ${CMAKE_CURRENT_BINARY_DIR}/*.pb.h)
+  FILE(GLOB inc ${CMAKE_CURRENT_BINARY_DIR}/*.pb.h)
   
-install(FILES ${inc}
+  install(FILES ${inc}
           DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/include/configure)
 endif()
 
 if (WITH_PYTHON)
 
-py_proto_compile(general_model_config_py_proto SRCS proto/general_model_config.proto)
-add_custom_target(general_model_config_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
-add_dependencies(general_model_config_py_proto general_model_config_py_proto_init)
+  py_proto_compile(general_model_config_py_proto SRCS proto/general_model_config.proto)
+  add_custom_target(general_model_config_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
+  add_dependencies(general_model_config_py_proto general_model_config_py_proto_init)
   
-py_grpc_proto_compile(multi_lang_general_model_service_py_proto SRCS proto/multi_lang_general_model_service.proto)
-add_custom_target(multi_lang_general_model_service_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
-add_dependencies(multi_lang_general_model_service_py_proto multi_lang_general_model_service_py_proto_init)
+  py_grpc_proto_compile(multi_lang_general_model_service_py_proto SRCS proto/multi_lang_general_model_service.proto)
+  add_custom_target(multi_lang_general_model_service_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
+  add_dependencies(multi_lang_general_model_service_py_proto multi_lang_general_model_service_py_proto_init)
   
-if (CLIENT)
-py_proto_compile(sdk_configure_py_proto SRCS proto/sdk_configure.proto)
-add_custom_target(sdk_configure_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
-add_dependencies(sdk_configure_py_proto sdk_configure_py_proto_init)
-add_custom_command(TARGET sdk_configure_py_proto POST_BUILD
+  if (CLIENT)
+    py_proto_compile(sdk_configure_py_proto SRCS proto/sdk_configure.proto)
+    add_custom_target(sdk_configure_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
+    add_dependencies(sdk_configure_py_proto sdk_configure_py_proto_init)
+    add_custom_command(TARGET sdk_configure_py_proto POST_BUILD
     		COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto
     		COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto
     		COMMENT "Copy generated python proto into directory paddle_serving_client/proto."
     		WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
     
-add_custom_command(TARGET general_model_config_py_proto POST_BUILD
+    add_custom_command(TARGET general_model_config_py_proto POST_BUILD
                     COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto
                     COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto
                     COMMENT "Copy generated general_model_config proto file into directory paddle_serving_client/proto."
                     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
     
-add_custom_command(TARGET multi_lang_general_model_service_py_proto POST_BUILD
+    add_custom_command(TARGET multi_lang_general_model_service_py_proto POST_BUILD
                     COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto
                     COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto
                     COMMENT "Copy generated multi_lang_general_model_service proto file into directory paddle_serving_client/proto."
                     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-endif()
+  endif()
   
-if (APP)
-add_custom_command(TARGET general_model_config_py_proto POST_BUILD
+  if (APP)
+    add_custom_command(TARGET general_model_config_py_proto POST_BUILD
                     COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_app/proto
                     COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_app/proto
                     COMMENT "Copy generated general_model_config proto file into directory paddle_serving_app/proto."
                     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-endif()
+  endif()
   
-if (SERVER)
-py_proto_compile(server_config_py_proto SRCS proto/server_configure.proto)
-add_custom_target(server_config_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
-add_dependencies(server_config_py_proto server_config_py_proto_init)
-if (NOT WITH_GPU AND NOT WITH_LITE)
-add_custom_command(TARGET server_config_py_proto POST_BUILD
+  if (SERVER)
+    py_proto_compile(server_config_py_proto SRCS proto/server_configure.proto)
+    add_custom_target(server_config_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
+    add_dependencies(server_config_py_proto server_config_py_proto_init)
+    add_custom_command(TARGET server_config_py_proto POST_BUILD
     		COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto
     		COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto
     		COMMENT "Copy generated python proto into directory paddle_serving_server/proto."
     		WORKING_DIRECTORY ${CMAKE_CURRENT_BINRARY_DIR})
     
-add_custom_command(TARGET general_model_config_py_proto POST_BUILD
+    add_custom_command(TARGET general_model_config_py_proto POST_BUILD
     		COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto
     		COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto
     		COMMENT "Copy generated general_model_config proto file into directory paddle_serving_server/proto."
     		WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
     
-add_custom_command(TARGET multi_lang_general_model_service_py_proto POST_BUILD
+    add_custom_command(TARGET multi_lang_general_model_service_py_proto POST_BUILD
                     COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto
                     COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto
                     COMMENT "Copy generated multi_lang_general_model_service proto file into directory paddle_serving_server/proto."
                     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-else()
-add_custom_command(TARGET server_config_py_proto POST_BUILD
-		COMMAND ${CMAKE_COMMAND} -E make_directory
-        ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server_gpu/proto
-		COMMAND cp -f *.py
-        ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server_gpu/proto
-		COMMENT "Copy generated python proto into directory
-        paddle_serving_server_gpu/proto."
-		WORKING_DIRECTORY ${CMAKE_CURRENT_BINRARY_DIR})
-
-add_custom_command(TARGET general_model_config_py_proto POST_BUILD
-		COMMAND ${CMAKE_COMMAND} -E make_directory
-        ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server_gpu/proto
-		COMMAND cp -f *.py
-        ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server_gpu/proto
-		COMMENT "Copy generated general_model_config proto file into directory
-        paddle_serving_server_gpu/proto."
-		WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-
-add_custom_command(TARGET multi_lang_general_model_service_py_proto POST_BUILD
-                COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server_gpu/proto
-                COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server_gpu/proto
-                COMMENT "Copy generated multi_lang_general_model_service proto file into directory paddle_serving_server_gpu/proto."
-                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-endif()
-endif()
+  endif()
 
 endif()

python/CMakeLists.txt

@@ -7,13 +7,13 @@ if (CLIENT)
 endif()
 
 if (SERVER)
-    if (NOT WITH_GPU AND NOT WITH_LITE)
+  if (WITH_GPU) 
+    set(SERVER_PACKAGE_NAME "paddle-serving-server-gpu")
+  elseif(WITH_XPU)
+    set(SERVER_PACKAGE_NAME "paddle-serving-server-xpu")
+  endif()
   file(INSTALL pipeline DESTINATION paddle_serving_server)
   file(GLOB_RECURSE SERVING_SERVER_PY_FILES paddle_serving_server/*.py)
-    else()
-        file(INSTALL pipeline DESTINATION paddle_serving_server_gpu)
-        file(GLOB_RECURSE SERVING_SERVER_PY_FILES paddle_serving_server_gpu/*.py)
-    endif()
   set(PY_FILES ${SERVING_SERVER_PY_FILES})
   SET(PACKAGE_NAME "serving_server")
   set(SETUP_LOG_FILE "setup.py.server.log")
@@ -22,25 +22,20 @@ endif()
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/util.py
     ${CMAKE_CURRENT_BINARY_DIR}/util.py)
 if (CLIENT)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.client.in
+  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.client.in
     ${CMAKE_CURRENT_BINARY_DIR}/setup.py)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/../tools/python_tag.py
+  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/../tools/python_tag.py
     ${CMAKE_CURRENT_BINARY_DIR}/python_tag.py)
 endif()
 
 if (APP)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.app.in
+  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.app.in
     ${CMAKE_CURRENT_BINARY_DIR}/setup.py)
 endif()
 
 if (SERVER)
-    if (NOT WITH_GPU AND NOT WITH_LITE)
   configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.server.in
     ${CMAKE_CURRENT_BINARY_DIR}/setup.py)
-    else()
-        configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.server_gpu.in
-            ${CMAKE_CURRENT_BINARY_DIR}/setup.py)
-    endif()
 endif()
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/gen_version.py
@@ -50,17 +45,17 @@ set (SERVING_CLIENT_CORE ${PADDLE_SERVING_BINARY_DIR}/core/general-client/*.so)
 message("python env: " ${py_env})
 
 if (APP)
-add_custom_command(
+  add_custom_command(
     OUTPUT ${PADDLE_SERVING_BINARY_DIR}/.timestamp
     COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/paddle_serving_app/ ${PADDLE_SERVING_BINARY_DIR}/python/
     COMMAND env ${py_env} ${PYTHON_EXECUTABLE} gen_version.py "app"
     COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
     DEPENDS ${SERVING_APP_CORE} general_model_config_py_proto ${PY_FILES})
-add_custom_target(paddle_python ALL DEPENDS ${PADDLE_SERVING_BINARY_DIR}/.timestamp)
+  add_custom_target(paddle_python ALL DEPENDS ${PADDLE_SERVING_BINARY_DIR}/.timestamp)
 endif()
 
 if (CLIENT)
-add_custom_command(
+  add_custom_command(
     OUTPUT ${PADDLE_SERVING_BINARY_DIR}/.timestamp
     COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/paddle_serving_client/ ${PADDLE_SERVING_BINARY_DIR}/python/
     COMMAND ${CMAKE_COMMAND} -E copy ${SERVING_CLIENT_CORE} ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/serving_client.so
@@ -68,90 +63,55 @@ add_custom_command(
     COMMAND env ${py_env} ${PYTHON_EXECUTABLE} gen_version.py "client"
     COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
     DEPENDS ${SERVING_CLIENT_CORE} sdk_configure_py_proto ${PY_FILES})
-add_custom_target(paddle_python ALL DEPENDS serving_client ${PADDLE_SERVING_BINARY_DIR}/.timestamp)
+  add_custom_target(paddle_python ALL DEPENDS serving_client ${PADDLE_SERVING_BINARY_DIR}/.timestamp)
 endif()
 
 if (SERVER)
-    if(NOT WITH_GPU AND NOT WITH_LITE)
-        add_custom_command(
-            OUTPUT ${PADDLE_SERVING_BINARY_DIR}/.timestamp
-            COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/paddle_serving_server/ ${PADDLE_SERVING_BINARY_DIR}/python/
-            COMMAND env ${py_env} ${PYTHON_EXECUTABLE} gen_version.py "server"
-            COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
-            DEPENDS ${SERVING_SERVER_CORE} server_config_py_proto ${PY_FILES})
-        add_custom_target(paddle_python ALL DEPENDS ${PADDLE_SERVING_BINARY_DIR}/.timestamp)
-    elseif(WITH_TRT)
+  # todo, generate suffix for cpu、gpu、arm
+  if(WITH_TRT)
     if(CUDA_VERSION EQUAL 10.1)
-            set(SUFFIX 101)
+      set(VERSION_SUFFIX 101)
     elseif(CUDA_VERSION EQUAL 10.2)
-            set(SUFFIX 102)
+      set(VERSION_SUFFIX 102)
     elseif(CUDA_VERSION EQUAL 11.0)
-            set(SUFFIX 11)
-
+      set(VERSION_SUFFIX 11)
     endif()
-        add_custom_command(
-            OUTPUT ${PADDLE_SERVING_BINARY_DIR}/.timestamp
-            COMMAND cp -r
-            ${CMAKE_CURRENT_SOURCE_DIR}/paddle_serving_server_gpu/ ${PADDLE_SERVING_BINARY_DIR}/python/
-            COMMAND env ${py_env} ${PYTHON_EXECUTABLE} gen_version.py
-            "server_gpu"  ${SUFFIX}
-            COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
-            DEPENDS ${SERVING_SERVER_CORE} server_config_py_proto ${PY_FILES})
-        add_custom_target(paddle_python ALL DEPENDS ${PADDLE_SERVING_BINARY_DIR}/.timestamp)
-    elseif(WITH_LITE)
-        if(WITH_XPU)
-            add_custom_command(
-                OUTPUT ${PADDLE_SERVING_BINARY_DIR}/.timestamp
-                COMMAND cp -r
-                ${CMAKE_CURRENT_SOURCE_DIR}/paddle_serving_server_gpu/ ${PADDLE_SERVING_BINARY_DIR}/python/
-                COMMAND env ${py_env} ${PYTHON_EXECUTABLE} gen_version.py
-                "server_gpu" arm-xpu 
-                COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
-                DEPENDS ${SERVING_SERVER_CORE} server_config_py_proto ${PY_FILES})
-            add_custom_target(paddle_python ALL DEPENDS ${PADDLE_SERVING_BINARY_DIR}/.timestamp)
-        else()
-            add_custom_command(
-                OUTPUT ${PADDLE_SERVING_BINARY_DIR}/.timestamp
-                COMMAND cp -r
-                ${CMAKE_CURRENT_SOURCE_DIR}/paddle_serving_server_gpu/ ${PADDLE_SERVING_BINARY_DIR}/python/
-                COMMAND env ${py_env} ${PYTHON_EXECUTABLE} gen_version.py
-                "server_gpu" arm 
-                COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
-                DEPENDS ${SERVING_SERVER_CORE} server_config_py_proto ${PY_FILES})
-            add_custom_target(paddle_python ALL DEPENDS ${PADDLE_SERVING_BINARY_DIR}/.timestamp)
   endif()
-    else()
+
+  if(WITH_LITE)
+    set(VERSION_SUFFIX 2)
+  endif()
+
   add_custom_command(
     OUTPUT ${PADDLE_SERVING_BINARY_DIR}/.timestamp
     COMMAND cp -r
-            ${CMAKE_CURRENT_SOURCE_DIR}/paddle_serving_server_gpu/ ${PADDLE_SERVING_BINARY_DIR}/python/
+    ${CMAKE_CURRENT_SOURCE_DIR}/paddle_serving_server/ ${PADDLE_SERVING_BINARY_DIR}/python/
     COMMAND env ${py_env} ${PYTHON_EXECUTABLE} gen_version.py
-            "server_gpu" ${CUDA_VERSION_MAJOR}
+    "server" ${VERSION_SUFFIX}
     COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
     DEPENDS ${SERVING_SERVER_CORE} server_config_py_proto ${PY_FILES})
   add_custom_target(paddle_python ALL DEPENDS ${PADDLE_SERVING_BINARY_DIR}/.timestamp)
-    endif()
 endif()
 
 set(SERVING_CLIENT_PYTHON_PACKAGE_DIR ${CMAKE_CURRENT_BINARY_DIR}/dist/)
 set(SERVING_SERVER_PYTHON_PACKAGE_DIR ${CMAKE_CURRENT_BINARY_DIR}/dist/)
 
 if (CLIENT)
-install(DIRECTORY ${SERVING_CLIENT_PYTHON_PACKAGE_DIR}
+  install(DIRECTORY ${SERVING_CLIENT_PYTHON_PACKAGE_DIR}
     DESTINATION opt/serving_client/share/wheels
-)
+  )
 endif()
 
 if (SERVER)
-install(DIRECTORY ${SERVING_SERVER_PYTHON_PACKAGE_DIR}
+  install(DIRECTORY ${SERVING_SERVER_PYTHON_PACKAGE_DIR}
       DESTINATION opt/serving_server/share/wheels
-)
+  )
 endif()
 
 if (CLIENT OR SERVER)
-find_program(PATCHELF_EXECUTABLE patchelf)
-if (NOT PATCHELF_EXECUTABLE)
+  find_program(PATCHELF_EXECUTABLE patchelf)
+  if (NOT PATCHELF_EXECUTABLE)
     message(FATAL_ERROR "patchelf not found, please install it.\n"
            "For Ubuntu, the command is: apt-get install -y patchelf.")
-endif()
+  endif()
 endif()

python/examples/bert/README.md

@@ -49,7 +49,7 @@ python -m paddle_serving_server.serve --model bert_seq128_model/ --port 9292  #c
 ```
 Or,start gpu inference service,Run
 ```
-python -m paddle_serving_server_gpu.serve --model bert_seq128_model/ --port 9292 --gpu_ids 0 #launch gpu inference service at GPU 0
+python -m paddle_serving_server.serve --model bert_seq128_model/ --port 9292 --gpu_ids 0 #launch gpu inference service at GPU 0
 ```
 
 ### RPC Inference

python/examples/bert/README_CN.md

@@ -48,7 +48,7 @@ python -m paddle_serving_server.serve --model bert_seq128_model/ --port 9292  #
 ```
 或者，启动gpu预测服务，执行
 ```
-python -m paddle_serving_server_gpu.serve --model bert_seq128_model/ --port 9292 --gpu_ids 0 #在gpu 0上启动gpu预测服务
+python -m paddle_serving_server.serve --model bert_seq128_model/ --port 9292 --gpu_ids 0 #在gpu 0上启动gpu预测服务
 
 ```
 

python/examples/bert/benchmark.sh

@@ -12,7 +12,7 @@ else
     mkdir utilization
 fi
 #start server
-$PYTHONROOT/bin/python3 -m paddle_serving_server_gpu.serve --model $1 --port 9292 --thread 4 --gpu_ids 0,1,2,3 --mem_optim  --ir_optim >  elog  2>&1 &
+$PYTHONROOT/bin/python3 -m paddle_serving_server.serve --model $1 --port 9292 --thread 4 --gpu_ids 0,1,2,3 --mem_optim  --ir_optim >  elog  2>&1 &
 sleep 5
 
 #warm up

python/examples/bert/benchmark_with_profile.sh

 export CUDA_VISIBLE_DEVICES=0,1,2,3
-python -m paddle_serving_server_gpu.serve --model bert_seq20_model/ --port 9295 --thread 4 --gpu_ids 0,1,2,3 2> elog > stdlog &
+python -m paddle_serving_server.serve --model bert_seq20_model/ --port 9295 --thread 4 --gpu_ids 0,1,2,3 2> elog > stdlog &
 export FLAGS_profile_client=1
 export FLAGS_profile_server=1
 sleep 5

python/examples/bert/bert_gpu_server.py

@@ -14,9 +14,9 @@
 
 import os
 import sys
-from paddle_serving_server_gpu import OpMaker
-from paddle_serving_server_gpu import OpSeqMaker
-from paddle_serving_server_gpu import Server
+from paddle_serving_server import OpMaker
+from paddle_serving_server import OpSeqMaker
+from paddle_serving_server import Server
 
 op_maker = OpMaker()
 read_op = op_maker.create('general_reader')

python/examples/bert/bert_web_service_gpu.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # pylint: disable=doc-string-missing
-from paddle_serving_server_gpu.web_service import WebService
+from paddle_serving_server.web_service import WebService
 from paddle_serving_app.reader import ChineseBertReader
 import sys
 import os

python/examples/cascade_rcnn/README.md

@@ -10,7 +10,7 @@ If you want to have more detection models, please refer to [Paddle Detection Mod
 
 ### Start the service
 ```
-python -m paddle_serving_server_gpu.serve --model serving_server --port 9292 --gpu_id 0
+python -m paddle_serving_server.serve --model serving_server --port 9292 --gpu_id 0
 ```
 
 ### Perform prediction

python/examples/cascade_rcnn/README_CN.md

@@ -10,7 +10,7 @@ sh get_data.sh
 
 ### 启动服务
 ```
-python -m paddle_serving_server_gpu.serve --model serving_server --port 9292 --gpu_id 0
+python -m paddle_serving_server.serve --model serving_server --port 9292 --gpu_id 0
 ```
 
 ### 执行预测

python/examples/criteo_ctr/README.md

@@ -20,7 +20,7 @@ the directories like `ctr_serving_model` and `ctr_client_conf` will appear.
 
 ```
 python -m paddle_serving_server.serve --model ctr_serving_model/ --port 9292 #CPU RPC Service
-python -m paddle_serving_server_gpu.serve --model ctr_serving_model/ --port 9292 --gpu_ids 0 #RPC Service on GPU 0
+python -m paddle_serving_server.serve --model ctr_serving_model/ --port 9292 --gpu_ids 0 #RPC Service on GPU 0
 ```
 
 ### RPC Infer

python/examples/criteo_ctr/README_CN.md

@@ -20,7 +20,7 @@ mv models/ctr_serving_model .
 
 ```
 python -m paddle_serving_server.serve --model ctr_serving_model/ --port 9292 #启动CPU预测服务
-python -m paddle_serving_server_gpu.serve --model ctr_serving_model/ --port 9292 --gpu_ids 0 #在GPU 0上启动预测服务
+python -m paddle_serving_server.serve --model ctr_serving_model/ --port 9292 --gpu_ids 0 #在GPU 0上启动预测服务
 ```
 
 ### 执行预测

python/examples/deeplabv3/README.md

@@ -12,7 +12,7 @@ tar -xzvf deeplabv3.tar.gz
 ### Start Service
 
 ```
-python -m paddle_serving_server_gpu.serve --model deeplabv3_server --gpu_ids 0 --port 9494
+python -m paddle_serving_server.serve --model deeplabv3_server --gpu_ids 0 --port 9494
 ```
 
 ### Client Prediction

python/examples/deeplabv3/README_CN.md

@@ -12,7 +12,7 @@ tar -xzvf deeplabv3.tar.gz
 ### 启动服务端
 
 ```
-python -m paddle_serving_server_gpu.serve --model deeplabv3_server --gpu_ids 0 --port 9494
+python -m paddle_serving_server.serve --model deeplabv3_server --gpu_ids 0 --port 9494
 ```
 
 ### 客户端预测

python/examples/detection/faster_rcnn_r50_fpn_1x_coco/README.md

@@ -10,7 +10,7 @@ wget --no-check-certificate https://paddle-serving.bj.bcebos.com/pddet_demo/2.0/
 ### Start the service
 ```
 tar xf faster_rcnn_r50_fpn_1x_coco.tar
-python -m paddle_serving_server_gpu.serve --model serving_server --port 9494 --gpu_ids 0
+python -m paddle_serving_server.serve --model serving_server --port 9494 --gpu_ids 0
 ```
 
 This model support TensorRT, if you want a faster inference, please use `--use_trt`. 

python/examples/detection/faster_rcnn_r50_fpn_1x_coco/README_CN.md

@@ -11,7 +11,7 @@ wget --no-check-certificate https://paddle-serving.bj.bcebos.com/pddet_demo/2.0/
 ### 启动服务
 ```
 tar xf faster_rcnn_r50_fpn_1x_coco.tar
-python -m paddle_serving_server_gpu.serve --model serving_server --port 9494 --gpu_ids 0
+python -m paddle_serving_server.serve --model serving_server --port 9494 --gpu_ids 0
 ```
 该模型支持TensorRT，如果想要更快的预测速度，可以开启`--use_trt`选项。
 

python/examples/detection/ppyolo_r50vd_dcn_1x_coco/README.md

@@ -10,7 +10,7 @@ wget --no-check-certificate https://paddle-serving.bj.bcebos.com/pddet_demo/2.0/
 ### Start the service
 ```
 tar xf ppyolo_r50vd_dcn_1x_coco.tar
-python -m paddle_serving_server_gpu.serve --model serving_server --port 9494 --gpu_ids 0
+python -m paddle_serving_server.serve --model serving_server --port 9494 --gpu_ids 0
 ```
 
 This model support TensorRT, if you want a faster inference, please use `--use_trt`.

python/examples/detection/ppyolo_r50vd_dcn_1x_coco/README_CN.md

@@ -11,7 +11,7 @@ wget --no-check-certificate https://paddle-serving.bj.bcebos.com/pddet_demo/2.0/
 ### 启动服务
 ```
 tar xf ppyolo_r50vd_dcn_1x_coco.tar
-python -m paddle_serving_server_gpu.serve --model serving_server --port 9494 --gpu_ids 0
+python -m paddle_serving_server.serve --model serving_server --port 9494 --gpu_ids 0
 ```
 
 该模型支持TensorRT，如果想要更快的预测速度，可以开启`--use_trt`选项。

python/examples/detection/ttfnet_darknet53_1x_coco/README.md

@@ -10,7 +10,7 @@ wget --no-check-certificate https://paddle-serving.bj.bcebos.com/pddet_demo/2.0/
 ### Start the service
 ```
 tar xf ttfnet_darknet53_1x_coco.tar
-python -m paddle_serving_server_gpu.serve --model serving_server --port 9494 --gpu_ids 0
+python -m paddle_serving_server.serve --model serving_server --port 9494 --gpu_ids 0
 ```
 This model support TensorRT, if you want a faster inference, please use `--use_trt`.
 

python/examples/detection/ttfnet_darknet53_1x_coco/README_CN.md

@@ -11,7 +11,7 @@ wget --no-check-certificate https://paddle-serving.bj.bcebos.com/pddet_demo/2.0/
 ### 启动服务
 ```
 tar xf ttfnet_darknet53_1x_coco.tar
-python -m paddle_serving_server_gpu.serve --model serving_server --port 9494 --gpu_ids 0
+python -m paddle_serving_server.serve --model serving_server --port 9494 --gpu_ids 0
 ```
 
 该模型支持TensorRT，如果想要更快的预测速度，可以开启`--use_trt`选项。

python/examples/detection/yolov3_darknet53_270e_coco/README.md

@@ -10,7 +10,7 @@ wget --no-check-certificate https://paddle-serving.bj.bcebos.com/pddet_demo/2.0/
 ### Start the service
 ```
 tar xf yolov3_darknet53_270e_coco.tar
-python -m paddle_serving_server_gpu.serve --model serving_server --port 9494 --gpu_ids 0
+python -m paddle_serving_server.serve --model serving_server --port 9494 --gpu_ids 0
 ```
 
 This model support TensorRT, if you want a faster inference, please use `--use_trt`.

python/examples/detection/yolov3_darknet53_270e_coco/README_CN.md

@@ -11,7 +11,7 @@ wget --no-check-certificate https://paddle-serving.bj.bcebos.com/pddet_demo/2.0/
 ### 启动服务
 ```
 tar xf yolov3_darknet53_270e_coco.tar
-python -m paddle_serving_server_gpu.serve --model serving_server --port 9494 --gpu_ids 0
+python -m paddle_serving_server.serve --model serving_server --port 9494 --gpu_ids 0
 ```
 
 该模型支持TensorRT，如果想要更快的预测速度，可以开启`--use_trt`选项。

python/examples/encryption/README.md

@@ -26,7 +26,7 @@ python -m paddle_serving_server.serve --model encrypt_server/ --port 9300 --use_
 ```
 GPU Service
 ```
-python -m paddle_serving_server_gpu.serve --model encrypt_server/ --port 9300 --use_encryption_model --gpu_ids 0
+python -m paddle_serving_server.serve --model encrypt_server/ --port 9300 --use_encryption_model --gpu_ids 0
 ```
 
 ## Prediction

python/examples/encryption/README_CN.md

@@ -24,7 +24,7 @@ python -m paddle_serving_server.serve --model encrypt_server/ --port 9300 --use_
 ```
 GPU预测服务
 ```
-python -m paddle_serving_server_gpu.serve --model encrypt_server/ --port 9300 --use_encryption_model --gpu_ids 0
+python -m paddle_serving_server.serve --model encrypt_server/ --port 9300 --use_encryption_model --gpu_ids 0
 ```
 
 ## 预测

python/examples/grpc_impl_example/fit_a_line/test_server_gpu.py

@@ -15,9 +15,9 @@
 
 import os
 import sys
-from paddle_serving_server_gpu import OpMaker
-from paddle_serving_server_gpu import OpSeqMaker
-from paddle_serving_server_gpu import MultiLangServer as Server
+from paddle_serving_server import OpMaker
+from paddle_serving_server import OpSeqMaker
+from paddle_serving_server import MultiLangServer as Server
 
 op_maker = OpMaker()
 read_op = op_maker.create('general_reader')

python/examples/grpc_impl_example/yolov4/README.md

@@ -12,7 +12,7 @@ tar -xzvf yolov4.tar.gz
 ## Start RPC Service
 
 ```
-python -m paddle_serving_server_gpu.serve --model yolov4_model --port 9393 --gpu_ids 0 --use_multilang
+python -m paddle_serving_server.serve --model yolov4_model --port 9393 --gpu_ids 0 --use_multilang
 ```
 
 ## Prediction

python/examples/grpc_impl_example/yolov4/README_CN.md

@@ -12,7 +12,7 @@ tar -xzvf yolov4.tar.gz
 ## 启动RPC服务
 
 ```
-python -m paddle_serving_server_gpu.serve --model yolov4_model --port 9393 --gpu_ids 0 --use_multilang
+python -m paddle_serving_server.serve --model yolov4_model --port 9393 --gpu_ids 0 --use_multilang
 ```
 
 ## 预测

python/examples/imagenet/README.md

@@ -39,7 +39,7 @@ python -m paddle_serving_server.serve --model ResNet50_vd_model --port 9696 #cpu
 ```
 
 ```
-python -m paddle_serving_server_gpu.serve --model ResNet50_vd_model --port 9696 --gpu_ids 0 #gpu inference service
+python -m paddle_serving_server.serve --model ResNet50_vd_model --port 9696 --gpu_ids 0 #gpu inference service
 ```
 
 client send inference request

python/examples/imagenet/README_CN.md

@@ -39,7 +39,7 @@ python -m paddle_serving_server.serve --model ResNet50_vd_model --port 9696 #cpu
 ```
 
 ```
-python -m paddle_serving_server_gpu.serve --model ResNet50_vd_model --port 9696 --gpu_ids 0 #gpu预测服务
+python -m paddle_serving_server.serve --model ResNet50_vd_model --port 9696 --gpu_ids 0 #gpu预测服务
 ```
 
 client端进行预测

python/examples/imagenet/benchmark.sh

@@ -2,7 +2,7 @@ rm profile_log*
 export CUDA_VISIBLE_DEVICES=0,1,2,3
 export FLAGS_profile_server=1
 export FLAGS_profile_client=1
-python -m paddle_serving_server_gpu.serve --model $1 --port 9292 --thread 4 --gpu_ids 0,1,2,3 --mem_optim --ir_optim  2> elog > stdlog &
+python -m paddle_serving_server.serve --model $1 --port 9292 --thread 4 --gpu_ids 0,1,2,3 --mem_optim --ir_optim  2> elog > stdlog &
 
 sleep 5
 gpu_id=0

python/examples/imagenet/resnet50_web_service.py

@@ -25,7 +25,7 @@ device = sys.argv[2]
 if device == "cpu":
     from paddle_serving_server.web_service import WebService
 else:
-    from paddle_serving_server_gpu.web_service import WebService
+    from paddle_serving_server.web_service import WebService
 
 
 class ImageService(WebService):

python/examples/mobilenet/README.md

@@ -12,7 +12,7 @@ tar -xzvf mobilenet_v2_imagenet.tar.gz
 ### Start Service
 
 ```
-python -m paddle_serving_server_gpu.serve --model mobilenet_v2_imagenet_model --gpu_ids 0 --port 9393
+python -m paddle_serving_server.serve --model mobilenet_v2_imagenet_model --gpu_ids 0 --port 9393
 ```
 
 ### Client Prediction

python/examples/mobilenet/README_CN.md

@@ -12,7 +12,7 @@ tar -xzvf mobilenet_v2_imagenet.tar.gz
 ### 启动服务端
 
 ```
-python -m paddle_serving_server_gpu.serve --model mobilenet_v2_imagenet_model --gpu_ids 0 --port 9393
+python -m paddle_serving_server.serve --model mobilenet_v2_imagenet_model --gpu_ids 0 --port 9393
 ```
 
 ### 客户端预测

python/examples/ocr/README.md

@@ -26,7 +26,7 @@ tar xf test_imgs.tar
 python -m paddle_serving_server.serve --model ocr_det_model --port 9293
 python ocr_web_server.py cpu
 #for gpu user
-python -m paddle_serving_server_gpu.serve --model ocr_det_model --port 9293 --gpu_id 0
+python -m paddle_serving_server.serve --model ocr_det_model --port 9293 --gpu_id 0
 python ocr_web_server.py gpu
 ```
 

python/examples/ocr/README_CN.md

@@ -25,7 +25,7 @@ tar xf test_imgs.tar
 python -m paddle_serving_server.serve --model ocr_det_model --port 9293
 python ocr_web_server.py cpu
 #for gpu user
-python -m paddle_serving_server_gpu.serve --model ocr_det_model --port 9293 --gpu_id 0
+python -m paddle_serving_server.serve --model ocr_det_model --port 9293 --gpu_id 0
 python ocr_web_server.py gpu
 ```
 

python/examples/ocr/det_debugger_server.py

@@ -22,7 +22,7 @@ from paddle_serving_app.reader import Sequential, ResizeByFactor
 from paddle_serving_app.reader import Div, Normalize, Transpose
 from paddle_serving_app.reader import DBPostProcess, FilterBoxes
 if sys.argv[1] == 'gpu':
-    from paddle_serving_server_gpu.web_service import WebService
+    from paddle_serving_server.web_service import WebService
 elif sys.argv[1] == 'cpu':
     from paddle_serving_server.web_service import WebService
 import time

python/examples/ocr/det_web_server.py

@@ -22,7 +22,7 @@ from paddle_serving_app.reader import Sequential, ResizeByFactor
 from paddle_serving_app.reader import Div, Normalize, Transpose
 from paddle_serving_app.reader import DBPostProcess, FilterBoxes
 if sys.argv[1] == 'gpu':
-    from paddle_serving_server_gpu.web_service import WebService
+    from paddle_serving_server.web_service import WebService
 elif sys.argv[1] == 'cpu':
     from paddle_serving_server.web_service import WebService
 import time

python/examples/ocr/ocr_debugger_server.py

@@ -23,7 +23,7 @@ from paddle_serving_app.reader import Sequential, URL2Image, ResizeByFactor
 from paddle_serving_app.reader import Div, Normalize, Transpose
 from paddle_serving_app.reader import DBPostProcess, FilterBoxes, GetRotateCropImage, SortedBoxes
 if sys.argv[1] == 'gpu':
-    from paddle_serving_server_gpu.web_service import WebService
+    from paddle_serving_server.web_service import WebService
 elif sys.argv[1] == 'cpu':
     from paddle_serving_server.web_service import WebService
 from paddle_serving_app.local_predict import LocalPredictor

python/examples/ocr/ocr_web_server.py

@@ -23,7 +23,7 @@ from paddle_serving_app.reader import Sequential, URL2Image, ResizeByFactor
 from paddle_serving_app.reader import Div, Normalize, Transpose
 from paddle_serving_app.reader import DBPostProcess, FilterBoxes, GetRotateCropImage, SortedBoxes
 if sys.argv[1] == 'gpu':
-    from paddle_serving_server_gpu.web_service import WebService
+    from paddle_serving_server.web_service import WebService
 elif sys.argv[1] == 'cpu':
     from paddle_serving_server.web_service import WebService
 import time

python/examples/ocr/rec_debugger_server.py

@@ -23,7 +23,7 @@ from paddle_serving_app.reader import Sequential, URL2Image, ResizeByFactor
 from paddle_serving_app.reader import Div, Normalize, Transpose
 from paddle_serving_app.reader import DBPostProcess, FilterBoxes, GetRotateCropImage, SortedBoxes
 if sys.argv[1] == 'gpu':
-    from paddle_serving_server_gpu.web_service import WebService
+    from paddle_serving_server.web_service import WebService
 elif sys.argv[1] == 'cpu':
     from paddle_serving_server.web_service import WebService
 import time

python/examples/ocr/rec_web_server.py

@@ -23,7 +23,7 @@ from paddle_serving_app.reader import Sequential, URL2Image, ResizeByFactor
 from paddle_serving_app.reader import Div, Normalize, Transpose
 from paddle_serving_app.reader import DBPostProcess, FilterBoxes, GetRotateCropImage, SortedBoxes
 if sys.argv[1] == 'gpu':
-    from paddle_serving_server_gpu.web_service import WebService
+    from paddle_serving_server.web_service import WebService
 elif sys.argv[1] == 'cpu':
     from paddle_serving_server.web_service import WebService
 import time

python/examples/pipeline/imagenet/pipeline_rpc_client.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 try:
-    from paddle_serving_server_gpu.pipeline import PipelineClient
+    from paddle_serving_server.pipeline import PipelineClient
 except ImportError:
     from paddle_serving_server.pipeline import PipelineClient
 import numpy as np

python/examples/pipeline/imagenet/resnet50_web_service.py

@@ -14,7 +14,7 @@
 import sys
 from paddle_serving_app.reader import Sequential, URL2Image, Resize, CenterCrop, RGB2BGR, Transpose, Div, Normalize, Base64ToImage
 try:
-    from paddle_serving_server_gpu.web_service import WebService, Op
+    from paddle_serving_server.web_service import WebService, Op
 except ImportError:
     from paddle_serving_server.web_service import WebService, Op
 import logging

python/examples/pipeline/imdb_model_ensemble/test_pipeline_server.py

@@ -22,7 +22,7 @@ import logging
 try:
     from paddle_serving_server.web_service import WebService
 except ImportError:
-    from paddle_serving_server_gpu.web_service import WebService
+    from paddle_serving_server.web_service import WebService
 
 _LOGGER = logging.getLogger()
 user_handler = logging.StreamHandler()

python/examples/pipeline/ocr/pipeline_rpc_client.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 try:
-    from paddle_serving_server_gpu.pipeline import PipelineClient
+    from paddle_serving_server.pipeline import PipelineClient
 except ImportError:
     from paddle_serving_server.pipeline import PipelineClient
 import numpy as np

python/examples/pipeline/ocr/web_service.py

@@ -14,7 +14,7 @@
 try:
     from paddle_serving_server.web_service import WebService, Op
 except ImportError:
-    from paddle_serving_server_gpu.web_service import WebService, Op
+    from paddle_serving_server.web_service import WebService, Op
 import logging
 import numpy as np
 import cv2

python/examples/pipeline/simple_web_service/web_service.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 try:
-    from paddle_serving_server_gpu.web_service import WebService, Op
+    from paddle_serving_server.web_service import WebService, Op
 except ImportError:
     from paddle_serving_server.web_service import WebService, Op
 import logging

python/examples/resnet_v2_50/README.md

@@ -12,7 +12,7 @@ tar -xzvf resnet_v2_50_imagenet.tar.gz
 ### Start Service
 
 ```
-python -m paddle_serving_server_gpu.serve --model resnet_v2_50_imagenet_model --gpu_ids 0 --port 9393
+python -m paddle_serving_server.serve --model resnet_v2_50_imagenet_model --gpu_ids 0 --port 9393
 ```
 
 ### Client Prediction

python/examples/resnet_v2_50/README_CN.md

@@ -12,7 +12,7 @@ tar -xzvf resnet_v2_50_imagenet.tar.gz
 ### 启动服务端
 
 ```
-python -m paddle_serving_server_gpu.serve --model resnet_v2_50_imagenet_model --gpu_ids 0 --port 9393
+python -m paddle_serving_server.serve --model resnet_v2_50_imagenet_model --gpu_ids 0 --port 9393
 ```
 
 ### 客户端预测

python/examples/unet_for_image_seg/README.md

@@ -12,7 +12,7 @@ tar -xzvf unet.tar.gz
 ### Start Service
 
 ```
-python -m paddle_serving_server_gpu.serve --model unet_model --gpu_ids 0 --port 9494
+python -m paddle_serving_server.serve --model unet_model --gpu_ids 0 --port 9494
 ```
 
 ### Client Prediction

python/examples/unet_for_image_seg/README_CN.md

@@ -12,7 +12,7 @@ tar -xzvf unet.tar.gz
 ### 启动服务端
 
 ```
-python -m paddle_serving_server_gpu.serve --model unet_model --gpu_ids 0 --port 9494
+python -m paddle_serving_server.serve --model unet_model --gpu_ids 0 --port 9494
 ```
 
 ### 客户端预测

python/examples/xpu/fit_a_line_xpu/README.md

@@ -15,7 +15,7 @@ sh get_data.sh
 ### Start server
 
 ```shell
-python -m paddle_serving_server_gpu.serve --model uci_housing_model --thread 10 --port 9393 --use_lite --use_xpu --ir_optim
+python -m paddle_serving_server.serve --model uci_housing_model --thread 10 --port 9393 --use_lite --use_xpu --ir_optim
 ```
 
 ### Client prediction

python/examples/xpu/fit_a_line_xpu/test_server.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 # pylint: disable=doc-string-missing
 
-from paddle_serving_server_gpu.web_service import WebService
+from paddle_serving_server.web_service import WebService
 import numpy as np
 
 

python/examples/xpu/resnet_v2_50_xpu/README.md

@@ -12,7 +12,7 @@ tar -xzvf resnet_v2_50_imagenet.tar.gz
 ### Start Service
 
 ```
-python -m paddle_serving_server_gpu.serve --model resnet_v2_50_imagenet_model --port 9393 --use_lite --use_xpu --ir_optim
+python -m paddle_serving_server.serve --model resnet_v2_50_imagenet_model --port 9393 --use_lite --use_xpu --ir_optim
 ```
 
 ### Client Prediction

python/examples/xpu/resnet_v2_50_xpu/README_CN.md

@@ -12,7 +12,7 @@ tar -xzvf resnet_v2_50_imagenet.tar.gz
 ### 启动服务端
 
 ```
-python -m paddle_serving_server_gpu.serve --model resnet_v2_50_imagenet_model --port 9393 --use_lite --use_xpu --ir_optim
+python -m paddle_serving_server.serve --model resnet_v2_50_imagenet_model --port 9393 --use_lite --use_xpu --ir_optim
 ```
 
 ### 客户端预测

python/examples/yolov4/README.md

@@ -12,7 +12,7 @@ tar -xzvf yolov4.tar.gz
 ## Start RPC Service
 
 ```
-python -m paddle_serving_server_gpu.serve --model yolov4_model --port 9393 --gpu_ids 0
+python -m paddle_serving_server.serve --model yolov4_model --port 9393 --gpu_ids 0
 ```
 
 ## Prediction

python/examples/yolov4/README_CN.md

@@ -12,7 +12,7 @@ tar -xzvf yolov4.tar.gz
 ## 启动RPC服务
 
 ```
-python -m paddle_serving_server_gpu.serve --model yolov4_model --port 9393 --gpu_ids 0
+python -m paddle_serving_server.serve --model yolov4_model --port 9393 --gpu_ids 0
 ```
 
 ## 预测

python/gen_version.py

@@ -34,10 +34,16 @@ def update_info(file_name, feature, info):
         f.write(new_str)
 
 
-if len(sys.argv) > 2:
-    update_info("paddle_serving_server_gpu/version.py", "cuda_version",
+if len(sys.argv) > 2 and len(sys.argv[2]) > 0:
+    update_info("paddle_serving_server/version.py", "version_suffix",
                 sys.argv[2])
 
+package_name = '${SERVER_PACKAGE_NAME}'
+if package_name.endswith('gpu'):
+    update_info("paddle_serving_server/version.py", "device_type", "1")
+elif package_name.endswith('xpu'):
+    update_info("paddle_serving_server/version.py", "device_type", "2")
+
 path = "paddle_serving_" + sys.argv[1]
 commit_id = subprocess.check_output(['git', 'rev-parse', 'HEAD'])
 update_info(path + "/version.py", "commit_id", commit_id)

python/paddle_serving_server/dag.py

+
+from .proto import server_configure_pb2 as server_sdk
+import google.protobuf.text_format
+import collections
+
+class OpMaker(object):
+    def __init__(self):
+        self.op_dict = {
+            "general_infer": "GeneralInferOp",
+            "general_reader": "GeneralReaderOp",
+            "general_response": "GeneralResponseOp",
+            "general_text_reader": "GeneralTextReaderOp",
+            "general_text_response": "GeneralTextResponseOp",
+            "general_single_kv": "GeneralSingleKVOp",
+            "general_dist_kv_infer": "GeneralDistKVInferOp",
+            "general_dist_kv": "GeneralDistKVOp"
+        }
+        self.node_name_suffix_ = collections.defaultdict(int)
+
+    def create(self, node_type, engine_name=None, inputs=[], outputs=[]):
+        if node_type not in self.op_dict:
+            raise Exception("Op type {} is not supported right now".format(
+                node_type))
+        node = server_sdk.DAGNode()
+        # node.name will be used as the infer engine name
+        if engine_name:
+            node.name = engine_name
+        else:
+            node.name = '{}_{}'.format(node_type,
+                                       self.node_name_suffix_[node_type])
+            self.node_name_suffix_[node_type] += 1
+
+        node.type = self.op_dict[node_type]
+        if inputs:
+            for dep_node_str in inputs:
+                dep_node = server_sdk.DAGNode()
+                google.protobuf.text_format.Parse(dep_node_str, dep_node)
+                dep = server_sdk.DAGNodeDependency()
+                dep.name = dep_node.name
+                dep.mode = "RO"
+                node.dependencies.extend([dep])
+        # Because the return value will be used as the key value of the
+        # dict, and the proto object is variable which cannot be hashed,
+        # so it is processed into a string. This has little effect on
+        # overall efficiency.
+        return google.protobuf.text_format.MessageToString(node)
+
+
+class OpSeqMaker(object):
+    def __init__(self):
+        self.workflow = server_sdk.Workflow()
+        self.workflow.name = "workflow1"
+        self.workflow.workflow_type = "Sequence"
+
+    def add_op(self, node_str):
+        node = server_sdk.DAGNode()
+        google.protobuf.text_format.Parse(node_str, node)
+        if len(node.dependencies) > 1:
+            raise Exception(
+                'Set more than one predecessor for op in OpSeqMaker is not allowed.'
+            )
+        if len(self.workflow.nodes) >= 1:
+            if len(node.dependencies) == 0:
+                dep = server_sdk.DAGNodeDependency()
+                dep.name = self.workflow.nodes[-1].name
+                dep.mode = "RO"
+                node.dependencies.extend([dep])
+            elif len(node.dependencies) == 1:
+                if node.dependencies[0].name != self.workflow.nodes[-1].name:
+                    raise Exception(
+                        'You must add op in order in OpSeqMaker. The previous op is {}, but the current op is followed by {}.'
+                        .format(node.dependencies[0].name, self.workflow.nodes[
+                            -1].name))
+        self.workflow.nodes.extend([node])
+
+    def get_op_sequence(self):
+        workflow_conf = server_sdk.WorkflowConf()
+        workflow_conf.workflows.extend([self.workflow])
+        return workflow_conf
+
+
+class OpGraphMaker(object):
+    def __init__(self):
+        self.workflow = server_sdk.Workflow()
+        self.workflow.name = "workflow1"
+        # Currently, SDK only supports "Sequence"
+        self.workflow.workflow_type = "Sequence"
+
+    def add_op(self, node_str):
+        node = server_sdk.DAGNode()
+        google.protobuf.text_format.Parse(node_str, node)
+        self.workflow.nodes.extend([node])
+
+    def get_op_graph(self):
+        workflow_conf = server_sdk.WorkflowConf()
+        workflow_conf.workflows.extend([self.workflow])
+        return workflow_conf

python/paddle_serving_server/monitor.py

@@ -28,7 +28,6 @@ import logging
 
 _LOGGER = logging.getLogger(__name__)
 
-
 class Monitor(object):
     '''
     Monitor base class. It is used to monitor the remote model, pull and update the local model.

python/paddle_serving_server/rpc_service.py

+import sys
+import os
+
+import google.protobuf.text_format
+
+from .proto import general_model_config_pb2 as m_config
+from .proto import multi_lang_general_model_service_pb2
+sys.path.append(
+    os.path.join(os.path.abspath(os.path.dirname(__file__)), 'proto'))
+from .proto import multi_lang_general_model_service_pb2_grpc
+
+class MultiLangServerServiceServicer(multi_lang_general_model_service_pb2_grpc.
+                                     MultiLangGeneralModelServiceServicer):
+    def __init__(self, model_config_path, is_multi_model, endpoints):
+        self.is_multi_model_ = is_multi_model
+        self.model_config_path_ = model_config_path
+        self.endpoints_ = endpoints
+        with open(self.model_config_path_) as f:
+            self.model_config_str_ = str(f.read())
+        self._parse_model_config(self.model_config_str_)
+        self._init_bclient(self.model_config_path_, self.endpoints_)
+
+    def _init_bclient(self, model_config_path, endpoints, timeout_ms=None):
+        from paddle_serving_client import Client
+        self.bclient_ = Client()
+        if timeout_ms is not None:
+            self.bclient_.set_rpc_timeout_ms(timeout_ms)
+        self.bclient_.load_client_config(model_config_path)
+        self.bclient_.connect(endpoints)
+
+    def _parse_model_config(self, model_config_str):
+        model_conf = m_config.GeneralModelConfig()
+        model_conf = google.protobuf.text_format.Merge(model_config_str,
+                                                       model_conf)
+        self.feed_names_ = [var.alias_name for var in model_conf.feed_var]
+        self.feed_types_ = {}
+        self.feed_shapes_ = {}
+        self.fetch_names_ = [var.alias_name for var in model_conf.fetch_var]
+        self.fetch_types_ = {}
+        self.lod_tensor_set_ = set()
+        for i, var in enumerate(model_conf.feed_var):
+            self.feed_types_[var.alias_name] = var.feed_type
+            self.feed_shapes_[var.alias_name] = var.shape
+            if var.is_lod_tensor:
+                self.lod_tensor_set_.add(var.alias_name)
+        for i, var in enumerate(model_conf.fetch_var):
+            self.fetch_types_[var.alias_name] = var.fetch_type
+            if var.is_lod_tensor:
+                self.lod_tensor_set_.add(var.alias_name)
+
+    def _flatten_list(self, nested_list):
+        for item in nested_list:
+            if isinstance(item, (list, tuple)):
+                for sub_item in self._flatten_list(item):
+                    yield sub_item
+            else:
+                yield item
+
+    def _unpack_inference_request(self, request):
+        feed_names = list(request.feed_var_names)
+        fetch_names = list(request.fetch_var_names)
+        is_python = request.is_python
+        log_id = request.log_id
+        feed_batch = []
+        for feed_inst in request.insts:
+            feed_dict = {}
+            for idx, name in enumerate(feed_names):
+                var = feed_inst.tensor_array[idx]
+                v_type = self.feed_types_[name]
+                data = None
+                if is_python:
+                    if v_type == 0:
+                        data = np.frombuffer(var.data, dtype="int64")
+                    elif v_type == 1:
+                        data = np.frombuffer(var.data, dtype="float32")
+                    elif v_type == 2:
+                        data = np.frombuffer(var.data, dtype="int32")
+                    else:
+                        raise Exception("error type.")
+                else:
+                    if v_type == 0:  # int64
+                        data = np.array(list(var.int64_data), dtype="int64")
+                    elif v_type == 1:  # float32
+                        data = np.array(list(var.float_data), dtype="float32")
+                    elif v_type == 2:
+                        data = np.array(list(var.int_data), dtype="int32")
+                    else:
+                        raise Exception("error type.")
+                data.shape = list(feed_inst.tensor_array[idx].shape)
+                feed_dict[name] = data
+                if len(var.lod) > 0:
+                    feed_dict["{}.lod".format(name)] = var.lod
+            feed_batch.append(feed_dict)
+        return feed_batch, fetch_names, is_python, log_id
+
+    def _pack_inference_response(self, ret, fetch_names, is_python):
+        resp = multi_lang_general_model_service_pb2.InferenceResponse()
+        if ret is None:
+            resp.err_code = 1
+            return resp
+        results, tag = ret
+        resp.tag = tag
+        resp.err_code = 0
+
+        if not self.is_multi_model_:
+            results = {'general_infer_0': results}
+        for model_name, model_result in results.items():
+            model_output = multi_lang_general_model_service_pb2.ModelOutput()
+            inst = multi_lang_general_model_service_pb2.FetchInst()
+            for idx, name in enumerate(fetch_names):
+                tensor = multi_lang_general_model_service_pb2.Tensor()
+                v_type = self.fetch_types_[name]
+                if is_python:
+                    tensor.data = model_result[name].tobytes()
+                else:
+                    if v_type == 0:  # int64
+                        tensor.int64_data.extend(model_result[name].reshape(-1)
+                                                 .tolist())
+                    elif v_type == 1:  # float32
+                        tensor.float_data.extend(model_result[name].reshape(-1)
+                                                 .tolist())
+                    elif v_type == 2:  # int32
+                        tensor.int_data.extend(model_result[name].reshape(-1)
+                                               .tolist())
+                    else:
+                        raise Exception("error type.")
+                tensor.shape.extend(list(model_result[name].shape))
+                if "{}.lod".format(name) in model_result:
+                    tensor.lod.extend(model_result["{}.lod".format(name)]
+                                      .tolist())
+                inst.tensor_array.append(tensor)
+            model_output.insts.append(inst)
+            model_output.engine_name = model_name
+            resp.outputs.append(model_output)
+        return resp
+
+    def SetTimeout(self, request, context):
+        # This porcess and Inference process cannot be operate at the same time.
+        # For performance reasons, do not add thread lock temporarily.
+        timeout_ms = request.timeout_ms
+        self._init_bclient(self.model_config_path_, self.endpoints_, timeout_ms)
+        resp = multi_lang_general_model_service_pb2.SimpleResponse()
+        resp.err_code = 0
+        return resp
+
+    def Inference(self, request, context):
+        feed_batch, fetch_names, is_python, log_id \
+                = self._unpack_inference_request(request)
+        ret = self.bclient_.predict(
+            feed=feed_batch,
+            fetch=fetch_names,
+            batch=True,
+            need_variant_tag=True,
+            log_id=log_id)
+        return self._pack_inference_response(ret, fetch_names, is_python)
+
+    def GetClientConfig(self, request, context):
+        resp = multi_lang_general_model_service_pb2.GetClientConfigResponse()
+        resp.client_config_str = self.model_config_str_
+        return resp
\ No newline at end of file

python/paddle_serving_server/serve.py

@@ -18,12 +18,11 @@ Usage:
         python -m paddle_serving_server.serve --model ./serving_server_model --port 9292
 """
 import argparse
-import sys
+import os
 import json
 import base64
 import time
-from multiprocessing import Process
-from .web_service import WebService, port_is_available
+from multiprocessing import Pool, Process
 from flask import Flask, request
 import sys
 if sys.version_info.major == 2:
@@ -32,23 +31,26 @@ elif sys.version_info.major == 3:
     from http.server import BaseHTTPRequestHandler, HTTPServer
 
 
-def parse_args():  # pylint: disable=doc-string-missing
+def serve_args():
     parser = argparse.ArgumentParser("serve")
     parser.add_argument(
-        "--thread", type=int, default=10, help="Concurrency of server")
+        "--thread", type=int, default=2, help="Concurrency of server")
     parser.add_argument(
-        "--model", type=str, default="", help="Model for serving")
+        "--port", type=int, default=9292, help="Port of the starting gpu")
     parser.add_argument(
-        "--port", type=int, default=9292, help="Port the server")
+        "--device", type=str, default="gpu", help="Type of device")
+    parser.add_argument("--gpu_ids", type=str, default="", help="gpu ids")
     parser.add_argument(
-        "--name", type=str, default="None", help="Web service name")
+        "--model", type=str, default="", help="Model for serving")
     parser.add_argument(
         "--workdir",
         type=str,
         default="workdir",
         help="Working dir of current service")
     parser.add_argument(
-        "--device", type=str, default="cpu", help="Type of device")
+        "--name", type=str, default="None", help="Default service name")
+    parser.add_argument(
+        "--use_mkl", default=False, action="store_true", help="Use MKL")
     parser.add_argument(
         "--mem_optim_off",
         default=False,
@@ -56,8 +58,6 @@ def parse_args():  # pylint: disable=doc-string-missing
         help="Memory optimize")
     parser.add_argument(
         "--ir_optim", default=False, action="store_true", help="Graph optimize")
-    parser.add_argument(
-        "--use_mkl", default=False, action="store_true", help="Use MKL")
     parser.add_argument(
         "--max_body_size",
         type=int,
@@ -73,6 +73,12 @@ def parse_args():  # pylint: disable=doc-string-missing
         default=False,
         action="store_true",
         help="Use Multi-language-service")
+    parser.add_argument(
+        "--use_trt", default=False, action="store_true", help="Use TensorRT")
+    parser.add_argument(
+        "--use_lite", default=False, action="store_true", help="Use PaddleLite")
+    parser.add_argument(
+        "--use_xpu", default=False, action="store_true", help="Use XPU")
     parser.add_argument(
         "--product_name",
         type=str,
@@ -138,6 +144,116 @@ def start_standard_model(serving_port):  # pylint: disable=doc-string-missing
     server.run_server()
 
 
+def start_gpu_card_model(index, gpuid, port, args):  # pylint: disable=doc-string-missing
+    workdir = args.workdir
+    gpuid = int(gpuid)
+    device = "gpu"
+    if gpuid == -1:
+        device = "cpu"
+    elif gpuid >= 0:
+        port = port + index
+    thread_num = args.thread
+    model = args.model
+    mem_optim = args.mem_optim_off is False
+    ir_optim = args.ir_optim
+    use_mkl = args.use_mkl
+    max_body_size = args.max_body_size
+    use_multilang = args.use_multilang
+    if gpuid >= 0:
+        workdir = "{}_{}".format(args.workdir, gpuid)
+
+    if model == "":
+        print("You must specify your serving model")
+        exit(-1)
+
+    import paddle_serving_server as serving
+    op_maker = serving.OpMaker()
+    read_op = op_maker.create('general_reader')
+    general_infer_op = op_maker.create('general_infer')
+    general_response_op = op_maker.create('general_response')
+
+    op_seq_maker = serving.OpSeqMaker()
+    op_seq_maker.add_op(read_op)
+    op_seq_maker.add_op(general_infer_op)
+    op_seq_maker.add_op(general_response_op)
+
+    if use_multilang:
+        server = serving.MultiLangServer()
+    else:
+        server = serving.Server()
+    server.set_op_sequence(op_seq_maker.get_op_sequence())
+    server.set_num_threads(thread_num)
+    server.use_mkl(use_mkl)
+    server.set_memory_optimize(mem_optim)
+    server.set_ir_optimize(ir_optim)
+    server.set_max_body_size(max_body_size)
+    if args.use_trt:
+        server.set_trt()
+
+    if args.use_lite:
+        server.set_lite()
+
+    server.set_device(device)
+    if args.use_xpu:
+        server.set_xpu()
+
+    if args.product_name != None:
+        server.set_product_name(args.product_name)
+    if args.container_id != None:
+        server.set_container_id(args.container_id)
+
+    server.load_model_config(model)
+    server.prepare_server(
+        workdir=workdir,
+        port=port,
+        device=device,
+        use_encryption_model=args.use_encryption_model)
+    if gpuid >= 0:
+        server.set_gpuid(gpuid)
+    server.run_server()
+
+
+def start_multi_card(args, serving_port=None):  # pylint: disable=doc-string-missing
+    gpus = ""
+    if serving_port == None:
+        serving_port = args.port
+    if args.gpu_ids == "":
+        gpus = []
+    else:
+        gpus = args.gpu_ids.split(",")
+        if "CUDA_VISIBLE_DEVICES" in os.environ:
+            env_gpus = os.environ["CUDA_VISIBLE_DEVICES"].split(",")
+            for ids in gpus:
+                if int(ids) >= len(env_gpus):
+                    print(
+                        " Max index of gpu_ids out of range, the number of CUDA_VISIBLE_DEVICES is {}."
+                        .format(len(env_gpus)))
+                    exit(-1)
+        else:
+            env_gpus = []
+    if args.use_lite:
+        print("run using paddle-lite.")
+        start_gpu_card_model(-1, -1, serving_port, args)
+    elif len(gpus) <= 0:
+        print("gpu_ids not set, going to run cpu service.")
+        start_gpu_card_model(-1, -1, serving_port, args)
+    else:
+        gpu_processes = []
+        for i, gpu_id in enumerate(gpus):
+            p = Process(
+                target=start_gpu_card_model,
+                args=(
+                    i,
+                    gpu_id,
+                    serving_port,
+                    args, ))
+            gpu_processes.append(p)
+        for p in gpu_processes:
+            p.start()
+        for p in gpu_processes:
+            p.join()
+
+
 class MainService(BaseHTTPRequestHandler):
     def get_available_port(self):
         default_port = 12000
@@ -146,7 +262,7 @@ class MainService(BaseHTTPRequestHandler):
                 return default_port + i
 
     def start_serving(self):
-        start_standard_model(serving_port)
+        start_multi_card(args, serving_port)
 
     def get_key(self, post_data):
         if "key" not in post_data:
@@ -207,9 +323,9 @@ class MainService(BaseHTTPRequestHandler):
 
 
 if __name__ == "__main__":
-
-    args = parse_args()
+    args = serve_args()
     if args.name == "None":
+        from .web_service import port_is_available
         if args.use_encryption_model:
             p_flag = False
             p = None
@@ -220,27 +336,39 @@ if __name__ == "__main__":
             )
             server.serve_forever()
         else:
-            start_standard_model(args.port)
+            start_multi_card(args)
     else:
-        service = WebService(name=args.name)
-        service.load_model_config(args.model)
-        service.prepare_server(
-            workdir=args.workdir, port=args.port, device=args.device)
-        service.run_rpc_service()
+        from .web_service import WebService
+        web_service = WebService(name=args.name)
+        web_service.load_model_config(args.model)
+        gpu_ids = args.gpu_ids
+        if gpu_ids == "":
+            if "CUDA_VISIBLE_DEVICES" in os.environ:
+                gpu_ids = os.environ["CUDA_VISIBLE_DEVICES"]
+        if len(gpu_ids) > 0:
+            web_service.set_gpus(gpu_ids)
+        web_service.prepare_server(
+            workdir=args.workdir,
+            port=args.port,
+            device=args.device,
+            use_lite=args.use_lite,
+            use_xpu=args.use_xpu,
+            ir_optim=args.ir_optim)
+        web_service.run_rpc_service()
 
         app_instance = Flask(__name__)
 
         @app_instance.before_first_request
         def init():
-            service._launch_web_service()
+            web_service._launch_web_service()
 
-        service_name = "/" + service.name + "/prediction"
+        service_name = "/" + web_service.name + "/prediction"
 
         @app_instance.route(service_name, methods=["POST"])
         def run():
-            return service.get_prediction(request)
+            return web_service.get_prediction(request)
 
         app_instance.run(host="0.0.0.0",
-                         port=service.port,
+                         port=web_service.port,
                          threaded=False,
                          processes=4)

python/paddle_serving_server_gpu/__init__.py → python/paddle_serving_server/server.py

-#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,188 +11,32 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# pylint: disable=doc-string-missing
 
 import os
+import tarfile
+import socket
+import paddle_serving_server as paddle_serving_server
 from .proto import server_configure_pb2 as server_sdk
 from .proto import general_model_config_pb2 as m_config
 import google.protobuf.text_format
-import tarfile
-import socket
-import paddle_serving_server_gpu as paddle_serving_server
 import time
-from .version import serving_server_version
+from .version import serving_server_version, version_suffix, device_type
 from contextlib import closing
 import argparse
-import collections
+
 import sys
 if sys.platform.startswith('win') is False:
     import fcntl
 import shutil
+import platform
 import numpy as np
 import grpc
-from .proto import multi_lang_general_model_service_pb2
 import sys
-sys.path.append(
-    os.path.join(os.path.abspath(os.path.dirname(__file__)), 'proto'))
-from .proto import multi_lang_general_model_service_pb2_grpc
+
 from multiprocessing import Pool, Process
 from concurrent import futures
 
 
-def serve_args():
-    parser = argparse.ArgumentParser("serve")
-    parser.add_argument(
-        "--thread", type=int, default=2, help="Concurrency of server")
-    parser.add_argument(
-        "--model", type=str, default="", help="Model for serving")
-    parser.add_argument(
-        "--port", type=int, default=9292, help="Port of the starting gpu")
-    parser.add_argument(
-        "--workdir",
-        type=str,
-        default="workdir",
-        help="Working dir of current service")
-    parser.add_argument(
-        "--device", type=str, default="gpu", help="Type of device")
-    parser.add_argument("--gpu_ids", type=str, default="", help="gpu ids")
-    parser.add_argument(
-        "--name", type=str, default="None", help="Default service name")
-    parser.add_argument(
-        "--mem_optim_off",
-        default=False,
-        action="store_true",
-        help="Memory optimize")
-    parser.add_argument(
-        "--ir_optim", default=False, action="store_true", help="Graph optimize")
-    parser.add_argument(
-        "--max_body_size",
-        type=int,
-        default=512 * 1024 * 1024,
-        help="Limit sizes of messages")
-    parser.add_argument(
-        "--use_encryption_model",
-        default=False,
-        action="store_true",
-        help="Use encryption model")
-    parser.add_argument(
-        "--use_multilang",
-        default=False,
-        action="store_true",
-        help="Use Multi-language-service")
-    parser.add_argument(
-        "--use_trt", default=False, action="store_true", help="Use TensorRT")
-    parser.add_argument(
-        "--use_lite", default=False, action="store_true", help="Use PaddleLite")
-    parser.add_argument(
-        "--use_xpu", default=False, action="store_true", help="Use XPU")
-    parser.add_argument(
-        "--product_name",
-        type=str,
-        default=None,
-        help="product_name for authentication")
-    parser.add_argument(
-        "--container_id",
-        type=str,
-        default=None,
-        help="container_id for authentication")
-    return parser.parse_args()
-
-
-class OpMaker(object):
-    def __init__(self):
-        self.op_dict = {
-            "general_infer": "GeneralInferOp",
-            "general_reader": "GeneralReaderOp",
-            "general_response": "GeneralResponseOp",
-            "general_text_reader": "GeneralTextReaderOp",
-            "general_text_response": "GeneralTextResponseOp",
-            "general_single_kv": "GeneralSingleKVOp",
-            "general_dist_kv_infer": "GeneralDistKVInferOp",
-            "general_dist_kv": "GeneralDistKVOp"
-        }
-        self.node_name_suffix_ = collections.defaultdict(int)
-
-    def create(self, node_type, engine_name=None, inputs=[], outputs=[]):
-        if node_type not in self.op_dict:
-            raise Exception("Op type {} is not supported right now".format(
-                node_type))
-        node = server_sdk.DAGNode()
-        # node.name will be used as the infer engine name
-        if engine_name:
-            node.name = engine_name
-        else:
-            node.name = '{}_{}'.format(node_type,
-                                       self.node_name_suffix_[node_type])
-            self.node_name_suffix_[node_type] += 1
-
-        node.type = self.op_dict[node_type]
-        if inputs:
-            for dep_node_str in inputs:
-                dep_node = server_sdk.DAGNode()
-                google.protobuf.text_format.Parse(dep_node_str, dep_node)
-                dep = server_sdk.DAGNodeDependency()
-                dep.name = dep_node.name
-                dep.mode = "RO"
-                node.dependencies.extend([dep])
-        # Because the return value will be used as the key value of the
-        # dict, and the proto object is variable which cannot be hashed,
-        # so it is processed into a string. This has little effect on
-        # overall efficiency.
-        return google.protobuf.text_format.MessageToString(node)
-
-
-class OpSeqMaker(object):
-    def __init__(self):
-        self.workflow = server_sdk.Workflow()
-        self.workflow.name = "workflow1"
-        self.workflow.workflow_type = "Sequence"
-
-    def add_op(self, node_str):
-        node = server_sdk.DAGNode()
-        google.protobuf.text_format.Parse(node_str, node)
-        if len(node.dependencies) > 1:
-            raise Exception(
-                'Set more than one predecessor for op in OpSeqMaker is not allowed.'
-            )
-        if len(self.workflow.nodes) >= 1:
-            if len(node.dependencies) == 0:
-                dep = server_sdk.DAGNodeDependency()
-                dep.name = self.workflow.nodes[-1].name
-                dep.mode = "RO"
-                node.dependencies.extend([dep])
-            elif len(node.dependencies) == 1:
-                if node.dependencies[0].name != self.workflow.nodes[-1].name:
-                    raise Exception(
-                        'You must add op in order in OpSeqMaker. The previous op is {}, but the current op is followed by {}.'
-                        .format(node.dependencies[0].name, self.workflow.nodes[
-                            -1].name))
-        self.workflow.nodes.extend([node])
-
-    def get_op_sequence(self):
-        workflow_conf = server_sdk.WorkflowConf()
-        workflow_conf.workflows.extend([self.workflow])
-        return workflow_conf
-
-
-class OpGraphMaker(object):
-    def __init__(self):
-        self.workflow = server_sdk.Workflow()
-        self.workflow.name = "workflow1"
-        # Currently, SDK only supports "Sequence"
-        self.workflow.workflow_type = "Sequence"
-
-    def add_op(self, node_str):
-        node = server_sdk.DAGNode()
-        google.protobuf.text_format.Parse(node_str, node)
-        self.workflow.nodes.extend([node])
-
-    def get_op_graph(self):
-        workflow_conf = server_sdk.WorkflowConf()
-        workflow_conf.workflows.extend([self.workflow])
-        return workflow_conf
-
-
 class Server(object):
     def __init__(self):
         self.server_handle_ = None
@@ -217,6 +61,7 @@ class Server(object):
         self.module_path = os.path.dirname(paddle_serving_server.__file__)
         self.cur_path = os.getcwd()
         self.use_local_bin = False
+        self.mkl_flag = False
         self.device = "cpu"
         self.gpuid = 0
         self.use_trt = False
@@ -317,31 +162,20 @@ class Server(object):
             engine.runtime_thread_num = 0
             engine.batch_infer_size = 0
             engine.enable_batch_align = 0
-            engine.model_data_path = model_config_path
+            engine.model_dir = model_config_path
             engine.enable_memory_optimization = self.memory_optimization
             engine.enable_ir_optimization = self.ir_optimization
-            engine.static_optimization = False
-            engine.force_update_static_cache = False
             engine.use_trt = self.use_trt
-            if os.path.exists('{}/__params__'.format(model_config_path)):
-                suffix = ""
-            else:
-                suffix = "_DIR"
-            if device == "arm":
             engine.use_lite = self.use_lite
             engine.use_xpu = self.use_xpu
-            if device == "cpu":
-                if use_encryption_model:
-                    engine.type = "FLUID_CPU_ANALYSIS_ENCRPT"
+            if os.path.exists('{}/__params__'.format(model_config_path)):
+                engine.combined_model = True
             else:
-                    engine.type = "FLUID_CPU_ANALYSIS" + suffix
-            elif device == "gpu":
+                engine.combined_model = False
             if use_encryption_model:
-                    engine.type = "FLUID_GPU_ANALYSIS_ENCRPT"
-                else:
-                    engine.type = "FLUID_GPU_ANALYSIS" + suffix
-            elif device == "arm":
-                engine.type = "FLUID_ARM_ANALYSIS" + suffix
+                engine.encrypted_model = True
+            engine.type = "PADDLE_INFER"
+
             self.model_toolkit_conf.engines.extend([engine])
 
     def _prepare_infer_service(self, port):
@@ -432,26 +266,53 @@ class Server(object):
         # check config here
         # print config here
 
+    def use_mkl(self, flag):
+        self.mkl_flag = flag
+
+    def get_device_version(self):
+        avx_flag = False
+        mkl_flag = self.mkl_flag
+        openblas_flag = False
+        r = os.system("cat /proc/cpuinfo | grep avx > /dev/null 2>&1")
+        if r == 0:
+            avx_flag = True
+        if avx_flag:
+            if mkl_flag:
+                device_version = "cpu-avx-mkl"
+            else:
+                device_version = "cpu-avx-openblas"
+        else:
+            if mkl_flag:
+                print(
+                    "Your CPU does not support AVX, server will running with noavx-openblas mode."
+                )
+            device_version = "cpu-noavx-openblas"
+        return device_version
+
+    def get_serving_bin_name(self):
+        if device_type == "0":
+            device_version = self.get_device_version()
+        elif device_type == "1":
+            if version_suffix == "101" or version_suffix == "102":
+                device_version = "gpu-" + version_suffix
+            else:
+                device_version = "gpu-cuda" + version_suffix
+        elif device_type == "2":
+            device_version = "xpu-" + platform.machine()
+        return device_version
+
     def download_bin(self):
         os.chdir(self.module_path)
         need_download = False
 
         #acquire lock
         version_file = open("{}/version.py".format(self.module_path), "r")
-        import re
-        for line in version_file.readlines():
-            if re.match("cuda_version", line):
-                cuda_version = line.split("\"")[1]
-                if cuda_version == "101" or cuda_version == "102":
-                    device_version = "serving-gpu-" + cuda_version + "-"
-                elif cuda_version == "arm" or cuda_version == "arm-xpu":
-                    device_version = "serving-" + cuda_version + "-"
-                else:
-                    device_version = "serving-gpu-cuda" + cuda_version + "-"
 
-        folder_name = device_version + serving_server_version
-        tar_name = folder_name + ".tar.gz"
-        bin_url = "https://paddle-serving.bj.bcebos.com/bin/" + tar_name
+        folder_name = "serving-%s-%s" % (self.get_serving_bin_name(),
+                                         serving_server_version)
+        tar_name = "%s.tar.gz" % folder_name
+        bin_url = "https://paddle-serving.bj.bcebos.com/bin/%s" % tar_name
+
         self.server_path = os.path.join(self.module_path, folder_name)
 
         download_flag = "{}/{}.is_download".format(self.module_path,
@@ -503,9 +364,9 @@ class Server(object):
                        cube_conf=None):
         if workdir == None:
             workdir = "./tmp"
-            os.system("mkdir {}".format(workdir))
+            os.system("mkdir -p {}".format(workdir))
         else:
-            os.system("mkdir {}".format(workdir))
+            os.system("mkdir -p {}".format(workdir))
         os.system("touch {}/fluid_time_file".format(workdir))
 
         if not self.port_is_available(port):
@@ -614,157 +475,6 @@ class Server(object):
         os.system(command)
 
 
-class MultiLangServerServiceServicer(multi_lang_general_model_service_pb2_grpc.
-                                     MultiLangGeneralModelServiceServicer):
-    def __init__(self, model_config_path, is_multi_model, endpoints):
-        self.is_multi_model_ = is_multi_model
-        self.model_config_path_ = model_config_path
-        self.endpoints_ = endpoints
-        with open(self.model_config_path_) as f:
-            self.model_config_str_ = str(f.read())
-        self._parse_model_config(self.model_config_str_)
-        self._init_bclient(self.model_config_path_, self.endpoints_)
-
-    def _init_bclient(self, model_config_path, endpoints, timeout_ms=None):
-        from paddle_serving_client import Client
-        self.bclient_ = Client()
-        if timeout_ms is not None:
-            self.bclient_.set_rpc_timeout_ms(timeout_ms)
-        self.bclient_.load_client_config(model_config_path)
-        self.bclient_.connect(endpoints)
-
-    def _parse_model_config(self, model_config_str):
-        model_conf = m_config.GeneralModelConfig()
-        model_conf = google.protobuf.text_format.Merge(model_config_str,
-                                                       model_conf)
-        self.feed_names_ = [var.alias_name for var in model_conf.feed_var]
-        self.feed_types_ = {}
-        self.feed_shapes_ = {}
-        self.fetch_names_ = [var.alias_name for var in model_conf.fetch_var]
-        self.fetch_types_ = {}
-        self.lod_tensor_set_ = set()
-        for i, var in enumerate(model_conf.feed_var):
-            self.feed_types_[var.alias_name] = var.feed_type
-            self.feed_shapes_[var.alias_name] = var.shape
-            if var.is_lod_tensor:
-                self.lod_tensor_set_.add(var.alias_name)
-        for i, var in enumerate(model_conf.fetch_var):
-            self.fetch_types_[var.alias_name] = var.fetch_type
-            if var.is_lod_tensor:
-                self.lod_tensor_set_.add(var.alias_name)
-
-    def _flatten_list(self, nested_list):
-        for item in nested_list:
-            if isinstance(item, (list, tuple)):
-                for sub_item in self._flatten_list(item):
-                    yield sub_item
-            else:
-                yield item
-
-    def _unpack_inference_request(self, request):
-        feed_names = list(request.feed_var_names)
-        fetch_names = list(request.fetch_var_names)
-        is_python = request.is_python
-        log_id = request.log_id
-        feed_batch = []
-        for feed_inst in request.insts:
-            feed_dict = {}
-            for idx, name in enumerate(feed_names):
-                var = feed_inst.tensor_array[idx]
-                v_type = self.feed_types_[name]
-                data = None
-                if is_python:
-                    if v_type == 0:
-                        data = np.frombuffer(var.data, dtype="int64")
-                    elif v_type == 1:
-                        data = np.frombuffer(var.data, dtype="float32")
-                    elif v_type == 2:
-                        data = np.frombuffer(var.data, dtype="int32")
-                    else:
-                        raise Exception("error type.")
-                else:
-                    if v_type == 0:  # int64
-                        data = np.array(list(var.int64_data), dtype="int64")
-                    elif v_type == 1:  # float32
-                        data = np.array(list(var.float_data), dtype="float32")
-                    elif v_type == 2:
-                        data = np.array(list(var.int_data), dtype="int32")
-                    else:
-                        raise Exception("error type.")
-                data.shape = list(feed_inst.tensor_array[idx].shape)
-                feed_dict[name] = data
-                if len(var.lod) > 0:
-                    feed_dict["{}.lod".format(name)] = var.lod
-            feed_batch.append(feed_dict)
-        return feed_batch, fetch_names, is_python, log_id
-
-    def _pack_inference_response(self, ret, fetch_names, is_python):
-        resp = multi_lang_general_model_service_pb2.InferenceResponse()
-        if ret is None:
-            resp.err_code = 1
-            return resp
-        results, tag = ret
-        resp.tag = tag
-        resp.err_code = 0
-
-        if not self.is_multi_model_:
-            results = {'general_infer_0': results}
-        for model_name, model_result in results.items():
-            model_output = multi_lang_general_model_service_pb2.ModelOutput()
-            inst = multi_lang_general_model_service_pb2.FetchInst()
-            for idx, name in enumerate(fetch_names):
-                tensor = multi_lang_general_model_service_pb2.Tensor()
-                v_type = self.fetch_types_[name]
-                if is_python:
-                    tensor.data = model_result[name].tobytes()
-                else:
-                    if v_type == 0:  # int64
-                        tensor.int64_data.extend(model_result[name].reshape(-1)
-                                                 .tolist())
-                    elif v_type == 1:  # float32
-                        tensor.float_data.extend(model_result[name].reshape(-1)
-                                                 .tolist())
-                    elif v_type == 2:  # int32
-                        tensor.int_data.extend(model_result[name].reshape(-1)
-                                               .tolist())
-                    else:
-                        raise Exception("error type.")
-                tensor.shape.extend(list(model_result[name].shape))
-                if "{}.lod".format(name) in model_result:
-                    tensor.lod.extend(model_result["{}.lod".format(name)]
-                                      .tolist())
-                inst.tensor_array.append(tensor)
-            model_output.insts.append(inst)
-            model_output.engine_name = model_name
-            resp.outputs.append(model_output)
-        return resp
-
-    def SetTimeout(self, request, context):
-        # This porcess and Inference process cannot be operate at the same time.
-        # For performance reasons, do not add thread lock temporarily.
-        timeout_ms = request.timeout_ms
-        self._init_bclient(self.model_config_path_, self.endpoints_, timeout_ms)
-        resp = multi_lang_general_model_service_pb2.SimpleResponse()
-        resp.err_code = 0
-        return resp
-
-    def Inference(self, request, context):
-        feed_batch, fetch_names, is_python, log_id \
-                = self._unpack_inference_request(request)
-        ret = self.bclient_.predict(
-            feed=feed_batch,
-            fetch=fetch_names,
-            batch=True,
-            need_variant_tag=True,
-            log_id=log_id)
-        return self._pack_inference_response(ret, fetch_names, is_python)
-
-    def GetClientConfig(self, request, context):
-        resp = multi_lang_general_model_service_pb2.GetClientConfigResponse()
-        resp.client_config_str = self.model_config_str_
-        return resp
-
-
 class MultiLangServer(object):
     def __init__(self):
         self.bserver_ = Server()
@@ -808,6 +518,9 @@ class MultiLangServer(object):
     def set_op_graph(self, op_graph):
         self.bserver_.set_op_graph(op_graph)
 
+    def use_mkl(self, flag):
+        self.bserver_.use_mkl(flag)
+
     def set_memory_optimize(self, flag=False):
         self.bserver_.set_memory_optimize(flag)
 

python/paddle_serving_server/version.py

@@ -11,8 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-""" Paddle Serving Client version string """
+""" Paddle Serving Server version string """
 serving_client_version = "0.0.0"
 serving_server_version = "0.0.0"
 module_proto_version = "0.0.0"
+version_suffix = ""
+device_type = "0"
+cuda_version = "9"
 commit_id = ""

python/paddle_serving_server/web_service.py

@@ -15,16 +15,19 @@
 # pylint: disable=doc-string-missing
 
 from flask import Flask, request, abort
-from multiprocessing import Pool, Process
-from paddle_serving_server import OpMaker, OpSeqMaker, Server
-from paddle_serving_client import Client
 from contextlib import closing
+from multiprocessing import Pool, Process, Queue
+from paddle_serving_client import Client
+from paddle_serving_server import OpMaker, OpSeqMaker, Server
+from paddle_serving_server.serve import start_multi_card
 import socket
+import sys
 import numpy as np
+import paddle_serving_server as serving
+
 from paddle_serving_server import pipeline
 from paddle_serving_server.pipeline import Op
 
-
 def port_is_available(port):
     with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
         sock.settimeout(2)
@@ -34,13 +37,15 @@ def port_is_available(port):
     else:
         return False
 
-
 class WebService(object):
     def __init__(self, name="default_service"):
         self.name = name
         # pipeline
         self._server = pipeline.PipelineServer(self.name)
 
+        self.gpus = []  # deprecated
+        self.rpc_service_list = []  # deprecated
+
     def get_pipeline_response(self, read_op):
         return None
 
@@ -77,58 +82,115 @@ class WebService(object):
         self.feed_vars = {var.name: var for var in model_conf.feed_var}
         self.fetch_vars = {var.name: var for var in model_conf.fetch_var}
 
-    def _launch_rpc_service(self):
-        op_maker = OpMaker()
+    def set_gpus(self, gpus):
+        print("This API will be deprecated later. Please do not use it")
+        self.gpus = [int(x) for x in gpus.split(",")]
+
+    def default_rpc_service(self,
+                            workdir="conf",
+                            port=9292,
+                            gpuid=0,
+                            thread_num=2,
+                            mem_optim=True,
+                            use_lite=False,
+                            use_xpu=False,
+                            ir_optim=False):
+        device = "gpu"
+        if gpuid == -1:
+            if use_lite:
+                device = "arm"
+            else:
+                device = "cpu"
+        op_maker = serving.OpMaker()
         read_op = op_maker.create('general_reader')
         general_infer_op = op_maker.create('general_infer')
         general_response_op = op_maker.create('general_response')
+
         op_seq_maker = OpSeqMaker()
         op_seq_maker.add_op(read_op)
         op_seq_maker.add_op(general_infer_op)
         op_seq_maker.add_op(general_response_op)
+
         server = Server()
         server.set_op_sequence(op_seq_maker.get_op_sequence())
-        server.set_num_threads(16)
-        server.set_memory_optimize(self.mem_optim)
-        server.set_ir_optimize(self.ir_optim)
+        server.set_num_threads(thread_num)
+        server.set_memory_optimize(mem_optim)
+        server.set_ir_optimize(ir_optim)
+        server.set_device(device)
+
+        if use_lite:
+            server.set_lite()
+        if use_xpu:
+            server.set_xpu()
+
         server.load_model_config(self.model_config)
-        server.prepare_server(
-            workdir=self.workdir, port=self.port_list[0], device=self.device)
-        server.run_server()
+        if gpuid >= 0:
+            server.set_gpuid(gpuid)
+        server.prepare_server(workdir=workdir, port=port, device=device)
+        return server
 
-    def port_is_available(self, port):
-        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
-            sock.settimeout(2)
-            result = sock.connect_ex(('0.0.0.0', port))
-        if result != 0:
-            return True
-        else:
-            return False
+    def _launch_rpc_service(self, service_idx):
+        self.rpc_service_list[service_idx].run_server()
 
     def prepare_server(self,
                        workdir="",
                        port=9393,
-                       device="cpu",
-                       mem_optim=True,
-                       ir_optim=False):
+                       device="gpu",
+                       use_lite=False,
+                       use_xpu=False,
+                       ir_optim=False,
+                       gpuid=0,
+                       mem_optim=True):
         print("This API will be deprecated later. Please do not use it")
         self.workdir = workdir
         self.port = port
         self.device = device
-        default_port = 12000
+        self.gpuid = gpuid
         self.port_list = []
-        self.mem_optim = mem_optim
-        self.ir_optim = ir_optim
+        default_port = 12000
         for i in range(1000):
             if port_is_available(default_port + i):
                 self.port_list.append(default_port + i)
+            if len(self.port_list) > len(self.gpus):
                 break
 
+        if len(self.gpus) == 0:
+            # init cpu service
+            self.rpc_service_list.append(
+                self.default_rpc_service(
+                    self.workdir,
+                    self.port_list[0],
+                    -1,
+                    thread_num=2,
+                    mem_optim=mem_optim,
+                    use_lite=use_lite,
+                    use_xpu=use_xpu,
+                    ir_optim=ir_optim))
+        else:
+            for i, gpuid in enumerate(self.gpus):
+                self.rpc_service_list.append(
+                    self.default_rpc_service(
+                        "{}_{}".format(self.workdir, i),
+                        self.port_list[i],
+                        gpuid,
+                        thread_num=2,
+                        mem_optim=mem_optim,
+                        use_lite=use_lite,
+                        use_xpu=use_xpu,
+                        ir_optim=ir_optim))
+
     def _launch_web_service(self):
+        gpu_num = len(self.gpus)
         self.client = Client()
         self.client.load_client_config("{}/serving_server_conf.prototxt".format(
             self.model_config))
-        self.client.connect(["0.0.0.0:{}".format(self.port_list[0])])
+        endpoints = ""
+        if gpu_num > 0:
+            for i in range(gpu_num):
+                endpoints += "127.0.0.1:{},".format(self.port_list[i])
+        else:
+            endpoints = "127.0.0.1:{}".format(self.port_list[0])
+        self.client.connect([endpoints])
 
     def get_prediction(self, request):
         if not request.json:
@@ -158,8 +220,12 @@ class WebService(object):
         print("web service address:")
         print("http://{}:{}/{}/prediction".format(localIP, self.port,
                                                   self.name))
-        p_rpc = Process(target=self._launch_rpc_service)
-        p_rpc.start()
+        server_pros = []
+        for i, service in enumerate(self.rpc_service_list):
+            p = Process(target=self._launch_rpc_service, args=(i, ))
+            server_pros.append(p)
+        for p in server_pros:
+            p.start()
 
         app_instance = Flask(__name__)
 
@@ -175,7 +241,9 @@ class WebService(object):
 
         self.app_instance = app_instance
 
-    def run_debugger_service(self):
+    # TODO: maybe change another API name: maybe run_local_predictor?
+    def run_debugger_service(self, gpu=False):
+        print("This API will be deprecated later. Please do not use it")
         import socket
         localIP = socket.gethostbyname(socket.gethostname())
         print("web service address:")
@@ -185,7 +253,7 @@ class WebService(object):
 
         @app_instance.before_first_request
         def init():
-            self._launch_local_predictor()
+            self._launch_local_predictor(gpu)
 
         service_name = "/" + self.name + "/prediction"
 
@@ -195,11 +263,11 @@ class WebService(object):
 
         self.app_instance = app_instance
 
-    def _launch_local_predictor(self):
+    def _launch_local_predictor(self, gpu):
         from paddle_serving_app.local_predict import LocalPredictor
         self.client = LocalPredictor()
         self.client.load_model_config(
-            "{}".format(self.model_config), use_gpu=False)
+            "{}".format(self.model_config), use_gpu=True, gpu_id=self.gpus[0])
 
     def run_web_service(self):
         print("This API will be deprecated later. Please do not use it")

python/paddle_serving_server_gpu/serve.py

-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-Usage:
-    Host a trained paddle model with one line command
-    Example:
-        python -m paddle_serving_server.serve --model ./serving_server_model --port 9292
-"""
-import argparse
-import os
-import json
-import base64
-import time
-from multiprocessing import Pool, Process
-from paddle_serving_server_gpu import serve_args
-from flask import Flask, request
-import sys
-if sys.version_info.major == 2:
-    from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
-elif sys.version_info.major == 3:
-    from http.server import BaseHTTPRequestHandler, HTTPServer
-
-
-def start_gpu_card_model(index, gpuid, port, args):  # pylint: disable=doc-string-missing
-    gpuid = int(gpuid)
-    device = "gpu"
-    if gpuid == -1:
-        device = "cpu"
-    elif gpuid >= 0:
-        port = port + index
-    thread_num = args.thread
-    model = args.model
-    mem_optim = args.mem_optim_off is False
-    ir_optim = args.ir_optim
-    max_body_size = args.max_body_size
-    use_multilang = args.use_multilang
-    workdir = args.workdir
-    if gpuid >= 0:
-        workdir = "{}_{}".format(args.workdir, gpuid)
-
-    if model == "":
-        print("You must specify your serving model")
-        exit(-1)
-
-    import paddle_serving_server_gpu as serving
-    op_maker = serving.OpMaker()
-    read_op = op_maker.create('general_reader')
-    general_infer_op = op_maker.create('general_infer')
-    general_response_op = op_maker.create('general_response')
-
-    op_seq_maker = serving.OpSeqMaker()
-    op_seq_maker.add_op(read_op)
-    op_seq_maker.add_op(general_infer_op)
-    op_seq_maker.add_op(general_response_op)
-
-    if use_multilang:
-        server = serving.MultiLangServer()
-    else:
-        server = serving.Server()
-    server.set_op_sequence(op_seq_maker.get_op_sequence())
-    server.set_num_threads(thread_num)
-    server.set_memory_optimize(mem_optim)
-    server.set_ir_optimize(ir_optim)
-    server.set_max_body_size(max_body_size)
-    if args.use_trt:
-        server.set_trt()
-
-    if args.use_lite:
-        server.set_lite()
-        device = "arm"
-
-    server.set_device(device)
-    if args.use_xpu:
-        server.set_xpu()
-
-    if args.product_name != None:
-        server.set_product_name(args.product_name)
-    if args.container_id != None:
-        server.set_container_id(args.container_id)
-
-    server.load_model_config(model)
-    server.prepare_server(
-        workdir=workdir,
-        port=port,
-        device=device,
-        use_encryption_model=args.use_encryption_model)
-    if gpuid >= 0:
-        server.set_gpuid(gpuid)
-    server.run_server()
-
-
-def start_multi_card(args, serving_port=None):  # pylint: disable=doc-string-missing
-    gpus = ""
-    if serving_port == None:
-        serving_port = args.port
-    if args.gpu_ids == "":
-        gpus = []
-    else:
-        gpus = args.gpu_ids.split(",")
-        if "CUDA_VISIBLE_DEVICES" in os.environ:
-            env_gpus = os.environ["CUDA_VISIBLE_DEVICES"].split(",")
-            for ids in gpus:
-                if int(ids) >= len(env_gpus):
-                    print(
-                        " Max index of gpu_ids out of range, the number of CUDA_VISIBLE_DEVICES is {}."
-                        .format(len(env_gpus)))
-                    exit(-1)
-        else:
-            env_gpus = []
-    if args.use_lite:
-        print("run arm server.")
-        start_gpu_card_model(-1, -1, args)
-    elif len(gpus) <= 0:
-        print("gpu_ids not set, going to run cpu service.")
-        start_gpu_card_model(-1, -1, serving_port, args)
-    else:
-        gpu_processes = []
-        for i, gpu_id in enumerate(gpus):
-            p = Process(
-                target=start_gpu_card_model,
-                args=(
-                    i,
-                    gpu_id,
-                    serving_port,
-                    args, ))
-            gpu_processes.append(p)
-        for p in gpu_processes:
-            p.start()
-        for p in gpu_processes:
-            p.join()
-
-
-class MainService(BaseHTTPRequestHandler):
-    def get_available_port(self):
-        default_port = 12000
-        for i in range(1000):
-            if port_is_available(default_port + i):
-                return default_port + i
-
-    def start_serving(self):
-        start_multi_card(args, serving_port)
-
-    def get_key(self, post_data):
-        if "key" not in post_data:
-            return False
-        else:
-            key = base64.b64decode(post_data["key"].encode())
-            with open(args.model + "/key", "wb") as f:
-                f.write(key)
-            return True
-
-    def check_key(self, post_data):
-        if "key" not in post_data:
-            return False
-        else:
-            key = base64.b64decode(post_data["key"].encode())
-            with open(args.model + "/key", "rb") as f:
-                cur_key = f.read()
-            return (key == cur_key)
-
-    def start(self, post_data):
-        post_data = json.loads(post_data)
-        global p_flag
-        if not p_flag:
-            if args.use_encryption_model:
-                print("waiting key for model")
-                if not self.get_key(post_data):
-                    print("not found key in request")
-                    return False
-            global serving_port
-            global p
-            serving_port = self.get_available_port()
-            p = Process(target=self.start_serving)
-            p.start()
-            time.sleep(3)
-            if p.is_alive():
-                p_flag = True
-            else:
-                return False
-        else:
-            if p.is_alive():
-                if not self.check_key(post_data):
-                    return False
-            else:
-                return False
-        return True
-
-    def do_POST(self):
-        content_length = int(self.headers['Content-Length'])
-        post_data = self.rfile.read(content_length)
-        if self.start(post_data):
-            response = {"endpoint_list": [serving_port]}
-        else:
-            response = {"message": "start serving failed"}
-        self.send_response(200)
-        self.send_header('Content-type', 'application/json')
-        self.end_headers()
-        self.wfile.write(json.dumps(response).encode())
-
-
-if __name__ == "__main__":
-    args = serve_args()
-    if args.name == "None":
-        from .web_service import port_is_available
-        if args.use_encryption_model:
-            p_flag = False
-            p = None
-            serving_port = 0
-            server = HTTPServer(('localhost', int(args.port)), MainService)
-            print(
-                'Starting encryption server, waiting for key from client, use <Ctrl-C> to stop'
-            )
-            server.serve_forever()
-        else:
-            start_multi_card(args)
-    else:
-        from .web_service import WebService
-        web_service = WebService(name=args.name)
-        web_service.load_model_config(args.model)
-        gpu_ids = args.gpu_ids
-        if gpu_ids == "":
-            if "CUDA_VISIBLE_DEVICES" in os.environ:
-                gpu_ids = os.environ["CUDA_VISIBLE_DEVICES"]
-        if len(gpu_ids) > 0:
-            web_service.set_gpus(gpu_ids)
-        web_service.prepare_server(
-            workdir=args.workdir,
-            port=args.port,
-            device=args.device,
-            use_lite=args.use_lite,
-            use_xpu=args.use_xpu,
-            ir_optim=args.ir_optim)
-        web_service.run_rpc_service()
-
-        app_instance = Flask(__name__)
-
-        @app_instance.before_first_request
-        def init():
-            web_service._launch_web_service()
-
-        service_name = "/" + web_service.name + "/prediction"
-
-        @app_instance.route(service_name, methods=["POST"])
-        def run():
-            return web_service.get_prediction(request)
-
-        app_instance.run(host="0.0.0.0",
-                         port=web_service.port,
-                         threaded=False,
-                         processes=4)

python/paddle_serving_server_gpu/version.py

-# Copyright (c) 2020  PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" Paddle Serving Client version string """
-serving_client_version = "0.0.0"
-serving_server_version = "0.0.0"
-module_proto_version = "0.0.0"
-cuda_version = "9"
-commit_id = ""