diff --git a/tools/Dockerfile.runtime_template b/tools/Dockerfile.runtime_template
index 4df0496445791c3210ee8e3f32602e7d1a29dafa..b900e772f07eae981f3d7f4dc46734a7a79939de 100644
--- a/tools/Dockerfile.runtime_template
+++ b/tools/Dockerfile.runtime_template
@@ -30,6 +30,10 @@ WORKDIR /home
       COPY tools/dockerfiles/build_scripts /build_scripts
       RUN bash /build_scripts/install_whl.sh <<serving_version>> <<paddle_version>> <<run_env>> <<python_version>> && rm -rf /build_scripts
 
+WORKDIR /home
+      COPY tools/dockerfiles/build_scripts /build_scripts
+      RUN bash /build_scripts/soft_link.sh <<run_env>>
+
 # install tensorrt
 WORKDIR /home
       COPY tools/dockerfiles/build_scripts /build_scripts
diff --git a/tools/dockerfiles/build_scripts/install_whl.sh b/tools/dockerfiles/build_scripts/install_whl.sh
index 46d8d226a63432d004a5ed3f53a6df41428ebd2a..5849db5b2cbef6a720578064978676552a8b6c70 100644
--- a/tools/dockerfiles/build_scripts/install_whl.sh
+++ b/tools/dockerfiles/build_scripts/install_whl.sh
@@ -42,7 +42,7 @@ if [[ $SERVING_VERSION == "0.5.0" ]]; then
         serving_bin="https://paddle-serving.bj.bcebos.com/bin/serving-gpu-102-${SERVING_VERSION}.tar.gz"
     elif [[ "$RUN_ENV" == "cuda11" ]];then
         server_release="paddle-serving-server-gpu==$SERVING_VERSION.post11"
-        serving_bin="https://paddle-serving.bj.bcebos.com/bin/serving-gpu-11-${SERVING_VERSION}.tar.gz"
+        serving_bin="https://paddle-serving.bj.bcebos.com/bin/serving-gpu-cuda11-${SERVING_VERSION}.tar.gz"
     fi
     client_release="paddle-serving-client==$SERVING_VERSION"
     app_release="paddle-serving-app==0.3.1"
@@ -58,7 +58,7 @@ elif [[ $SERVING_VERSION == "0.6.0" ]]; then
         serving_bin="https://paddle-serving.bj.bcebos.com/test-dev/bin/serving-gpu-102-$SERVING_VERSION.tar.gz"
     elif [[ "$RUN_ENV" == "cuda11" ]];then
         server_release="https://paddle-serving.bj.bcebos.com/test-dev/whl/paddle_serving_server_gpu-$SERVING_VERSION.post11-py3-none-any.whl"
-        serving_bin="https://paddle-serving.bj.bcebos.com/test-dev/bin/serving-gpu-11-$SERVING_VERSION.tar.gz"
+        serving_bin="https://paddle-serving.bj.bcebos.com/test-dev/bin/serving-gpu-cuda11-$SERVING_VERSION.tar.gz"
     fi
     client_release="https://paddle-serving.bj.bcebos.com/test-dev/whl/paddle_serving_client-$SERVING_VERSION-cp$CPYTHON-none-any.whl"
     app_release="https://paddle-serving.bj.bcebos.com/test-dev/whl/paddle_serving_app-$SERVING_VERSION-py3-none-any.whl"
@@ -99,10 +99,10 @@ elif [[ "$RUN_ENV" == "cuda11" ]];then
     python$PYTHON_VERSION -m pip install paddlepaddle-gpu==${PADDLE_VERSION}
     cd /usr/local/
     wget $serving_bin
-    tar xf serving-gpu-11-${SERVING_VERSION}.tar.gz
-    mv $PWD/serving-gpu-11-${SERVING_VERSION} $PWD/serving_bin
+    tar xf serving-gpu-cuda11-${SERVING_VERSION}.tar.gz
+    mv $PWD/serving-gpu-cuda11-${SERVING_VERSION} $PWD/serving_bin
     echo "export SERVING_BIN=$PWD/serving_bin/serving">>/root/.bashrc
-    rm -rf serving-gpu-11-${SERVING_VERSION}.tar.gz
+    rm -rf serving-gpu-cuda11-${SERVING_VERSION}.tar.gz
     cd -
 fi
 
diff --git a/tools/dockerfiles/build_scripts/soft_link.sh b/tools/dockerfiles/build_scripts/soft_link.sh
new file mode 100644
index 0000000000000000000000000000000000000000..ff72bd842f73b5426424bb48759d6843d03efe28
--- /dev/null
+++ b/tools/dockerfiles/build_scripts/soft_link.sh
@@ -0,0 +1,22 @@
+RUN_ENV=$1
+if [[ "$RUN_ENV" == "cuda10.1" ]];then
+    ln -sf /usr/local/cuda-10.1/targets/x86_64-linux/lib/libcudart.so.10.1 /usr/lib/libcudart.so && \
+    ln -sf /usr/local/cuda-10.1/targets/x86_64-linux/lib/libcusolver.so.10 /usr/lib/libcusolver.so && \
+    ln -sf /usr/lib/x86_64-linux-gnu/libcuda.so /usr/lib/libcuda.so && \
+    ln -sf /usr/lib/x86_64-linux-gnu/libcublas.so.10 /usr/lib/libcublas.so && \
+    ln -sf /usr/lib/x86_64-linux-gnu/libcudnn.so.7 /usr/lib/libcudnn.so
+
+elif [[ "$RUN_ENV" == "cuda10.1" ]];then
+    ln -sf /usr/local/cuda-10.2/targets/x86_64-linux/lib/libcudart.so.10.2 /usr/lib/libcudart.so && \
+    ln -sf /usr/local/cuda-10.2/targets/x86_64-linux/lib/libcusolver.so.10 /usr/lib/libcusolver.so && \
+    ln -sf /usr/lib/x86_64-linux-gnu/libcuda.so /usr/lib/libcuda.so && \
+    ln -sf /usr/lib/x86_64-linux-gnu/libcublas.so.10 /usr/lib/libcublas.so && \
+    ln -sf /usr/lib/x86_64-linux-gnu/libcudnn.so.8 /usr/lib/libcudnn.so
+
+elif [[ "$RUN_ENV" == "cuda10.1" ]];then
+    ln -sf /usr/local/cuda-11.0/targets/x86_64-linux/lib/libcudart.so.11.0 /usr/lib/libcudart.so && \
+    ln -sf /usr/local/cuda-11.0/targets/x86_64-linux/lib/libcusolver.so.10 /usr/lib/libcusolver.so && \
+    ln -sf /usr/lib/x86_64-linux-gnu/libcuda.so /usr/lib/libcuda.so && \
+    ln -sf /usr/local/cuda-11.0/targets/x86_64-linux/lib/libcublas.so.11 /usr/lib/libcublas.so && \
+    ln -sf /usr/lib/x86_64-linux-gnu/libcudnn.so.8 /usr/lib/libcudnn.so
+fi