Merge pull request #472 from barrierye/add-monitor-ut

add UT for monitor(ftp&general type)

tools/serving_build.sh

@@ -323,6 +323,9 @@ function python_test_bert() {
             echo "bert RPC inference pass"
             ;;
         *)
+            echo "error type"
+            exit 1
+            ;;
     esac
     echo "test bert $TYPE finished as expected."
     unset SERVING_BIN
@@ -357,6 +360,9 @@ function python_test_imdb() {
             echo "imdb ignore GPU test"
             ;;
         *)
+            echo "error type"
+            exit 1
+            ;;
     esac
     echo "test imdb $TYPE finished as expected."
     unset SERVING_BIN
@@ -389,6 +395,9 @@ function python_test_lac() {
             echo "lac ignore GPU test"
             ;;
         *)
+            echo "error type"
+            exit 1
+            ;;
     esac
     echo "test lac $TYPE finished as expected."
     unset SERVING_BIN
@@ -408,6 +417,248 @@ function python_run_test() {
     cd ../.. # pwd: /Serving
 }
 
+function monitor_test() {
+    local TYPE=$1 # pwd: /Serving
+    mkdir _monitor_test && cd _monitor_test # pwd: /Serving/_monitor_test
+    case $TYPE in
+        CPU):
+            pip install pyftpdlib
+            mkdir remote_path
+            mkdir local_path
+            cd remote_path # pwd: /Serving/_monitor_test/remote_path
+            check_cmd "python -m pyftpdlib -p 8000 &>/dev/null &"
+            cd .. # pwd: /Serving/_monitor_test
+
+            # type: ftp
+            # remote_path: /
+            # remote_model_name: uci_housing.tar.gz
+            # local_tmp_path: ___tmp
+            # local_path: local_path
+            cd remote_path # pwd: /Serving/_monitor_test/remote_path
+            wget --no-check-certificate https://paddle-serving.bj.bcebos.com/uci_housing.tar.gz
+            touch donefile
+            cd ..  # pwd: /Serving/_monitor_test
+            mkdir -p local_path/uci_housing_model
+            python -m paddle_serving_server.monitor \
+                    --type='ftp' --ftp_host='127.0.0.1' --ftp_port='8000' \
+                    --remote_path='/' --remote_model_name='uci_housing.tar.gz' \
+                    --remote_donefile_name='donefile' --local_path='local_path' \
+                    --local_model_name='uci_housing_model' --local_timestamp_file='fluid_time_file' \
+                    --local_tmp_path='___tmp' --unpacked_filename='uci_housing_model' \
+                    --interval='1' >/dev/null &
+            sleep 10
+            if [ ! -f "local_path/uci_housing_model/fluid_time_file" ]; then
+                echo "local_path/uci_housing_model/fluid_time_file not exist."
+                exit 1
+            fi
+            ps -ef | grep "monitor" | grep -v grep | awk '{print $2}' | xargs kill
+            rm -rf remote_path/*
+            rm -rf local_path/*
+
+            # type: ftp
+            # remote_path: /tmp_dir
+            # remote_model_name: uci_housing_model
+            # local_tmp_path: ___tmp
+            # local_path: local_path
+            mkdir -p remote_path/tmp_dir && cd remote_path/tmp_dir # pwd: /Serving/_monitor_test/remote_path/tmp_dir
+            wget --no-check-certificate https://paddle-serving.bj.bcebos.com/uci_housing.tar.gz
+            tar -xzf uci_housing.tar.gz
+            touch donefile
+            cd ../.. # pwd: /Serving/_monitor_test
+            mkdir -p local_path/uci_housing_model
+            python -m paddle_serving_server.monitor \
+                    --type='ftp' --ftp_host='127.0.0.1' --ftp_port='8000' \
+                    --remote_path='/tmp_dir' --remote_model_name='uci_housing_model' \
+                    --remote_donefile_name='donefile' --local_path='local_path' \
+                    --local_model_name='uci_housing_model' --local_timestamp_file='fluid_time_file' \
+                    --local_tmp_path='___tmp' --interval='1' >/dev/null &
+            sleep 10
+            if [ ! -f "local_path/uci_housing_model/fluid_time_file" ]; then
+                echo "local_path/uci_housing_model/fluid_time_file not exist."
+                exit 1
+            fi
+            ps -ef | grep "monitor" | grep -v grep | awk '{print $2}' | xargs kill
+            rm -rf remote_path/*
+            rm -rf local_path/*
+
+            # type: general
+            # remote_path: /
+            # remote_model_name: uci_housing.tar.gz
+            # local_tmp_path: ___tmp
+            # local_path: local_path
+            cd remote_path # pwd: /Serving/_monitor_test/remote_path
+            wget --no-check-certificate https://paddle-serving.bj.bcebos.com/uci_housing.tar.gz
+            touch donefile
+            cd ..  # pwd: /Serving/_monitor_test
+            mkdir -p local_path/uci_housing_model
+            python -m paddle_serving_server.monitor \
+                    --type='general' --general_host='ftp://127.0.0.1:8000' \
+                    --remote_path='/' --remote_model_name='uci_housing.tar.gz' \
+                    --remote_donefile_name='donefile' --local_path='local_path' \
+                    --local_model_name='uci_housing_model' --local_timestamp_file='fluid_time_file' \
+                    --local_tmp_path='___tmp' --unpacked_filename='uci_housing_model' \
+                    --interval='1' >/dev/null &
+            sleep 10
+            if [ ! -f "local_path/uci_housing_model/fluid_time_file" ]; then
+                echo "local_path/uci_housing_model/fluid_time_file not exist."
+                exit 1
+            fi
+            ps -ef | grep "monitor" | grep -v grep | awk '{print $2}' | xargs kill
+            rm -rf remote_path/*
+            rm -rf local_path/*
+
+            # type: general
+            # remote_path: /tmp_dir
+            # remote_model_name: uci_housing_model
+            # local_tmp_path: ___tmp
+            # local_path: local_path
+            mkdir -p remote_path/tmp_dir && cd remote_path/tmp_dir # pwd: /Serving/_monitor_test/remote_path/tmp_dir
+            wget --no-check-certificate https://paddle-serving.bj.bcebos.com/uci_housing.tar.gz
+            tar -xzf uci_housing.tar.gz
+            touch donefile
+            cd ../.. # pwd: /Serving/_monitor_test
+            mkdir -p local_path/uci_housing_model
+            python -m paddle_serving_server.monitor \
+                    --type='general' --general_host='ftp://127.0.0.1:8000' \
+                    --remote_path='/tmp_dir' --remote_model_name='uci_housing_model' \
+                    --remote_donefile_name='donefile' --local_path='local_path' \
+                    --local_model_name='uci_housing_model' --local_timestamp_file='fluid_time_file' \
+                    --local_tmp_path='___tmp' --interval='1' >/dev/null &
+            sleep 10
+            if [ ! -f "local_path/uci_housing_model/fluid_time_file" ]; then
+                echo "local_path/uci_housing_model/fluid_time_file not exist."
+                exit 1
+            fi
+            ps -ef | grep "monitor" | grep -v grep | awk '{print $2}' | xargs kill
+            rm -rf remote_path/*
+            rm -rf local_path/*
+
+            ps -ef | grep "pyftpdlib" | grep -v grep | awk '{print $2}' | xargs kill
+            ;;
+        GPU):
+            pip install pyftpdlib
+            mkdir remote_path
+            mkdir local_path
+            cd remote_path # pwd: /Serving/_monitor_test/remote_path
+            check_cmd "python -m pyftpdlib -p 8000 &>/dev/null &"
+            cd .. # pwd: /Serving/_monitor_test
+
+            # type: ftp
+            # remote_path: /
+            # remote_model_name: uci_housing.tar.gz
+            # local_tmp_path: ___tmp
+            # local_path: local_path
+            cd remote_path # pwd: /Serving/_monitor_test/remote_path
+            wget --no-check-certificate https://paddle-serving.bj.bcebos.com/uci_housing.tar.gz
+            touch donefile
+            cd ..  # pwd: /Serving/_monitor_test
+            mkdir -p local_path/uci_housing_model
+            python -m paddle_serving_server_gpu.monitor \
+                    --type='ftp' --ftp_host='127.0.0.1' --ftp_port='8000' \
+                    --remote_path='/' --remote_model_name='uci_housing.tar.gz' \
+                    --remote_donefile_name='donefile' --local_path='local_path' \
+                    --local_model_name='uci_housing_model' --local_timestamp_file='fluid_time_file' \
+                    --local_tmp_path='___tmp' --unpacked_filename='uci_housing_model' \
+                    --interval='1' >/dev/null &
+            sleep 10
+            if [ ! -f "local_path/uci_housing_model/fluid_time_file" ]; then
+                echo "local_path/uci_housing_model/fluid_time_file not exist."
+                exit 1
+            fi
+            ps -ef | grep "monitor" | grep -v grep | awk '{print $2}' | xargs kill
+            rm -rf remote_path/*
+            rm -rf local_path/*
+
+            # type: ftp
+            # remote_path: /tmp_dir
+            # remote_model_name: uci_housing_model
+            # local_tmp_path: ___tmp
+            # local_path: local_path
+            mkdir -p remote_path/tmp_dir && cd remote_path/tmp_dir # pwd: /Serving/_monitor_test/remote_path/tmp_dir
+            wget --no-check-certificate https://paddle-serving.bj.bcebos.com/uci_housing.tar.gz
+            tar -xzf uci_housing.tar.gz
+            touch donefile
+            cd ../.. # pwd: /Serving/_monitor_test
+            mkdir -p local_path/uci_housing_model
+            python -m paddle_serving_server_gpu.monitor \
+                    --type='ftp' --ftp_host='127.0.0.1' --ftp_port='8000' \
+                    --remote_path='/tmp_dir' --remote_model_name='uci_housing_model' \
+                    --remote_donefile_name='donefile' --local_path='local_path' \
+                    --local_model_name='uci_housing_model' --local_timestamp_file='fluid_time_file' \
+                    --local_tmp_path='___tmp' --interval='1' >/dev/null &
+            sleep 10
+            if [ ! -f "local_path/uci_housing_model/fluid_time_file" ]; then
+                echo "local_path/uci_housing_model/fluid_time_file not exist."
+                exit 1
+            fi
+            ps -ef | grep "monitor" | grep -v grep | awk '{print $2}' | xargs kill
+            rm -rf remote_path/*
+            rm -rf local_path/*
+
+            # type: general
+            # remote_path: /
+            # remote_model_name: uci_housing.tar.gz
+            # local_tmp_path: ___tmp
+            # local_path: local_path
+            cd remote_path # pwd: /Serving/_monitor_test/remote_path
+            wget --no-check-certificate https://paddle-serving.bj.bcebos.com/uci_housing.tar.gz
+            touch donefile
+            cd ..  # pwd: /Serving/_monitor_test
+            mkdir -p local_path/uci_housing_model
+            python -m paddle_serving_server_gpu.monitor \
+                    --type='general' --general_host='ftp://127.0.0.1:8000' \
+                    --remote_path='/' --remote_model_name='uci_housing.tar.gz' \
+                    --remote_donefile_name='donefile' --local_path='local_path' \
+                    --local_model_name='uci_housing_model' --local_timestamp_file='fluid_time_file' \
+                    --local_tmp_path='___tmp' --unpacked_filename='uci_housing_model' \
+                    --interval='1' >/dev/null &
+            sleep 10
+            if [ ! -f "local_path/uci_housing_model/fluid_time_file" ]; then
+                echo "local_path/uci_housing_model/fluid_time_file not exist."
+                exit 1
+            fi
+            ps -ef | grep "monitor" | grep -v grep | awk '{print $2}' | xargs kill
+            rm -rf remote_path/*
+            rm -rf local_path/*
+
+            # type: general
+            # remote_path: /tmp_dir
+            # remote_model_name: uci_housing_model
+            # local_tmp_path: ___tmp
+            # local_path: local_path
+            mkdir -p remote_path/tmp_dir && cd remote_path/tmp_dir # pwd: /Serving/_monitor_test/remote_path/tmp_dir
+            wget --no-check-certificate https://paddle-serving.bj.bcebos.com/uci_housing.tar.gz
+            tar -xzf uci_housing.tar.gz
+            touch donefile
+            cd ../.. # pwd: /Serving/_monitor_test
+            mkdir -p local_path/uci_housing_model
+            python -m paddle_serving_server_gpu.monitor \
+                    --type='general' --general_host='ftp://127.0.0.1:8000' \
+                    --remote_path='/tmp_dir' --remote_model_name='uci_housing_model' \
+                    --remote_donefile_name='donefile' --local_path='local_path' \
+                    --local_model_name='uci_housing_model' --local_timestamp_file='fluid_time_file' \
+                    --local_tmp_path='___tmp' --interval='1' >/dev/null &
+            sleep 10
+            if [ ! -f "local_path/uci_housing_model/fluid_time_file" ]; then
+                echo "local_path/uci_housing_model/fluid_time_file not exist."
+                exit 1
+            fi
+            ps -ef | grep "monitor" | grep -v grep | awk '{print $2}' | xargs kill
+            rm -rf remote_path/*
+            rm -rf local_path/*
+
+            ps -ef | grep "pyftpdlib" | grep -v grep | awk '{print $2}' | xargs kill
+            ;;
+        *)
+            echo "error type"
+            exit 1
+            ;;
+    esac
+    cd .. # pwd: /Serving
+    rm -rf _monitor_test
+    echo "test monitor $TYPE finished as expected."
+}
+
 function main() {
     local TYPE=$1 # pwd: /
     init # pwd: /Serving
@@ -415,6 +666,7 @@ function main() {
     build_server $TYPE # pwd: /Serving
     build_app $TYPE # pwd: /Serving
     python_run_test $TYPE # pwd: /Serving
+    monitor_test $TYPE # pwd: /Serving
     echo "serving $TYPE part finished as expected."
 }