From 6a7feb35e5980f91a784e8fe9dc08d2ad8f6bd3c Mon Sep 17 00:00:00 2001
From: xuwei06 <xuwei06@baidu.com>
Date: Thu, 4 May 2017 16:53:19 -0700
Subject: [PATCH] run python tests in virtualenv

When using system python, "pip install" needs root previlege. Using virtualenv to avoid this.
---
 cmake/external/python.cmake                   |  8 ++---
 cmake/util.cmake                              | 10 ++++++
 paddle/api/test/CMakeLists.txt                |  4 +--
 .../run_python_tests.sh}                      | 24 +++++++++----
 python/paddle/v2/plot/tests/CMakeLists.txt    |  4 +--
 python/paddle/v2/plot/tests/run_tests.sh      | 36 -------------------
 python/paddle/v2/reader/tests/CMakeLists.txt  |  4 +--
 python/paddle/v2/reader/tests/run_tests.sh    | 35 ------------------
 python/paddle/v2/tests/CMakeLists.txt         | 17 +--------
 python/paddle/v2/tests/run_tests.sh           | 36 -------------------
 10 files changed, 36 insertions(+), 142 deletions(-)
 rename paddle/{api/test/run_tests.sh => scripts/run_python_tests.sh} (64%)
 delete mode 100755 python/paddle/v2/plot/tests/run_tests.sh
 delete mode 100755 python/paddle/v2/reader/tests/run_tests.sh
 delete mode 100755 python/paddle/v2/tests/run_tests.sh

diff --git a/cmake/external/python.cmake b/cmake/external/python.cmake
index 9fd3afd0998..fc66d6b2154 100644
--- a/cmake/external/python.cmake
+++ b/cmake/external/python.cmake
@@ -1,11 +1,11 @@
 # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,7 +19,7 @@ FIND_PACKAGE(PythonInterp 2.7)
 FIND_PACKAGE(PythonLibs 2.7)
 
 SET(py_env "")
-
+SET(USE_VIRTUALENV_FOR_TEST 1)
 IF(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND)
     find_python_module(pip REQUIRED)
     find_python_module(numpy REQUIRED)
diff --git a/cmake/util.cmake b/cmake/util.cmake
index be4c591da84..b828eef322b 100644
--- a/cmake/util.cmake
+++ b/cmake/util.cmake
@@ -144,3 +144,13 @@ function(create_resources res_file output_file)
     COMMAND python ARGS ${PROJ_ROOT}/cmake/make_resource.py ${res_file} ${output_file}
     DEPENDS ${res_file} ${PROJ_ROOT}/cmake/make_resource.py)
 endfunction()
+
+
+# Create a python unittest using run_python_tests.sh,
+# which takes care of making correct running environment
+function(add_python_test TEST_NAME)
+    add_test(NAME ${TEST_NAME}
+        COMMAND bash ${PROJ_ROOT}/paddle/scripts/run_python_tests.sh
+        ${USE_VIRTUALENV_FOR_TEST} ${PYTHON_EXECUTABLE} ${ARGN}
+        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+endfunction()
diff --git a/paddle/api/test/CMakeLists.txt b/paddle/api/test/CMakeLists.txt
index a2fa623c800..f3b1c2c4d43 100644
--- a/paddle/api/test/CMakeLists.txt
+++ b/paddle/api/test/CMakeLists.txt
@@ -1,2 +1,2 @@
-add_test(NAME test_swig_api
-    COMMAND bash ${PROJ_ROOT}/paddle/api/test/run_tests.sh ${PYTHON_EXECUTABLE})
+add_python_test(test_swig_api
+    testArguments.py testGradientMachine.py testMatrix.py testVector.py testTrain.py testTrainer.py)
diff --git a/paddle/api/test/run_tests.sh b/paddle/scripts/run_python_tests.sh
similarity index 64%
rename from paddle/api/test/run_tests.sh
rename to paddle/scripts/run_python_tests.sh
index bcf06afa86a..9a23c10d14e 100755
--- a/paddle/api/test/run_tests.sh
+++ b/paddle/scripts/run_python_tests.sh
@@ -18,19 +18,29 @@ pushd `dirname $0` > /dev/null
 SCRIPTPATH=$PWD
 popd > /dev/null
 
-cd $SCRIPTPATH
+USE_VIRTUALENV_FOR_TEST=$1; shift
+PYTHON=$1; shift
 
-$1 -m pip install ../../dist/*.whl
+if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then
+   rm -rf .test_env
+   virtualenv .test_env
+   source .test_env/bin/activate
+   PYTHON=python
+fi
+$PYTHON -m pip install $SCRIPTPATH/../dist/*.whl requests matplotlib IPython
 
-test_list="testArguments.py testGradientMachine.py testMatrix.py  testVector.py testTrain.py testTrainer.py"
+export PYTHONPATH=$SCRIPTPATH/../../python/
 
-export PYTHONPATH=$PWD/../../../python/
-
-for fn in $test_list
+for fn in "$@"
 do
   echo "test $fn"
-  $1 $fn
+  $PYTHON $fn
   if [ $? -ne 0 ]; then
     exit 1
   fi
 done
+
+if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then
+    deactivate
+    rm -rf .test_env
+fi
diff --git a/python/paddle/v2/plot/tests/CMakeLists.txt b/python/paddle/v2/plot/tests/CMakeLists.txt
index da550a178ce..b1132f13173 100644
--- a/python/paddle/v2/plot/tests/CMakeLists.txt
+++ b/python/paddle/v2/plot/tests/CMakeLists.txt
@@ -1,3 +1 @@
-add_test(NAME test_ploter
-  COMMAND bash ${PROJ_ROOT}/python/paddle/v2/plot/tests/run_tests.sh
-  ${PYTHON_EXECUTABLE})
+add_python_test(test_ploter test_ploter.py)
diff --git a/python/paddle/v2/plot/tests/run_tests.sh b/python/paddle/v2/plot/tests/run_tests.sh
deleted file mode 100755
index 9c1a4a71ce4..00000000000
--- a/python/paddle/v2/plot/tests/run_tests.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-pushd `dirname $0` > /dev/null
-SCRIPTPATH=$PWD
-popd > /dev/null
-
-cd $SCRIPTPATH
-$1 -m pip install ../../../../../paddle/dist/*.whl
-
-export DISABLE_PLOT="True"
-test_list="test_ploter.py"
-
-export PYTHONPATH=$PWD/../../../../../python/
-
-for fn in $test_list
-do
-  echo "test $fn"
-  $1 $fn
-  if [ $? -ne 0 ]; then
-    exit 1
-  fi
-done
diff --git a/python/paddle/v2/reader/tests/CMakeLists.txt b/python/paddle/v2/reader/tests/CMakeLists.txt
index a51f700406b..6a1d337b232 100644
--- a/python/paddle/v2/reader/tests/CMakeLists.txt
+++ b/python/paddle/v2/reader/tests/CMakeLists.txt
@@ -1,3 +1 @@
-add_test(NAME reader_tests
-  COMMAND bash ${PROJ_ROOT}/python/paddle/v2/reader/tests/run_tests.sh
-  ${PYTHON_EXECUTABLE})
+add_python_test(reader_tests creator_test.py decorator_test.py)
diff --git a/python/paddle/v2/reader/tests/run_tests.sh b/python/paddle/v2/reader/tests/run_tests.sh
deleted file mode 100755
index a544a563602..00000000000
--- a/python/paddle/v2/reader/tests/run_tests.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-pushd `dirname $0` > /dev/null
-SCRIPTPATH=$PWD
-popd > /dev/null
-
-cd $SCRIPTPATH
-$1 -m pip install ../../../../../paddle/dist/*.whl
-
-test_list="creator_test.py decorator_test.py"
-
-export PYTHONPATH=$PWD/../../../../../python/
-
-for fn in $test_list
-do
-  echo "test $fn"
-  $1 $fn
-  if [ $? -ne 0 ]; then
-    exit 1
-  fi
-done
diff --git a/python/paddle/v2/tests/CMakeLists.txt b/python/paddle/v2/tests/CMakeLists.txt
index 572deaff356..5554a37df03 100644
--- a/python/paddle/v2/tests/CMakeLists.txt
+++ b/python/paddle/v2/tests/CMakeLists.txt
@@ -1,16 +1 @@
-add_test(NAME test_v2_api
-        COMMAND bash ${PROJ_ROOT}/python/paddle/v2/tests/run_tests.sh ${PYTHON_EXECUTABLE})
-
-add_test(NAME test_v2_layer
-        COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
-        ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/test_layer.py
-        WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle)
-
-add_test(NAME test_v2_rnn_layer
-        COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
-        ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/test_rnn_layer.py)
-
-add_test(NAME test_topology
-        COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
-        ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/test_topology.py
-        WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle)
+add_python_test(test_v2_api test_data_feeder.py test_parameters.py test_layer.py test_rnn_layer.py test_topology.py)
diff --git a/python/paddle/v2/tests/run_tests.sh b/python/paddle/v2/tests/run_tests.sh
deleted file mode 100755
index dda1b1bd222..00000000000
--- a/python/paddle/v2/tests/run_tests.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-pushd `dirname $0` > /dev/null
-SCRIPTPATH=$PWD
-popd > /dev/null
-
-cd $SCRIPTPATH
-
-$1 -m pip install ../../../../paddle/dist/*.whl
-
-test_list="test_data_feeder.py test_parameters.py"
-
-export PYTHONPATH=$PWD/../../../../python/
-
-for fn in $test_list
-do
-  echo "test $fn"
-  $1 $fn
-  if [ $? -ne 0 ]; then
-    exit 1
-  fi
-done
-- 
GitLab