run python tests in virtualenv

When using system python, "pip install" needs root previlege. Using virtualenv to avoid this.

cmake/external/python.cmake

 # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,7 +19,7 @@ FIND_PACKAGE(PythonInterp 2.7)
 FIND_PACKAGE(PythonLibs 2.7)
 
 SET(py_env "")
-
+SET(USE_VIRTUALENV_FOR_TEST 1)
 IF(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND)
     find_python_module(pip REQUIRED)
     find_python_module(numpy REQUIRED)

cmake/util.cmake

@@ -144,3 +144,13 @@ function(create_resources res_file output_file)
     COMMAND python ARGS ${PROJ_ROOT}/cmake/make_resource.py ${res_file} ${output_file}
     DEPENDS ${res_file} ${PROJ_ROOT}/cmake/make_resource.py)
 endfunction()
+
+
+# Create a python unittest using run_python_tests.sh,
+# which takes care of making correct running environment
+function(add_python_test TEST_NAME)
+    add_test(NAME ${TEST_NAME}
+        COMMAND bash ${PROJ_ROOT}/paddle/scripts/run_python_tests.sh
+        ${USE_VIRTUALENV_FOR_TEST} ${PYTHON_EXECUTABLE} ${ARGN}
+        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+endfunction()

paddle/api/test/CMakeLists.txt

-add_test(NAME test_swig_api
-    COMMAND bash ${PROJ_ROOT}/paddle/api/test/run_tests.sh ${PYTHON_EXECUTABLE})
+add_python_test(test_swig_api
+    testArguments.py testGradientMachine.py testMatrix.py testVector.py testTrain.py testTrainer.py)

paddle/api/test/run_tests.sh → paddle/scripts/run_python_tests.sh

@@ -18,19 +18,29 @@ pushd `dirname $0` > /dev/null
 SCRIPTPATH=$PWD
 popd > /dev/null
 
-cd $SCRIPTPATH
+USE_VIRTUALENV_FOR_TEST=$1; shift
+PYTHON=$1; shift
 
-$1 -m pip install ../../dist/*.whl
+if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then
+   rm -rf .test_env
+   virtualenv .test_env
+   source .test_env/bin/activate
+   PYTHON=python
+fi
+$PYTHON -m pip install $SCRIPTPATH/../dist/*.whl requests matplotlib IPython
 
-test_list="testArguments.py testGradientMachine.py testMatrix.py  testVector.py testTrain.py testTrainer.py"
+export PYTHONPATH=$SCRIPTPATH/../../python/
 
-export PYTHONPATH=$PWD/../../../python/
-
-for fn in $test_list
+for fn in "$@"
 do
   echo "test $fn"
-  $1 $fn
+  $PYTHON $fn
   if [ $? -ne 0 ]; then
     exit 1
   fi
 done
+
+if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then
+    deactivate
+    rm -rf .test_env
+fi

python/paddle/v2/plot/tests/CMakeLists.txt

-add_test(NAME test_ploter
-  COMMAND bash ${PROJ_ROOT}/python/paddle/v2/plot/tests/run_tests.sh
-  ${PYTHON_EXECUTABLE})
+add_python_test(test_ploter test_ploter.py)

python/paddle/v2/plot/tests/run_tests.sh

-#!/bin/bash
-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-pushd `dirname $0` > /dev/null
-SCRIPTPATH=$PWD
-popd > /dev/null
-
-cd $SCRIPTPATH
-$1 -m pip install ../../../../../paddle/dist/*.whl
-
-export DISABLE_PLOT="True"
-test_list="test_ploter.py"
-
-export PYTHONPATH=$PWD/../../../../../python/
-
-for fn in $test_list
-do
-  echo "test $fn"
-  $1 $fn
-  if [ $? -ne 0 ]; then
-    exit 1
-  fi
-done

python/paddle/v2/reader/tests/CMakeLists.txt

-add_test(NAME reader_tests
-  COMMAND bash ${PROJ_ROOT}/python/paddle/v2/reader/tests/run_tests.sh
-  ${PYTHON_EXECUTABLE})
+add_python_test(reader_tests creator_test.py decorator_test.py)

python/paddle/v2/reader/tests/run_tests.sh

-#!/bin/bash
-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-pushd `dirname $0` > /dev/null
-SCRIPTPATH=$PWD
-popd > /dev/null
-
-cd $SCRIPTPATH
-$1 -m pip install ../../../../../paddle/dist/*.whl
-
-test_list="creator_test.py decorator_test.py"
-
-export PYTHONPATH=$PWD/../../../../../python/
-
-for fn in $test_list
-do
-  echo "test $fn"
-  $1 $fn
-  if [ $? -ne 0 ]; then
-    exit 1
-  fi
-done

python/paddle/v2/tests/CMakeLists.txt

-add_test(NAME test_v2_api
-        COMMAND bash ${PROJ_ROOT}/python/paddle/v2/tests/run_tests.sh ${PYTHON_EXECUTABLE})
-
-add_test(NAME test_v2_layer
-        COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
-        ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/test_layer.py
-        WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle)
-
-add_test(NAME test_v2_rnn_layer
-        COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
-        ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/test_rnn_layer.py)
-
-add_test(NAME test_topology
-        COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
-        ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/test_topology.py
-        WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle)
+add_python_test(test_v2_api test_data_feeder.py test_parameters.py test_layer.py test_rnn_layer.py test_topology.py)

python/paddle/v2/tests/run_tests.sh

-#!/bin/bash
-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-pushd `dirname $0` > /dev/null
-SCRIPTPATH=$PWD
-popd > /dev/null
-
-cd $SCRIPTPATH
-
-$1 -m pip install ../../../../paddle/dist/*.whl
-
-test_list="test_data_feeder.py test_parameters.py"
-
-export PYTHONPATH=$PWD/../../../../python/
-
-for fn in $test_list
-do
-  echo "test $fn"
-  $1 $fn
-  if [ $? -ne 0 ]; then
-    exit 1
-  fi
-done