Merge branch 'feature/fast_python_unittest' into feature/change_op_creation

CMakeLists.txt

@@ -55,6 +55,7 @@ option(WITH_C_API       "Compile PaddlePaddle with C-API(Prediction)"   OFF)
 option(WITH_GOLANG      "Compile PaddlePaddle with GOLANG"              OFF)
 option(GLIDE_INSTALL    "Download and install go dependencies "         ON)
 option(USE_NNPACK       "Compile PaddlePaddle with NNPACK library"      OFF)
+option(UNITTEST_USE_VIRTUALENV "Python unittest with virtualenv"        ON)
 
 # CMAKE_BUILD_TYPE
 if(NOT CMAKE_BUILD_TYPE)

cmake/external/python.cmake

@@ -24,7 +24,6 @@ IF(WITH_PYTHON)
 ENDIF(WITH_PYTHON)
 
 SET(py_env "")
-SET(USE_VIRTUALENV_FOR_TEST 1)
 IF(PYTHONINTERP_FOUND)
     find_python_module(pip REQUIRED)
     find_python_module(numpy REQUIRED)

cmake/util.cmake

@@ -150,9 +150,19 @@ endfunction()
 # Create a python unittest using run_python_tests.sh,
 # which takes care of making correct running environment
 function(add_python_test TEST_NAME)
+    if (UNITTEST_USE_VIRTUALENV)
         add_test(NAME ${TEST_NAME}
             COMMAND env PADDLE_PACKAGE_DIR=${PADDLE_PYTHON_PACKAGE_DIR}
-        bash ${PROJ_ROOT}/paddle/scripts/run_python_tests.sh
-        ${USE_VIRTUALENV_FOR_TEST} ${PYTHON_EXECUTABLE} ${ARGN}
+            bash ${PROJ_ROOT}/paddle/scripts/run_python_tests.sh ${ARGN}
             WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+    else()
+        foreach(arg ${ARGN})
+            get_filename_component(py_fn ${arg} NAME_WE)
+            set(TRG_NAME ${TEST_NAME}_${py_fn})
+            add_test(NAME ${TRG_NAME}
+                    COMMAND env PYTHONPATH=${PADDLE_PYTHON_PACKAGE_DIR}
+                    python2 ${arg}
+                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+        endforeach()
+    endif()
 endfunction()

paddle/.set_python_path.sh

@@ -21,22 +21,15 @@
 # 
 # It same as PYTHONPATH=${YOUR_PYTHON_PATH}:$PYTHONPATH {exec...}
 #
-
-if ! python -c "import paddle" >/dev/null 2>/dev/null; then
-  PYPATH=""
-  set -x
-  while getopts "d:" opt; do
+PYPATH=""
+set -x
+while getopts "d:" opt; do
   case $opt in
     d)
       PYPATH=$OPTARG
       ;;
   esac
-  done
-  shift $(($OPTIND - 1))
-  export PYTHONPATH=$PYPATH:$PYTHONPATH
-  $@
-else
-  echo "paddle package is already in your PYTHONPATH. But unittest need a clean environment."
-  echo "Please uninstall paddle package before start unittest. Try to 'pip uninstall paddle'"
-  exit 1
-fi
+done
+shift $(($OPTIND - 1))
+export PYTHONPATH=$PYPATH:$PYTHONPATH
+$@

paddle/scripts/docker/build.sh

@@ -49,29 +49,27 @@ cmake .. \
       -DCUDNN_ROOT=/usr/ \
       -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} \
       -DWITH_TESTING=${WITH_TESTING:-OFF} \
-      -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+      -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+      -DUNITTEST_USE_VIRTUALENV=OFF
 
 cat <<EOF
-========================================
-Building in /paddle/build ...
+============================================
+Building and installing in /paddle/build ...
    Build unit tests: ${WITH_TESTING:-OFF}
-========================================
+============================================
 EOF
-make -j `nproc`
-if [ ${WITH_TESTING:-OFF} == "ON" ] && [ ${RUN_TEST:-OFF} == "ON" ] ; then
-    pip uninstall -y py-paddle paddle || true
-    ctest --output-on-failure
-fi
-
+make install -j `nproc`
+pip install /usr/local/opt/paddle/share/wheels/*.whl
+paddle version
 
+if [ ${WITH_TESTING:-OFF} == "ON" ] && [ ${RUN_TEST:-OFF} == "ON" ] ; then
 cat <<EOF
 ========================================
-Installing ...
+Running unit tests ...
 ========================================
 EOF
-make install -j `nproc`
-pip install /usr/local/opt/paddle/share/wheels/*.whl
-paddle version
+    ctest --output-on-failure
+fi
 
 
 # To build documentation, we need to run cmake again after installing

paddle/scripts/run_python_tests.sh

@@ -18,17 +18,12 @@ pushd `dirname $0` > /dev/null
 SCRIPTPATH=$PWD
 popd > /dev/null
 
-USE_VIRTUALENV_FOR_TEST=$1; shift
-PYTHON=$1; shift
-
-if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then
-   rm -rf .test_env
-   virtualenv .test_env
-   unset PYTHONHOME
-   unset PYTHONPATH
-   source .test_env/bin/activate
-   PYTHON=python
-fi
+rm -rf .test_env
+virtualenv .test_env
+unset PYTHONHOME
+unset PYTHONPATH
+source .test_env/bin/activate
+PYTHON=python
 
 $PYTHON -m pip install $SCRIPTPATH/../dist/*.whl
 
@@ -49,7 +44,5 @@ do
   fi
 done
 
-if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then
-    deactivate
-    rm -rf .test_env
-fi
+deactivate
+rm -rf .test_env

paddle/setup.py.in

@@ -22,7 +22,7 @@ setup(name="py_paddle",
       package_data={'py_paddle':['*.py','_swig_paddle.so']},
       install_requires = [
         'nltk>=3.2.2',
-        'numpy>=1.8.0',      # The numpy is required.
+        'numpy>=1.12.0',      # The numpy is required.
         'protobuf==${PROTOBUF_VERSION}'    # The paddle protobuf version
       ],
       url='http://www.paddlepaddle.org/',

python/setup.py.in

@@ -14,7 +14,7 @@ packages=['paddle',
           'paddle.v2.framework.proto']
 
 setup_requires=["requests",
-                "numpy",
+                "numpy>=1.12",
                 "protobuf==3.1",
                 "recordio",
                 "matplotlib",