Faster Python unittest

* Do not use `virtualenv` in Docker
* Split unittest into many target, make them parallel

CMakeLists.txt

@@ -55,6 +55,7 @@ option(WITH_C_API       "Compile PaddlePaddle with C-API(Prediction)"   OFF)
 option(WITH_GOLANG      "Compile PaddlePaddle with GOLANG"              OFF)
 option(GLIDE_INSTALL    "Download and install go dependencies "         ON)
 option(USE_NNPACK       "Compile PaddlePaddle with NNPACK library"      OFF)
+option(UNITTEST_USE_VIRTUALENV "Python unittest with virtualenv"        ON)
 
 # CMAKE_BUILD_TYPE
 if(NOT CMAKE_BUILD_TYPE)

cmake/external/python.cmake

@@ -24,7 +24,6 @@ IF(WITH_PYTHON)
 ENDIF(WITH_PYTHON)
 
 SET(py_env "")
-SET(USE_VIRTUALENV_FOR_TEST 1)
 IF(PYTHONINTERP_FOUND)
     find_python_module(pip REQUIRED)
     find_python_module(numpy REQUIRED)

cmake/util.cmake

@@ -150,9 +150,19 @@ endfunction()
 # Create a python unittest using run_python_tests.sh,
 # which takes care of making correct running environment
 function(add_python_test TEST_NAME)
+    if (UNITTEST_USE_VIRTUALENV)
         add_test(NAME ${TEST_NAME}
             COMMAND env PADDLE_PACKAGE_DIR=${PADDLE_PYTHON_PACKAGE_DIR}
-        bash ${PROJ_ROOT}/paddle/scripts/run_python_tests.sh
-        ${USE_VIRTUALENV_FOR_TEST} ${PYTHON_EXECUTABLE} ${ARGN}
+            bash ${PROJ_ROOT}/paddle/scripts/run_python_tests.sh ${ARGN}
             WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+    else()
+        foreach(arg ${ARGN})
+            get_filename_component(py_fn ${arg} NAME_WE)
+            set(TRG_NAME ${TEST_NAME}_${py_fn})
+            add_test(NAME ${TRG_NAME}
+                    COMMAND env PYTHONPATH=${PADDLE_PYTHON_PACKAGE_DIR}
+                    python2 ${arg}
+                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+        endforeach()
+    endif()
 endfunction()

paddle/scripts/docker/build.sh

@@ -49,29 +49,27 @@ cmake .. \
       -DCUDNN_ROOT=/usr/ \
       -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} \
       -DWITH_TESTING=${WITH_TESTING:-OFF} \
-      -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+      -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+      -DUNITTEST_USE_VIRTUALENV=OFF
 
 cat <<EOF
-========================================
-Building in /paddle/build ...
+============================================
+Building and installing in /paddle/build ...
    Build unit tests: ${WITH_TESTING:-OFF}
-========================================
+============================================
 EOF
-make -j `nproc`
-if [ ${WITH_TESTING:-OFF} == "ON" ] && [ ${RUN_TEST:-OFF} == "ON" ] ; then
-    pip uninstall -y py-paddle paddle || true
-    ctest --output-on-failure
-fi
-
+make install -j `nproc`
+pip install /usr/local/opt/paddle/share/wheels/*.whl
+paddle version
 
+if [ ${WITH_TESTING:-OFF} == "ON" ] && [ ${RUN_TEST:-OFF} == "ON" ] ; then
 cat <<EOF
 ========================================
-Installing ...
+Running unit tests ...
 ========================================
 EOF
-make install -j `nproc`
-pip install /usr/local/opt/paddle/share/wheels/*.whl
-paddle version
+    ctest --output-on-failure
+fi
 
 
 # To build documentation, we need to run cmake again after installing

paddle/scripts/run_python_tests.sh

@@ -18,17 +18,12 @@ pushd `dirname $0` > /dev/null
 SCRIPTPATH=$PWD
 popd > /dev/null
 
-USE_VIRTUALENV_FOR_TEST=$1; shift
-PYTHON=$1; shift
-
-if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then
-   rm -rf .test_env
-   virtualenv .test_env
-   unset PYTHONHOME
-   unset PYTHONPATH
-   source .test_env/bin/activate
-   PYTHON=python
-fi
+rm -rf .test_env
+virtualenv .test_env
+unset PYTHONHOME
+unset PYTHONPATH
+source .test_env/bin/activate
+PYTHON=python
 
 $PYTHON -m pip install $SCRIPTPATH/../dist/*.whl
 
@@ -49,7 +44,5 @@ do
   fi
 done
 
-if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then
-    deactivate
-    rm -rf .test_env
-fi
+deactivate
+rm -rf .test_env