Merge pull request #3210 from reyoung/feature/fast_python_unittest

Faster Python unittest

Dockerfile

@@ -27,13 +27,16 @@ RUN apt-get update && \
     git python-pip python-dev openssh-server bison  \
     wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \
     curl sed grep graphviz libjpeg-dev zlib1g-dev  \
-    python-numpy python-matplotlib gcc-4.8 g++-4.8 \
+    python-matplotlib gcc-4.8 g++-4.8 \
     automake locales clang-format-3.8 swig doxygen cmake  \
     liblapack-dev liblapacke-dev libboost-dev \
     clang-3.8 llvm-3.8 libclang-3.8-dev \
     net-tools && \
     apt-get clean -y
 
+# paddle is using numpy.flip, which is introduced since 1.12.0
+RUN pip --no-cache-dir install 'numpy>=1.12.0'
+
 # Install Go and glide
 RUN wget -O go.tgz https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz && \
     tar -C /usr/local -xzf go.tgz && \

cmake/external/python.cmake

@@ -24,7 +24,6 @@ IF(WITH_PYTHON)
 ENDIF(WITH_PYTHON)
 
 SET(py_env "")
-SET(USE_VIRTUALENV_FOR_TEST 1)
 IF(PYTHONINTERP_FOUND)
     find_python_module(pip REQUIRED)
     find_python_module(numpy REQUIRED)

cmake/util.cmake

@@ -149,9 +149,12 @@ endfunction()
 # Create a python unittest using run_python_tests.sh,
 # which takes care of making correct running environment
 function(add_python_test TEST_NAME)
-  add_test(NAME ${TEST_NAME}
-        COMMAND env PADDLE_PACKAGE_DIR=${PADDLE_PYTHON_PACKAGE_DIR}
-        bash ${PROJ_ROOT}/paddle/scripts/run_python_tests.sh
-        ${USE_VIRTUALENV_FOR_TEST} ${PYTHON_EXECUTABLE} ${ARGN}
-        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+    foreach(arg ${ARGN})
+        get_filename_component(py_fn ${arg} NAME_WE)
+        set(TRG_NAME ${TEST_NAME}_${py_fn})
+        add_test(NAME ${TRG_NAME}
+                COMMAND env PYTHONPATH=${PADDLE_PYTHON_PACKAGE_DIR}
+                python2 ${arg}
+                WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+    endforeach()
 endfunction()

paddle/.set_python_path.sh

@@ -21,22 +21,15 @@
 # 
 # It same as PYTHONPATH=${YOUR_PYTHON_PATH}:$PYTHONPATH {exec...}
 #
-
-if ! python -c "import paddle" >/dev/null 2>/dev/null; then
-  PYPATH=""
-  set -x
-  while getopts "d:" opt; do
-    case $opt in
-      d)
-        PYPATH=$OPTARG
-        ;;
-    esac
-  done
-  shift $(($OPTIND - 1))
-  export PYTHONPATH=$PYPATH:$PYTHONPATH
-  $@
-else
-  echo "paddle package is already in your PYTHONPATH. But unittest need a clean environment."
-  echo "Please uninstall paddle package before start unittest. Try to 'pip uninstall paddle'"
-  exit 1
-fi
+PYPATH=""
+set -x
+while getopts "d:" opt; do
+  case $opt in
+    d)
+      PYPATH=$OPTARG
+      ;;
+  esac
+done
+shift $(($OPTIND - 1))
+export PYTHONPATH=$PYPATH:$PYTHONPATH
+$@

paddle/scripts/docker/build.sh

@@ -39,6 +39,10 @@ Configuring cmake in /paddle/build ...
       -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
 ========================================
 EOF
+
+# Disable UNITTEST_USE_VIRTUALENV in docker because
+# docker environment is fully controlled by this script.
+# See /Paddle/CMakeLists.txt, UNITTEST_USE_VIRTUALENV option.
 cmake .. \
       -DCMAKE_BUILD_TYPE=Release \
       -DWITH_DOC=OFF \
@@ -52,26 +56,23 @@ cmake .. \
       -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
 
 cat <<EOF
-========================================
-Building in /paddle/build ...
+============================================
+Building and installing in /paddle/build ...
    Build unit tests: ${WITH_TESTING:-OFF}
-========================================
+============================================
 EOF
-make -j `nproc`
-if [ ${WITH_TESTING:-OFF} == "ON" ] && [ ${RUN_TEST:-OFF} == "ON" ] ; then
-    pip uninstall -y py-paddle paddle || true
-    ctest --output-on-failure
-fi
-
+make install -j `nproc`
+pip install /usr/local/opt/paddle/share/wheels/*.whl
+paddle version
 
+if [ ${WITH_TESTING:-OFF} == "ON" ] && [ ${RUN_TEST:-OFF} == "ON" ] ; then
 cat <<EOF
 ========================================
-Installing ...
+Running unit tests ...
 ========================================
 EOF
-make install -j `nproc`
-pip install /usr/local/opt/paddle/share/wheels/*.whl
-paddle version
+    ctest --output-on-failure
+fi
 
 
 # To build documentation, we need to run cmake again after installing
@@ -122,9 +123,15 @@ cat <<EOF
 Generating .deb package ...
 ========================================
 EOF
+set +e
 cpack -D CPACK_GENERATOR='DEB' -j `nproc` ..
-
-
+err_code=$?
+if [ ${err_code} -ne 0 ]; then
+  # cat error logs if cpack failed.
+  cat /paddle/build/_CPack_Packages/Linux/DEB/PreinstallOutput.log
+  exit ${err_code}
+fi
+set -e
 cat <<EOF
 ========================================
 Generate /paddle/build/Dockerfile ...

paddle/scripts/run_python_tests.sh

-#!/bin/bash
-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-pushd `dirname $0` > /dev/null
-SCRIPTPATH=$PWD
-popd > /dev/null
-
-USE_VIRTUALENV_FOR_TEST=$1; shift
-PYTHON=$1; shift
-
-if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then
-   rm -rf .test_env
-   virtualenv .test_env
-   unset PYTHONHOME
-   unset PYTHONPATH
-   source .test_env/bin/activate
-   PYTHON=python
-fi
-
-$PYTHON -m pip install $SCRIPTPATH/../dist/*.whl
-
-if [ "X${PADDLE_PACKAGE_DIR}" != "X" ]; then
-   $PYTHON -m pip install ${PADDLE_PACKAGE_DIR}/*.whl
-else
-   export PYTHONPATH=$SCRIPTPATH/../../python/
-fi
-
-$PYTHON -m pip install ipython==5.3
-
-for fn in "$@"
-do
-  echo "test $fn"
-  $PYTHON $fn
-  if [ $? -ne 0 ]; then
-    exit 1
-  fi
-done
-
-if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then
-    deactivate
-    rm -rf .test_env
-fi

paddle/setup.py.in

@@ -22,7 +22,9 @@ setup(name="py_paddle",
       package_data={'py_paddle':['*.py','_swig_paddle.so']},
       install_requires = [
         'nltk>=3.2.2',
-        'numpy>=1.8.0',      # The numpy is required.
+        # We use `numpy.flip` in `test_image.py`.
+        # `numpy.flip` is introduced in `1.12.0`
+        'numpy>=1.12.0',      # The numpy is required.
         'protobuf==${PROTOBUF_VERSION}'    # The paddle protobuf version
       ],
       url='http://www.paddlepaddle.org/',

python/setup.py.in

@@ -14,7 +14,7 @@ packages=['paddle',
           'paddle.v2.framework.proto']
 
 setup_requires=["requests",
-                "numpy",
+                "numpy>=1.12",
                 "protobuf==3.1",
                 "recordio",
                 "matplotlib",