diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 42960c0cfb56fc27cb0b6924f2ed29150344bf89..87a51f6b1fec770c7cce56d51c29a84d399ae9eb 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -121,6 +121,18 @@ function cmake_base() { else exit 1 fi + elif [ "$1" == "cp38-cp38" ]; then + if [ -d "/Library/Frameworks/Python.framework/Versions/3.8" ]; then + export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.8/lib/ + export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.8/lib/ + export PATH=/Library/Frameworks/Python.framework/Versions/3.8/bin/:${PATH} + PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.8/bin/python3 + -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.8/include/python3.8/ + -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.8/lib/libpython3.8.dylib" + pip3.8 install --user -r ${PADDLE_ROOT}/python/requirements.txt + else + exit 1 + fi fi # delete `gym` to avoid modifying requirements.txt in *.whl sed -i .bak "/^gym$/d" ${PADDLE_ROOT}/python/requirements.txt @@ -176,6 +188,13 @@ function cmake_base() { -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.7.0/include/python3.7m -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.7.0/lib/libpython3.so" pip3.7 install -r ${PADDLE_ROOT}/python/requirements.txt + elif [ "$1" == "cp38-cp38" ]; then + export LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH} + export PATH=/opt/_internal/cpython-3.8.0/bin/:${PATH} + export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.8.0/bin/python3.8 + -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.8.0/include/python3.8 + -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.8.0/lib/libpython3.so" + pip3.8 install -r ${PADDLE_ROOT}/python/requirements.txt fi else pip install -r ${PADDLE_ROOT}/python/requirements.txt @@ -506,6 +525,8 @@ EOF pip3.6 uninstall -y paddlepaddle elif [ "$1" == "cp37-cp37m" ]; then pip3.7 uninstall -y paddlepaddle + elif [ "$1" == "cp38-cp38" ]; then + pip3.8 uninstall -y paddlepaddle fi set -ex @@ -519,6 +540,8 @@ EOF pip3.6 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl elif [ "$1" == "cp37-cp37m" ]; then pip3.7 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl + elif [ "$1" == "cp38-cp38" ]; then + pip3.8 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl fi ut_startTime_s=`date +%s` ctest --output-on-failure -j $2 @@ -592,7 +615,7 @@ function generate_api_spec() { awk -F '(' '{print $NF}' $spec_path >${spec_path}.doc awk -F '(' '{$NF="";print $0}' $spec_path >${spec_path}.api - if [ "$1" == "cp35-cp35m" ] || [ "$1" == "cp36-cp36m" ] || [ "$1" == "cp37-cp37m" ]; then + if [ "$1" == "cp35-cp35m" ] || [ "$1" == "cp36-cp36m" ] || [ "$1" == "cp37-cp37m" ] || [ "$1" == "cp38-cp38" ]; then # Use sed to make python2 and python3 sepc keeps the same sed -i 's/arg0: str/arg0: unicode/g' $spec_path sed -i "s/\(.*Transpiler.*\).__init__ (ArgSpec(args=\['self'].*/\1.__init__ /g" $spec_path @@ -1075,21 +1098,25 @@ EOF ref_paddle35=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp35-cp35m-linux_x86_64.whl ref_paddle36=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp36-cp36m-linux_x86_64.whl ref_paddle37=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp37-cp37m-linux_x86_64.whl + ref_paddle38=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp38-cp38-linux_x86_64.whl ref_paddle2_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp27-cp27mu-linux_x86_64.whl ref_paddle35_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp35-cp35m-linux_x86_64.whl ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp36-cp36m-linux_x86_64.whl ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp37-cp37m-linux_x86_64.whl + ref_paddle38_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp38-cp38-linux_x86_64.whl if [[ ${PADDLE_BRANCH} != "0.0.0" && ${WITH_MKL} == "ON" && ${WITH_GPU} == "ON" ]]; then ref_paddle2=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp27-cp27mu-linux_x86_64.whl ref_paddle35=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp35-cp35m-linux_x86_64.whl ref_paddle36=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp36-cp36m-linux_x86_64.whl ref_paddle37=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp37-cp37m-linux_x86_64.whl + ref_paddle38=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp38-cp38-linux_x86_64.whl ref_paddle2_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp27-cp27mu-linux_x86_64.whl ref_paddle35_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp35-cp35m-linux_x86_64.whl ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp36-cp36m-linux_x86_64.whl ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp37-cp37m-linux_x86_64.whl + ref_paddle38_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp38-cp38-linux_x86_64.whl fi #ref_paddle2_mv1="" @@ -1194,6 +1221,22 @@ EOF apt-get clean -y && \ rm -f ${ref_paddle37} && \ ldconfig +EOF + cat >> ${PADDLE_ROOT}/build/Dockerfile < /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && cd ../ && rm Python-3.8.0.tgz + RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && ldconfig && \ + pip3.8 install opencv-python && wget ${ref_web}/${ref_paddle38} && pip3.8 install ${ref_paddle38_whl}; apt-get install -f -y && \ + apt-get clean -y && \ + rm -f ${ref_paddle38} && \ + ldconfig EOF cat >> ${PADDLE_ROOT}/build/Dockerfile <= (3, 8): + os.add_dll_directory(third_lib_path) except ImportError as e: from .. import compat as cpt diff --git a/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py b/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py index c7476a8a74256d8eb656778c945c96ee0aa88df4..c176ff09e024db90ea5a81bcf2afe18939c4f538 100644 --- a/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py +++ b/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py @@ -16,6 +16,7 @@ from __future__ import print_function import ast import gast +import sys import textwrap import unittest @@ -143,47 +144,60 @@ class TestPythonCompatibility(unittest.TestCase): """ self._check_compatibility(source, target) - def test_with(self): - """ - The fileds `context_expr/optional_vars` of `ast.With` in PY2 - is moved into `ast.With.items.withitem` in PY3. - """ - source = """ - with guard(): - a = 1 - """ - target = """ - with guard_new(): - a = 1 - """ - self._check_compatibility(source, target) - - def test_subscript_Index(self): - source = """ - x = y()[10] - """ - target = """ - x = y()[20] - """ - self._check_compatibility(source, target) - - def test_subscript_Slice(self): - source = """ - x = y()[10:20] - """ - target = """ - x = y()[20:40] - """ - self._check_compatibility(source, target) - - def test_call(self): - source = """ - y = foo(*arg) - """ - target = """ - y = foo(*arg_new) - """ - self._check_compatibility(source, target) + # The 0.3.3 version of gast has a bug in python3.8 that + # would cause the following tests to fail. But this + # problem doesn't affect the use of Paddle's related + # functions, therefore, the following tests would be + # disable in python3.8. + # + # This problem had been fixed and updated to version + # 0.4.1 of gast. + # + # More information please refer to: + # https://github.com/serge-sans-paille/gast/issues/49 + if sys.version_info < (3, 8): + + def test_with(self): + """ + The fileds `context_expr/optional_vars` of `ast.With` in PY2 + is moved into `ast.With.items.withitem` in PY3. + """ + source = """ + with guard(): + a = 1 + """ + target = """ + with guard_new(): + a = 1 + """ + self._check_compatibility(source, target) + + def test_subscript_Index(self): + source = """ + x = y()[10] + """ + target = """ + x = y()[20] + """ + self._check_compatibility(source, target) + + def test_subscript_Slice(self): + source = """ + x = y()[10:20] + """ + target = """ + x = y()[20:40] + """ + self._check_compatibility(source, target) + + def test_call(self): + source = """ + y = foo(*arg) + """ + target = """ + y = foo(*arg_new) + """ + self._check_compatibility(source, target) if __name__ == '__main__': diff --git a/python/paddle/reader/decorator.py b/python/paddle/reader/decorator.py index a81746e4a2c4004999891f3a8cf1bf9233d8572f..d76c877761fb1e4e9649ce8e5436b20b8c6aefb9 100644 --- a/python/paddle/reader/decorator.py +++ b/python/paddle/reader/decorator.py @@ -32,6 +32,21 @@ import random import zlib import paddle.compat as cpt +# On macOS, the 'spawn' start method is now the default in Python3.8 multiprocessing, +# Paddle is currently unable to solve this, so forces the process to start using +# the 'fork' start method. +# +# TODO: This solution is not good, because the fork start method could lead to +# crashes of the subprocess. Figure out how to make 'spawn' work. +# +# For more details, please refer to +# https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods +# https://bugs.python.org/issue33725 +if sys.version_info >= (3, 8) and sys.platform == 'darwin': + fork_context = multiprocessing.get_context('fork') +else: + fork_context = multiprocessing + def cache(reader): """ @@ -556,9 +571,9 @@ def multiprocess_reader(readers, use_pipe=True, queue_size=1000): six.reraise(*sys.exc_info()) def queue_reader(): - queue = multiprocessing.Queue(queue_size) + queue = fork_context.Queue(queue_size) for reader in readers: - p = multiprocessing.Process( + p = fork_context.Process( target=_read_into_queue, args=(reader, queue)) p.start() @@ -589,9 +604,9 @@ def multiprocess_reader(readers, use_pipe=True, queue_size=1000): def pipe_reader(): conns = [] for reader in readers: - parent_conn, child_conn = multiprocessing.Pipe() + parent_conn, child_conn = fork_context.Pipe() conns.append(parent_conn) - p = multiprocessing.Process( + p = fork_context.Process( target=_read_into_pipe, args=(reader, child_conn)) p.start()