diff --git a/python/paddle/reader/decorator.py b/python/paddle/reader/decorator.py index ff09f4c562aeb8216dea6e1d40b9492c257aeab4..aadfb3f49ed61367b9502e1a00ad5b9c027a32b7 100644 --- a/python/paddle/reader/decorator.py +++ b/python/paddle/reader/decorator.py @@ -32,6 +32,21 @@ import random import zlib import paddle.compat as cpt +# On macOS, the 'spawn' start method is now the default in Python3.8 multiprocessing, +# Paddle is currently unable to solve this, so forces the process to start using +# the 'fork' start method. +# +# TODO: This solution is not good, because the fork start method could lead to +# crashes of the subprocess. Figure out how to make 'spawn' work. +# +# For more details, please refer to +# https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods +# https://bugs.python.org/issue33725 +if sys.version_info >= (3, 8): + fork_context = multiprocessing.get_context('fork') +else: + fork_context = multiprocessing + def cache(reader): """ @@ -560,9 +575,9 @@ def multiprocess_reader(readers, use_pipe=True, queue_size=1000): six.reraise(*sys.exc_info()) def queue_reader(): - queue = multiprocessing.Queue(queue_size) + queue = fork_context.Queue(queue_size) for reader in readers: - p = multiprocessing.Process( + p = fork_context.Process( target=_read_into_queue, args=(reader, queue)) p.start() @@ -593,9 +608,9 @@ def multiprocess_reader(readers, use_pipe=True, queue_size=1000): def pipe_reader(): conns = [] for reader in readers: - parent_conn, child_conn = multiprocessing.Pipe() + parent_conn, child_conn = fork_context.Pipe() conns.append(parent_conn) - p = multiprocessing.Process( + p = fork_context.Process( target=_read_into_pipe, args=(reader, child_conn)) p.start()