From ed7a21dea0ddcffb6f7f33ce21c5c368f5c7866b Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Wed, 24 Nov 2021 21:05:28 -0600 Subject: [PATCH] Hot fix for dataloader thread error because of pten (#37520) * hot fix for dataloader thread error * polish comment * fix type in comment, test=document_fix --- .../fluid/dataloader/dataloader_iter.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/dataloader/dataloader_iter.py b/python/paddle/fluid/dataloader/dataloader_iter.py index 70c7b01b05..d8cb3e0918 100644 --- a/python/paddle/fluid/dataloader/dataloader_iter.py +++ b/python/paddle/fluid/dataloader/dataloader_iter.py @@ -24,7 +24,7 @@ import threading import numpy as np import multiprocessing from collections import namedtuple -from paddle.fluid.framework import _set_expected_place, _current_expected_place +from paddle.fluid.framework import _set_expected_place, _current_expected_place, set_flags # NOTE: queue has a different name in python2 and python3 import queue @@ -202,6 +202,22 @@ class _DataLoaderIterSingleProcess(_DataLoaderIterBase): # APIs in this thread. _set_expected_place(legacy_expected_place) + # NOTE(chenweihang): [ Why need to set not to execute pten kernel here? ] + # Now, in order to ensure that the execution performance of the dynamic + # graph mode in pten compatible state does not decline significantly, + # we have adopted the approach of caching a KernelContext globally for + # the dynamic graph tracer to reduce the construction and deconstruction + # overhead of data interfaces such as the compatible state DenseTensor. + # The static graph is each op caches a KernelContext, but the op of + # the dynamic graph will be constructed and destroyed every round of + # execution, so it is impossible to cache KernelContext for each op. + # However, it is not thread-safe if using only one global kernel context in + # dynamic graph. If the pten op of paddle is used in the DataLoader thread, + # it may cause access errors. We temporarily do not execute pten kernel + # in this scenario and will find a better solution later and remove + # this setting. + set_flags({'FLAGS_run_pten_kernel': False}) + while not self._thread_done_event.is_set(): try: indices = next(self._sampler_iter) @@ -501,6 +517,9 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): # APIs in this thread. _set_expected_place(legacy_expected_place) + # NOTE(chenweihang): See Note [ Why need to set not to execute pten kernel here? ] + set_flags({'FLAGS_run_pten_kernel': False}) + while not self._thread_done_event.is_set(): batch = self._get_data() if not self._thread_done_event.is_set(): -- GitLab