From a1e7fd4a139bd95abc29960ef97e4369af1a1f4c Mon Sep 17 00:00:00 2001 From: Huihuang Zheng Date: Fri, 23 Oct 2020 10:31:02 +0800 Subject: [PATCH] Fix test_parallel_executor_test_while_train Random Failure by Decreasing GPU Usage (#28213) Recently, test_parallel_executor_test_while_train randomly failed on CI. On all CI logs, it showed NCCL initialization failed or cusolver initialization failed. I found online that those failure is usually caused by GPU shortage. Those API calls CUDA APIs directly so it shouldn't be the problem of allocator. It may be somewhere in PaddlePaddle increases GPU usage. However, I run this test for 1000 times on my machine and the CI machine, either of them can reproduce the random failure. Maybe there is something related to the environment only happened in test env. To verify my assumption that somewhere in PaddlePaddle increases GPU usage and also fix this CI, I decreased the batch_size to see whether the random failure disappears in test env. --- .../tests/unittests/test_parallel_executor_test_while_train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py index fd47dc37e76..76d93259a64 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py @@ -36,7 +36,7 @@ class ParallelExecutorTestingDuringTraining(unittest.TestCase): opt = fluid.optimizer.SGD(learning_rate=0.001) opt.minimize(loss) - batch_size = 32 + batch_size = 16 image = np.random.normal(size=(batch_size, 784)).astype('float32') label = np.random.randint(0, 10, (batch_size, 1), dtype="int64") -- GitLab