From 1a660c8a4c962318f4254a8113cd267cdd89d4ed Mon Sep 17 00:00:00 2001 From: zhangbopd <105368690+zhangbopd@users.noreply.github.com> Date: Thu, 16 Jun 2022 21:08:15 +0800 Subject: [PATCH] [cherry pick] Unit test with tempfile to place the temporary files (#43522) Use tempfile for unit test & custom op test to replace temporary files to ensure that all temporary files will be deleted normally after a single measurement, avoiding the usage of disk files. The PR only involves single-test and op test modifications and does not affect existing functionality. Release/2.3 branch modified in PR43521; --- .../tests/custom_op/test_custom_relu_model.py | 18 ++++++++++++++--- .../custom_op/test_custom_relu_op_setup.py | 1 + .../test_gpu_package_without_gpu_device.py | 10 +++++++++- .../test_imperative_auto_mixed_precision.py | 19 ++++++++++++++++-- .../fluid/tests/unittests/test_newprofiler.py | 20 ++++++++++++------- .../fluid/tests/unittests/test_profiler.py | 2 ++ .../tests/unittests/test_translated_layer.py | 9 ++++++++- 7 files changed, 65 insertions(+), 14 deletions(-) diff --git a/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py b/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py index 4980a159225..9f6d1fbe62a 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py @@ -15,6 +15,7 @@ import os import unittest import numpy as np +import tempfile import paddle from paddle import nn @@ -72,6 +73,9 @@ class Net(nn.Layer): class TestDygraphModel(unittest.TestCase): + def tearDown(self): + self.temp_dir.cleanup() + def setUp(self): self.seed = 2021 @@ -92,8 +96,12 @@ class TestDygraphModel(unittest.TestCase): self.devices = ['cpu', 'gpu'] if not IS_MAC else ['cpu'] # for saving model - self.model_path_template = "infer_model/custom_relu_dygaph_model_{}.pdparams" - self.model_dy2stat_path = "infer_model/custom_relu_model_dy2sta" + self.temp_dir = tempfile.TemporaryDirectory() + self.model_save_dir = os.path.join(self.temp_dir.name, 'infer_model') + self.model_path_template = os.path.join( + self.model_save_dir, 'custom_relu_dygaph_model_{}.pdparams') + self.model_dy2stat_path = os.path.join( + self.model_save_dir, 'infer_model/custom_relu_model_dy2sta') # for dy2stat self.x_spec = paddle.static.InputSpec( @@ -207,12 +215,16 @@ class TestStaticModel(unittest.TestCase): self.devices = ['cpu', 'gpu'] if not IS_MAC else ['cpu'] # for saving model - self.model_path_template = "infer_model/custom_relu_static_model_{}_{}" + self.temp_dir = tempfile.TemporaryDirectory() + self.model_save_dir = os.path.join(self.temp_dir.name, 'infer_model') + self.model_path_template = os.path.join( + self.model_save_dir, 'custom_relu_static_model_{}_{}') paddle.enable_static() def tearDown(self): paddle.disable_static() + self.temp_dir.cleanup() def test_train_eval(self): for device in self.devices: diff --git a/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py b/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py index 55c9571d44f..7098fc23302 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py @@ -18,6 +18,7 @@ import site import unittest import paddle import paddle.static as static +import tempfile import subprocess import numpy as np from paddle.vision.transforms import Compose, Normalize diff --git a/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py b/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py index e528e742a27..f256b425bd7 100644 --- a/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py +++ b/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py @@ -19,18 +19,26 @@ import sys import subprocess import unittest import paddle +import tempfile import paddle.fluid as fluid from paddle.fluid import core class TestGPUPackagePaddle(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDwon(self): + self.temp_dir.cleanup() + def test_import_paddle(self): if core.is_compiled_with_cuda(): if core.is_compiled_with_rocm(): os.environ['HIP_VISIBLE_DEVICES'] = '' else: os.environ['CUDA_VISIBLE_DEVICES'] = '' - test_file = 'test_no_gpu_run_rand.py' + test_file = os.path.join(self.temp_dir.name, + 'test_no_gpu_run_rand.py') with open(test_file, 'w') as wb: cmd_test = """ import paddle diff --git a/python/paddle/fluid/tests/unittests/test_imperative_auto_mixed_precision.py b/python/paddle/fluid/tests/unittests/test_imperative_auto_mixed_precision.py index d200b77eea8..62458881305 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_auto_mixed_precision.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_auto_mixed_precision.py @@ -17,6 +17,9 @@ import paddle import paddle.fluid as fluid import numpy as np import six +import cv2 +import os +import tempfile from test_imperative_resnet import ResNet, BottleneckBlock, ConvBNLayer, train_parameters, optimizer_setting import paddle.nn as nn from paddle.static import InputSpec @@ -694,6 +697,12 @@ class TestAmpDecorator(unittest.TestCase): class TestPureFp16SaveLoad(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def test_save_dtype_exception(self): def func(): paddle.disable_static() @@ -803,7 +812,7 @@ class TestPureFp16SaveLoad(unittest.TestCase): 'opt': optimizer.state_dict(), 'scaler': scaler.state_dict() } - path = 'model.pdparams' + path = os.path.join(self.temp_dir.name, 'model.pdparams') paddle.save(obj, path) # paddle.load obj_load = paddle.load(path) @@ -840,6 +849,12 @@ class TestPureFp16SaveLoad(unittest.TestCase): class TestPureFp16InferenceSaveLoad(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def inference_save_load(self): BATCH_SIZE = 16 BATCH_NUM = 4 @@ -903,7 +918,7 @@ class TestPureFp16InferenceSaveLoad(unittest.TestCase): train(layer, loader, loss_fn, adam) # save - path = "example_model/linear" + path = os.path.join(self.temp_dir.name, 'example_model/linear') paddle.jit.save( layer, path, input_spec=[InputSpec( shape=[IMAGE_SIZE], name='x')]) diff --git a/python/paddle/fluid/tests/unittests/test_newprofiler.py b/python/paddle/fluid/tests/unittests/test_newprofiler.py index 53ade0dfb79..2471fc64349 100755 --- a/python/paddle/fluid/tests/unittests/test_newprofiler.py +++ b/python/paddle/fluid/tests/unittests/test_newprofiler.py @@ -17,7 +17,7 @@ from __future__ import print_function import unittest import numpy as np import tempfile - +import os import paddle import paddle.profiler as profiler import paddle.profiler.utils as utils @@ -27,12 +27,18 @@ from paddle.io import Dataset, DataLoader class TestProfiler(unittest.TestCase): + def tearDown(self): + self.temp_dir.cleanup() + def test_profiler(self): def my_trace_back(prof): - profiler.export_chrome_tracing('./test_profiler_chrometracing/')( - prof) - profiler.export_protobuf('./test_profiler_pb/')(prof) + path = os.path.join(self.temp_dir.name, + './test_profiler_chrometracing') + profiler.export_chrome_tracing(path)(prof) + path = os.path.join(self.temp_dir.name, './test_profiler_pb') + profiler.export_protobuf(path)(prof) + self.temp_dir = tempfile.TemporaryDirectory() x_value = np.random.randn(2, 3, 3) x = paddle.to_tensor( x_value, stop_gradient=False, place=paddle.CPUPlace()) @@ -130,10 +136,10 @@ class TestProfiler(unittest.TestCase): y = x / 2.0 paddle.grad(outputs=y, inputs=[x], grad_outputs=ones_like_y) prof.step() - - prof.export(path='./test_profiler_pb.pb', format='pb') + path = os.path.join(self.temp_dir.name, './test_profiler_pb.pb') + prof.export(path=path, format='pb') prof.summary() - result = profiler.utils.load_profiler_result('./test_profiler_pb.pb') + result = profiler.utils.load_profiler_result(path) prof = None dataset = RandomDataset(10 * 4) simple_net = SimpleNet() diff --git a/python/paddle/fluid/tests/unittests/test_profiler.py b/python/paddle/fluid/tests/unittests/test_profiler.py index 1b8852810f2..99da036f3a9 100644 --- a/python/paddle/fluid/tests/unittests/test_profiler.py +++ b/python/paddle/fluid/tests/unittests/test_profiler.py @@ -18,6 +18,7 @@ import unittest import os import tempfile import numpy as np +import paddle import paddle.utils as utils import paddle.fluid as fluid import paddle.fluid.profiler as profiler @@ -200,4 +201,5 @@ class TestProfilerAPIError(unittest.TestCase): if __name__ == '__main__': + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_translated_layer.py b/python/paddle/fluid/tests/unittests/test_translated_layer.py index 79652b37b77..4c699a68d48 100644 --- a/python/paddle/fluid/tests/unittests/test_translated_layer.py +++ b/python/paddle/fluid/tests/unittests/test_translated_layer.py @@ -16,6 +16,8 @@ from __future__ import print_function import unittest import numpy as np +import tempfile +import os import paddle import paddle.nn as nn import paddle.optimizer as opt @@ -72,6 +74,9 @@ def train(layer, loader, loss_fn, opt): class TestTranslatedLayer(unittest.TestCase): + def tearDown(self): + self.temp_dir.cleanup() + def setUp(self): # enable dygraph mode place = paddle.CPUPlace() @@ -96,12 +101,14 @@ class TestTranslatedLayer(unittest.TestCase): shuffle=True, drop_last=True, num_workers=0) + self.temp_dir = tempfile.TemporaryDirectory() # train train(self.layer, self.loader, self.loss_fn, self.sgd) # save - self.model_path = "linear.example.model" + self.model_path = os.path.join(self.temp_dir.name, + './linear.example.model') paddle.jit.save(self.layer, self.model_path) def test_inference_and_fine_tuning(self): -- GitLab