From 1c98e27e824ee88bb077bcf193bd3e1e9a8674ee Mon Sep 17 00:00:00 2001 From: zhangbopd <105368690+zhangbopd@users.noreply.github.com> Date: Thu, 16 Jun 2022 21:00:03 +0800 Subject: [PATCH] Unit test with tempfile to place the temporary files (#43521) * Unit test with tempfile to place the temporary files * Update test_custom_relu_model.py --- .../tests/custom_op/test_custom_relu_model.py | 18 +++++++++++++++--- .../custom_op/test_custom_relu_op_setup.py | 1 + .../test_gpu_package_without_gpu_device.py | 10 +++++++++- .../test_imperative_auto_mixed_precision.py | 19 +++++++++++++++++-- .../fluid/tests/unittests/test_newprofiler.py | 19 +++++++++++++------ .../fluid/tests/unittests/test_profiler.py | 2 ++ .../tests/unittests/test_translated_layer.py | 10 +++++++++- 7 files changed, 66 insertions(+), 13 deletions(-) diff --git a/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py b/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py index 78078963a7..ff0b11128a 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py @@ -15,6 +15,7 @@ import os import unittest import numpy as np +import tempfile import paddle from paddle import nn @@ -73,6 +74,9 @@ class Net(nn.Layer): class TestDygraphModel(unittest.TestCase): + def tearDown(self): + self.temp_dir.cleanup() + def setUp(self): self.seed = 2021 @@ -93,8 +97,12 @@ class TestDygraphModel(unittest.TestCase): self.devices = ['cpu', 'gpu'] if not IS_MAC else ['cpu'] # for saving model - self.model_path_template = "infer_model/custom_relu_dygaph_model_{}.pdparams" - self.model_dy2stat_path = "infer_model/custom_relu_model_dy2sta" + self.temp_dir = tempfile.TemporaryDirectory() + self.model_save_dir = os.path.join(self.temp_dir.name, 'infer_model') + self.model_path_template = os.path.join( + self.model_save_dir, 'custom_relu_dygaph_model_{}.pdparams') + self.model_dy2stat_path = os.path.join( + self.model_save_dir, 'infer_model/custom_relu_model_dy2sta') # for dy2stat self.x_spec = paddle.static.InputSpec(shape=[None, self.in_dim], @@ -210,12 +218,16 @@ class TestStaticModel(unittest.TestCase): self.devices = ['cpu', 'gpu'] if not IS_MAC else ['cpu'] # for saving model - self.model_path_template = "infer_model/custom_relu_static_model_{}_{}" + self.temp_dir = tempfile.TemporaryDirectory() + self.model_save_dir = os.path.join(self.temp_dir.name, 'infer_model') + self.model_path_template = os.path.join( + self.model_save_dir, 'custom_relu_static_model_{}_{}') paddle.enable_static() def tearDown(self): paddle.disable_static() + self.temp_dir.cleanup() def test_train_eval(self): for device in self.devices: diff --git a/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py b/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py index 29433b1715..1a53bf3354 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py @@ -18,6 +18,7 @@ import site import unittest import paddle import paddle.static as static +import tempfile import subprocess import numpy as np from paddle.vision.transforms import Compose, Normalize diff --git a/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py b/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py index d4dc21e764..30a86d02f3 100644 --- a/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py +++ b/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py @@ -19,19 +19,27 @@ import sys import subprocess import unittest import paddle +import tempfile import paddle.fluid as fluid from paddle.fluid import core class TestGPUPackagePaddle(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDwon(self): + self.temp_dir.cleanup() + def test_import_paddle(self): if core.is_compiled_with_cuda(): if core.is_compiled_with_rocm(): os.environ['HIP_VISIBLE_DEVICES'] = '' else: os.environ['CUDA_VISIBLE_DEVICES'] = '' - test_file = 'test_no_gpu_run_rand.py' + test_file = os.path.join(self.temp_dir.name, + 'test_no_gpu_run_rand.py') with open(test_file, 'w') as wb: cmd_test = """ import paddle diff --git a/python/paddle/fluid/tests/unittests/test_imperative_auto_mixed_precision.py b/python/paddle/fluid/tests/unittests/test_imperative_auto_mixed_precision.py index e9266a4643..7310d19a52 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_auto_mixed_precision.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_auto_mixed_precision.py @@ -17,6 +17,9 @@ import paddle import paddle.fluid as fluid import numpy as np import six +import cv2 +import os +import tempfile from test_imperative_resnet import ResNet, BottleneckBlock, ConvBNLayer, train_parameters, optimizer_setting import paddle.nn as nn from paddle.static import InputSpec @@ -737,6 +740,12 @@ class TestStateDictHookForAMP(unittest.TestCase): class TestPureFp16SaveLoad(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def test_save_dtype_exception(self): def func(): @@ -848,7 +857,7 @@ class TestPureFp16SaveLoad(unittest.TestCase): 'opt': optimizer.state_dict(), 'scaler': scaler.state_dict() } - path = 'model.pdparams' + path = os.path.join(self.temp_dir.name, 'model.pdparams') paddle.save(obj, path) # paddle.load obj_load = paddle.load(path) @@ -888,6 +897,12 @@ class TestPureFp16SaveLoad(unittest.TestCase): class TestPureFp16InferenceSaveLoad(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def inference_save_load(self): BATCH_SIZE = 16 BATCH_NUM = 4 @@ -951,7 +966,7 @@ class TestPureFp16InferenceSaveLoad(unittest.TestCase): train(layer, loader, loss_fn, adam) # save - path = "example_model/linear" + path = os.path.join(self.temp_dir.name, 'example_model/linear') paddle.jit.save(layer, path, input_spec=[InputSpec(shape=[IMAGE_SIZE], name='x')]) diff --git a/python/paddle/fluid/tests/unittests/test_newprofiler.py b/python/paddle/fluid/tests/unittests/test_newprofiler.py index 0143bdb532..99097aaf00 100755 --- a/python/paddle/fluid/tests/unittests/test_newprofiler.py +++ b/python/paddle/fluid/tests/unittests/test_newprofiler.py @@ -17,7 +17,7 @@ from __future__ import print_function import unittest import numpy as np import tempfile - +import os import paddle import paddle.profiler as profiler import paddle.profiler.utils as utils @@ -28,13 +28,19 @@ from paddle.io import Dataset, DataLoader class TestProfiler(unittest.TestCase): + def tearDown(self): + self.temp_dir.cleanup() + def test_profiler(self): def my_trace_back(prof): - profiler.export_chrome_tracing('./test_profiler_chrometracing/')( - prof) - profiler.export_protobuf('./test_profiler_pb/')(prof) + path = os.path.join(self.temp_dir.name, + './test_profiler_chrometracing') + profiler.export_chrome_tracing(path)(prof) + path = os.path.join(self.temp_dir.name, './test_profiler_pb') + profiler.export_protobuf(path)(prof) + self.temp_dir = tempfile.TemporaryDirectory() x_value = np.random.randn(2, 3, 3) x = paddle.to_tensor(x_value, stop_gradient=False, @@ -135,9 +141,10 @@ class TestProfiler(unittest.TestCase): paddle.grad(outputs=y, inputs=[x], grad_outputs=ones_like_y) prof.step() - prof.export(path='./test_profiler_pb.pb', format='pb') + path = os.path.join(self.temp_dir.name, './test_profiler_pb.pb') + prof.export(path=path, format='pb') prof.summary() - result = profiler.utils.load_profiler_result('./test_profiler_pb.pb') + result = profiler.utils.load_profiler_result(path) prof = None dataset = RandomDataset(10 * 4) simple_net = SimpleNet() diff --git a/python/paddle/fluid/tests/unittests/test_profiler.py b/python/paddle/fluid/tests/unittests/test_profiler.py index 0eec7633a2..4f5cfba0c1 100644 --- a/python/paddle/fluid/tests/unittests/test_profiler.py +++ b/python/paddle/fluid/tests/unittests/test_profiler.py @@ -18,6 +18,7 @@ import unittest import os import tempfile import numpy as np +import paddle import paddle.utils as utils import paddle.fluid as fluid import paddle.fluid.profiler as profiler @@ -205,4 +206,5 @@ class TestProfilerAPIError(unittest.TestCase): if __name__ == '__main__': + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_translated_layer.py b/python/paddle/fluid/tests/unittests/test_translated_layer.py index 4b0be989ef..ba44c78f2c 100644 --- a/python/paddle/fluid/tests/unittests/test_translated_layer.py +++ b/python/paddle/fluid/tests/unittests/test_translated_layer.py @@ -16,6 +16,8 @@ from __future__ import print_function import unittest import numpy as np +import tempfile +import os import paddle import paddle.nn as nn import paddle.optimizer as opt @@ -76,6 +78,9 @@ def train(layer, loader, loss_fn, opt): class TestTranslatedLayer(unittest.TestCase): + def tearDown(self): + self.temp_dir.cleanup() + def setUp(self): # enable dygraph mode place = paddle.CPUPlace() @@ -100,11 +105,14 @@ class TestTranslatedLayer(unittest.TestCase): drop_last=True, num_workers=0) + self.temp_dir = tempfile.TemporaryDirectory() + # train train(self.layer, self.loader, self.loss_fn, self.sgd) # save - self.model_path = "linear.example.model" + self.model_path = os.path.join(self.temp_dir.name, + './linear.example.model') paddle.jit.save(self.layer, self.model_path) def test_inference_and_fine_tuning(self): -- GitLab