From afd0c1db8a7bf0d173a5d4d7d4bacfd7b85afe9a Mon Sep 17 00:00:00 2001 From: freeliuzc Date: Tue, 14 Jun 2022 12:28:23 +0800 Subject: [PATCH] Use tempfile to place all the temporary files. (#43392) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 使用 tempfile 替换临时文件，保证在单测结束后，所有临时文件都会被正常的删除，避免占用磁盘文件。 此 PR 仅涉及单测修改，不影响现有功能。 develop 分支修改在 PR 43376 --- .../unittests/test_directory_migration.py | 13 +++++++-- .../tests/unittests/test_imperative_gnn.py | 1 + .../fluid/tests/unittests/test_input_spec.py | 17 +++++++++-- .../tests/unittests/test_layout_autotune.py | 14 ++++++---- .../fluid/tests/unittests/test_optimizer.py | 28 +++++++++++++------ .../tests/unittests/test_switch_autotune.py | 9 +++--- .../unittests/test_traced_layer_err_msg.py | 12 ++++++-- 7 files changed, 68 insertions(+), 26 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_directory_migration.py b/python/paddle/fluid/tests/unittests/test_directory_migration.py index 2ec16a9dcab..7d8e0ec9a7d 100644 --- a/python/paddle/fluid/tests/unittests/test_directory_migration.py +++ b/python/paddle/fluid/tests/unittests/test_directory_migration.py @@ -17,13 +17,21 @@ from __future__ import print_function import os import sys import time +import tempfile import subprocess import unittest import numpy as np + import paddle class TestDirectory(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def get_import_command(self, module): paths = module.split('.') if len(paths) == 1: @@ -76,7 +84,7 @@ class TestDirectory(unittest.TestCase): 'paddle.static.nn.spectral_norm', 'paddle.static.nn.embedding' ] - import_file = 'run_import_modules.py' + import_file = os.path.join(self.temp_dir.name, 'run_import_modules.py') with open(import_file, "w") as wb: for module in new_directory: @@ -137,7 +145,8 @@ class TestDirectory(unittest.TestCase): 'paddle.declarative.spectral_norm', 'paddle.declarative.embedding' ] - import_file = 'run_old_import_modules.py' + import_file = os.path.join(self.temp_dir.name, + 'run_old_import_modules.py') with open(import_file, "w") as wb: cmd_context_count = """ diff --git a/python/paddle/fluid/tests/unittests/test_imperative_gnn.py b/python/paddle/fluid/tests/unittests/test_imperative_gnn.py index a5a90461551..5d27811d553 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_gnn.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_gnn.py @@ -176,4 +176,5 @@ class TestDygraphGNN(unittest.TestCase): if __name__ == '__main__': + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_input_spec.py b/python/paddle/fluid/tests/unittests/test_input_spec.py index 4e0aa4a9bca..979b33d275a 100644 --- a/python/paddle/fluid/tests/unittests/test_input_spec.py +++ b/python/paddle/fluid/tests/unittests/test_input_spec.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import tempfile import unittest import numpy as np + import paddle import paddle.fluid as fluid from paddle.static import InputSpec @@ -157,6 +160,10 @@ class TestNetWithNonTensorSpec(unittest.TestCase): self.out_num = 16 self.x_spec = paddle.static.InputSpec([-1, 16], name='x') self.x = paddle.randn([4, 16]) + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() @classmethod def setUpClass(cls): @@ -179,7 +186,7 @@ class TestNetWithNonTensorSpec(unittest.TestCase): self.check_result(specs, 'list') def check_result(self, specs, path): - path = './net_non_tensor_' + path + path = os.path.join(self.temp_dir.name, './net_non_tensor_', path) net = NetWithNonTensorSpec(self.in_num, self.out_num) net.eval() @@ -215,7 +222,7 @@ class TestNetWithNonTensorSpec(unittest.TestCase): net = paddle.jit.to_static(net, input_spec=specs) net.eval() - path = './net_twice' + path = os.path.join(self.temp_dir.name, './net_twice') # NOTE: check input_specs_compatible new_specs = [self.x_spec, True, "bn", 10] @@ -259,6 +266,10 @@ class TestNetWithNonTensorSpecWithPrune(unittest.TestCase): self.y_spec = paddle.static.InputSpec([16], name='y') self.x = paddle.randn([4, 16]) self.y = paddle.randn([16]) + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() @classmethod def setUpClass(cls): @@ -266,7 +277,7 @@ class TestNetWithNonTensorSpecWithPrune(unittest.TestCase): def test_non_tensor_with_prune(self): specs = [self.x_spec, self.y_spec, True] - path = './net_non_tensor_prune_' + path = os.path.join(self.temp_dir.name, './net_non_tensor_prune_') net = NetWithNonTensorSpecPrune(self.in_num, self.out_num) net.eval() diff --git a/python/paddle/fluid/tests/unittests/test_layout_autotune.py b/python/paddle/fluid/tests/unittests/test_layout_autotune.py index a1440f8587a..da925a49e31 100644 --- a/python/paddle/fluid/tests/unittests/test_layout_autotune.py +++ b/python/paddle/fluid/tests/unittests/test_layout_autotune.py @@ -12,14 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import paddle -import unittest +import os +import json import numpy -import paddle.nn.functional as F +import unittest import tempfile import warnings -import json -import os + +import paddle +import paddle.nn.functional as F +from paddle.fluid.framework import _enable_legacy_dygraph + +_enable_legacy_dygraph() class SimpleNet(paddle.nn.Layer): diff --git a/python/paddle/fluid/tests/unittests/test_optimizer.py b/python/paddle/fluid/tests/unittests/test_optimizer.py index e8820d5a870..f5e750cd7e9 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer.py @@ -14,6 +14,9 @@ from __future__ import print_function +import os +import numpy as np +import tempfile import unittest import paddle.fluid as fluid @@ -21,12 +24,11 @@ import paddle.fluid.framework as framework import paddle.fluid.optimizer as optimizer import paddle.fluid.core as core import paddle.compat as cpt -import numpy as np from paddle.fluid.backward import append_backward from paddle.fluid.framework import Program, program_guard, convert_np_dtype_to_dtype_ import paddle from paddle.io import Dataset -import numpy + paddle.enable_static() @@ -1121,9 +1123,15 @@ class TestMasterWeightSaveForFP16(unittest.TestCase): Master weights will be saved by optimizer::state_dict. ''' + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def check_with_opt_state_dict(self, use_save_load=True): paddle.seed(100) - numpy.random.seed(100) + np.random.seed(100) class SimpleNet(paddle.nn.Layer): def __init__(self, input_size, output_size): @@ -1147,8 +1155,8 @@ class TestMasterWeightSaveForFP16(unittest.TestCase): self.num_samples = num_samples def __getitem__(self, idx): - data = numpy.random.random([input_size]).astype('float16') - label = numpy.random.random([output_size]).astype('float16') + data = np.random.random([input_size]).astype('float16') + label = np.random.random([output_size]).astype('float16') return data, label def __len__(self): @@ -1182,10 +1190,12 @@ class TestMasterWeightSaveForFP16(unittest.TestCase): optimizer.clear_grad(set_to_zero=False) if use_save_load and i == 5: - paddle.save(model.state_dict(), "model.pdparams") - paddle.save(optimizer.state_dict(), "opt.pdopt") - model.set_state_dict(paddle.load("model.pdparams")) - optimizer.set_state_dict(paddle.load("opt.pdopt")) + model_path = os.path.join(self.temp_dir.name, "model.pdparams") + optimizer_path = os.path.join(self.temp_dir.name, "opt.pdopt") + paddle.save(model.state_dict(), model_path) + paddle.save(optimizer.state_dict(), optimizer_path) + model.set_state_dict(paddle.load(model_path)) + optimizer.set_state_dict(paddle.load(optimizer_path)) return loss.numpy() diff --git a/python/paddle/fluid/tests/unittests/test_switch_autotune.py b/python/paddle/fluid/tests/unittests/test_switch_autotune.py index 0049a922b91..67489356836 100644 --- a/python/paddle/fluid/tests/unittests/test_switch_autotune.py +++ b/python/paddle/fluid/tests/unittests/test_switch_autotune.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import paddle -import unittest +import os +import json import numpy as np +import unittest import tempfile import warnings -import json -import os + +import paddle class SimpleNet(paddle.nn.Layer): diff --git a/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py b/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py index 5703ce13131..1de92d3ccf3 100644 --- a/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py +++ b/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py @@ -13,12 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import tempfile +import unittest import numpy as np + import paddle import paddle.fluid as fluid -import unittest import paddle.nn as nn -import os class SimpleFCLayer(nn.Layer): @@ -189,11 +191,15 @@ class TestTracedLayerSaveInferenceModel(unittest.TestCase): """test save_inference_model will automaticlly create non-exist dir""" def setUp(self): - self.save_path = "./nonexist_dir/fc" + self.temp_dir = tempfile.TemporaryDirectory() + self.save_path = os.path.join(self.temp_dir.name, "./nonexist_dir/fc") import shutil if os.path.exists(os.path.dirname(self.save_path)): shutil.rmtree(os.path.dirname(self.save_path)) + def tearDown(self): + self.temp_dir.cleanup() + def test_mkdir_when_input_path_non_exist(self): if fluid.framework.in_dygraph_mode(): return -- GitLab