From 0d0258f86279db27b9eceae3976e3e01038a1b16 Mon Sep 17 00:00:00 2001 From: YuanRisheng Date: Mon, 13 Jun 2022 10:56:40 +0800 Subject: [PATCH] change saved file dir for save_load unittest (#43416) --- .../unittests/test_faster_tokenizer_op.py | 7 +- .../test_imperative_load_static_param.py | 10 +- .../unittests/test_imperative_save_load_v2.py | 43 +++- .../tests/unittests/test_io_save_load.py | 31 ++- .../tests/unittests/test_jit_save_load.py | 236 ++++++++++++++---- .../fluid/tests/unittests/test_load_op.py | 12 +- .../test_load_state_dict_from_old_format.py | 21 +- .../tests/unittests/test_paddle_save_load.py | 120 ++++++--- .../unittests/test_paddle_save_load_binary.py | 41 ++- .../tests/unittests/test_static_save_load.py | 120 +++++---- .../unittests/test_static_save_load_bf16.py | 15 +- .../unittests/test_static_save_load_large.py | 9 +- 12 files changed, 501 insertions(+), 164 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py b/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py index d6ccec25a43..b707640923a 100755 --- a/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py +++ b/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py @@ -27,6 +27,7 @@ from paddle.fluid.layer_helper import LayerHelper from paddle import _C_ops import sys +import tempfile sys.path.append("./tokenizer") from tokenizer.bert_tokenizer import BertTokenizer @@ -157,11 +158,15 @@ class Predictor(object): class TestBertTokenizerOp(unittest.TestCase): def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() self.bert_tokenizer = BertTokenizer.from_pretrained("bert-base-chinese") - self.save_path = os.path.join(DATA_HOME, "fast_tokenizer") + self.save_path = os.path.join(self.temp_dir.name, "fast_tokenizer") self.param_path = os.path.join(self.save_path, "model.pdparams") self.inference_path = os.path.join(self.save_path, "inference") + def tearDown(self): + self.temp_dir.cleanup() + def init_data(self): self.faster_tokenizer = FasterTokenizer(self.bert_tokenizer.vocab) self.text = [ diff --git a/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py b/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py index 2c860a0a624..36bec7fb030 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py @@ -17,12 +17,15 @@ import paddle.fluid as fluid import paddle.fluid.framework as framework from paddle.fluid.dygraph.nn import * import numpy as np +import os +import tempfile class TestDygraphLoadStatic(unittest.TestCase): def testLoadStaticModel(self): # static mode + temp_dir = tempfile.TemporaryDirectory() a = fluid.data(name="a", shape=[10, 10]) conv_in = fluid.data(name="conv_in", shape=[None, 10, 10, 10]) @@ -144,9 +147,11 @@ class TestDygraphLoadStatic(unittest.TestCase): ) if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0)) out = exe.run(framework.default_startup_program()) - fluid.save(framework.default_main_program(), "./test_1") + fluid.save(framework.default_main_program(), + os.path.join(temp_dir.name, "test_1")) - para_dict = fluid.load_program_state("./test_1") + para_dict = fluid.load_program_state( + os.path.join(temp_dir.name, "test_1")) new_dict = {} for k, v in para_dict.items(): @@ -214,6 +219,7 @@ class TestDygraphLoadStatic(unittest.TestCase): my_test.set_dict(new_dict, use_structured_name=False) for k, v in my_test.state_dict().items(): self.assertTrue(np.array_equal(v.numpy(), new_dict[v.name])) + temp_dir.cleanup() if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py b/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py index 91bb1b7e94f..f0026f8ef33 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py @@ -29,6 +29,8 @@ import six import paddle from paddle.fluid.framework import _test_eager_guard +import tempfile + class SimpleLSTMRNN(fluid.Layer): @@ -219,6 +221,12 @@ class PtbModel(fluid.Layer): class TestDygraphPtbRnn(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def func_setUp(self): seed = 90 hidden_size = 10 @@ -295,7 +303,8 @@ class TestDygraphPtbRnn(unittest.TestCase): else: self.base_opti[k] = v - paddle.save(self.opti_dict, "./test_dy_v2.pdopt") + paddle.save(self.opti_dict, + os.path.join(self.temp_dir.name, "test_dy_v2.pdopt")) self.state_dict = ptb_model.state_dict() @@ -304,10 +313,10 @@ class TestDygraphPtbRnn(unittest.TestCase): np_t = v.numpy() self.model_base[k] = np_t - paddle.save(self.state_dict, "./test_dy_v2.pdparams") + paddle.save(self.state_dict, + os.path.join(self.temp_dir.name, "test_dy_v2.pdparams")) def func_testLoadAndSetVarBase(self): - self.setUp() seed = 90 hidden_size = 10 vocab_size = 1000 @@ -383,8 +392,10 @@ class TestDygraphPtbRnn(unittest.TestCase): self.assertTrue(np.sum(np.abs(v.numpy())) == 0) - para_state_dict = paddle.load("./test_dy_v2.pdparams") - opti_state_dict = paddle.load("./test_dy_v2.pdopt") + para_state_dict = paddle.load( + os.path.join(self.temp_dir.name, "test_dy_v2.pdparams")) + opti_state_dict = paddle.load( + os.path.join(self.temp_dir.name, "test_dy_v2.pdopt")) adam.set_state_dict(opti_state_dict) opti_dict = adam.state_dict() @@ -752,7 +763,8 @@ class TestDygraphPtbRnn(unittest.TestCase): last_hidden = None last_cell = None - state_dict, opti_dict = fluid.load_dygraph("./test_dy_v2") + state_dict, opti_dict = fluid.load_dygraph( + os.path.join(self.temp_dir.name, "test_dy_v2")) adam.set_state_dict(opti_dict) ptb_model.set_dict(state_dict) @@ -907,19 +919,23 @@ class TestDygraphPtbRnn(unittest.TestCase): with fluid.dygraph.guard(): emb = fluid.dygraph.Embedding([10, 10]) state_dict = emb.state_dict() - paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams')) + paddle.save( + state_dict, + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) para_state_dict = paddle.load( - os.path.join('saved_dy', 'emb_dy.pdparams')) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) def func_test_no_state_in_input_dict(self): with fluid.dygraph.guard(): emb = fluid.dygraph.Embedding([10, 10]) state_dict = emb.state_dict() - paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams')) + paddle.save( + state_dict, + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) para_state_dict = paddle.load( - os.path.join('saved_dy', 'emb_dy.pdparams')) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) para_state_dict.pop('weight') emb.set_state_dict(para_state_dict) @@ -928,9 +944,12 @@ class TestDygraphPtbRnn(unittest.TestCase): with fluid.dygraph.guard(): emb = fluid.dygraph.Embedding([10, 10]) state_dict = emb.state_dict() - paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams')) + paddle.save( + state_dict, + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) - para_state_dict = paddle.load(os.path.join('saved_dy', + para_state_dict = paddle.load(os.path.join(self.temp_dir.name, + 'saved_dy', 'emb_dy.pdparams'), return_numpy=True) para_state_dict['weight'] = np.expand_dims( diff --git a/python/paddle/fluid/tests/unittests/test_io_save_load.py b/python/paddle/fluid/tests/unittests/test_io_save_load.py index 0d5573ae702..995adcca732 100644 --- a/python/paddle/fluid/tests/unittests/test_io_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_io_save_load.py @@ -19,10 +19,19 @@ import paddle import paddle.fluid as fluid from paddle.fluid import core from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph +import tempfile +import os class TestSaveLoadAPIError(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.save_dir = os.path.join(self.temp_dir.name, "fake_dir") + + def tearDown(self): + self.temp_dir.cleanup() + def func_test_get_valid_program_error(self): # case 1: CompiledProgram no program graph = core.Graph(core.ProgramDesc()) @@ -45,13 +54,13 @@ class TestSaveLoadAPIError(unittest.TestCase): # case 1: main_program type error when vars None with self.assertRaises(TypeError): fluid.io.load_vars(executor=exe, - dirname="./fake_dir", + dirname=self.save_dir, main_program="program") # case 2: main_program type error when vars not None with self.assertRaises(TypeError): fluid.io.load_vars(executor=exe, - dirname="./fake_dir", + dirname=self.save_dir, main_program="program", vars="vars") @@ -63,6 +72,12 @@ class TestSaveLoadAPIError(unittest.TestCase): class TestSaveInferenceModelAPIError(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def func_test_useless_feeded_var_names(self): start_prog = fluid.Program() main_prog = fluid.Program() @@ -75,7 +90,8 @@ class TestSaveInferenceModelAPIError(unittest.TestCase): exe.run(start_prog) with self.assertRaisesRegexp( ValueError, "not involved in the target_vars calculation"): - fluid.io.save_inference_model(dirname='./model', + fluid.io.save_inference_model(dirname=os.path.join( + self.temp_dir.name, 'model'), feeded_var_names=['x', 'y'], target_vars=[z], executor=exe, @@ -89,13 +105,20 @@ class TestSaveInferenceModelAPIError(unittest.TestCase): class TestWhenTrainWithNoGrad(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def func_test_when_train_with_no_grad(self): paddle.disable_static() net = paddle.nn.Linear(1024, 1) net = paddle.jit.to_static(net) x = paddle.rand([1024], 'float32') net(x) - save_path = './train_with_no_grad' + save_path = os.path.join(self.temp_dir.name, 'train_with_no_grad') + paddle.jit.save(net, save_path) net = paddle.jit.load(save_path) net.train() diff --git a/python/paddle/fluid/tests/unittests/test_jit_save_load.py b/python/paddle/fluid/tests/unittests/test_jit_save_load.py index 3c6bd9baea4..bf5ccf1a854 100644 --- a/python/paddle/fluid/tests/unittests/test_jit_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_jit_save_load.py @@ -18,6 +18,7 @@ import os import pickle import shutil import unittest +import tempfile import numpy as np import paddle from paddle.static import InputSpec @@ -347,13 +348,18 @@ def train_with_label(layer, input_size=784, label_size=1): class TestJitSaveLoad(unittest.TestCase): def setUp(self): - self.model_path = "test_jit_save_load/model" + self.temp_dir = tempfile.TemporaryDirectory() + self.model_path = os.path.join(self.temp_dir.name, + "test_jit_save_load/model") # enable dygraph mode fluid.enable_dygraph() # config seed paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) + def tearDown(self): + self.temp_dir.cleanup() + def train_and_save_model(self, model_path=None): layer = LinearNet(784, 1) example_inputs, layer, _ = train(layer) @@ -412,12 +418,14 @@ class TestJitSaveLoad(unittest.TestCase): new_layer(x).numpy())) def test_load_dygraph_no_path(self): - model_path = "test_jit_save_load.no_path/model_path" + model_path = os.path.join(self.temp_dir.name, + "test_jit_save_load.no_path/model_path") with self.assertRaises(ValueError): model_dict, _ = fluid.dygraph.load_dygraph(model_path) def test_jit_load_no_path(self): - path = "test_jit_save_load.no_path/model_path" + path = os.path.join(self.temp_dir.name, + "test_jit_save_load.no_path/model_path") with self.assertRaises(ValueError): loaded_layer = paddle.jit.load(path) @@ -427,6 +435,10 @@ class TestSaveLoadWithNestOut(unittest.TestCase): def setUp(self): # enable dygraph mode fluid.enable_dygraph() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_nest_output(self): x = fluid.dygraph.to_variable( @@ -436,7 +448,7 @@ class TestSaveLoadWithNestOut(unittest.TestCase): dy_outs = flatten(net(x)) net = declarative(net, input_spec=[InputSpec([None, 8], name='x')]) - model_path = "net_with_nest_out/model" + model_path = os.path.join(self.temp_dir.name, "net_with_nest_out/model") paddle.jit.save(net, model_path) load_net = paddle.jit.load(model_path) @@ -459,8 +471,9 @@ class TestSaveLoadWithDictInput(unittest.TestCase): # {'img': var img : fluid.VarType.LOD_TENSOR.shape(-1, 8).astype(VarType.FP32)}, # {'label': var label : fluid.VarType.LOD_TENSOR.shape(-1, 1).astype(VarType.INT64)}) self.assertEqual(len(net.forward.concrete_program.inputs), 3) - - path = "test_jit_save_load_with_dict_input/model" + temp_dir = tempfile.TemporaryDirectory() + path = os.path.join(temp_dir.name, + "test_jit_save_load_with_dict_input/model") # prune inputs paddle.jit.save(layer=net, path=path, @@ -478,14 +491,16 @@ class TestSaveLoadWithDictInput(unittest.TestCase): # loaded_net._input_spec(): # [InputSpec(shape=(-1, 8), dtype=VarType.FP32, name=img)] self.assertEqual(len(loaded_net._input_spec()), 1) + temp_dir.cleanup() class TestSaveLoadWithDictInputNoPrune(unittest.TestCase): def test_dict_input(self): net = LinearNetWithDictInputNoPrune(8, 8) - - path = "test_jit_save_load_with_dict_input_no_prune/model" + temp_dir = tempfile.TemporaryDirectory() + path = os.path.join( + temp_dir.name, "test_jit_save_load_with_dict_input_no_prune/model") # prune inputs paddle.jit.save(layer=net, path=path, @@ -506,6 +521,7 @@ class TestSaveLoadWithDictInputNoPrune(unittest.TestCase): loaded_out = loaded_net(img, img2) self.assertEqual(len(loaded_net._input_spec()), 2) + temp_dir.cleanup() class TestSaveLoadWithInputSpec(unittest.TestCase): @@ -513,6 +529,10 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): def setUp(self): # enable dygraph mode fluid.enable_dygraph() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_with_input_spec(self): net = LinearNetReturnLoss(8, 8) @@ -520,7 +540,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): net.forward = declarative(net.forward, input_spec=[InputSpec([None, 8], name='x')]) - model_path = "input_spec.output_spec/model" + model_path = os.path.join(self.temp_dir.name, + "input_spec.output_spec/model") # check inputs and outputs self.assertTrue(len(net.forward.inputs) == 1) input_x = net.forward.inputs[0] @@ -540,7 +561,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): def test_multi_in_out(self): net = LinearNetMultiInput(8, 8) - model_path = "multi_inout.output_spec1/model" + model_path = os.path.join(self.temp_dir.name, + "multi_inout.output_spec1/model") # 1. check inputs and outputs self.assertTrue(len(net.forward.inputs) == 2) input_x = net.forward.inputs[0] @@ -562,7 +584,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): pred_x, pred_y = infer_layer(x, y) # 1. prune y and loss - model_path = "multi_inout.output_spec2/model" + model_path = os.path.join(self.temp_dir.name, + "multi_inout.output_spec2/model") output_spec = net.forward.outputs[:1] paddle.jit.save(net, model_path, [input_x], output_spec=output_spec) # 2. load again @@ -576,7 +599,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): def test_multi_in_out1(self): net = LinearNetMultiInput1(8, 8) - model_path = "multi_inout1.output_spec1/model" + model_path = os.path.join(self.temp_dir.name, + "multi_inout1.output_spec1/model") # 1. check inputs and outputs self.assertTrue(len(net.forward.inputs) == 2) input_x = net.forward.inputs[0] @@ -598,7 +622,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): pred_x, pred_y = infer_layer(x, y) # 1. prune y and loss - model_path = "multi_inout1.output_spec2/model" + model_path = os.path.join(self.temp_dir.name, + "multi_inout1.output_spec2/model") output_spec = net.forward.outputs[:1] paddle.jit.save(net, model_path, (input_x, ), output_spec=output_spec) # 2. load again @@ -618,6 +643,10 @@ class TestJitSaveLoadConfig(unittest.TestCase): # config seed paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_output_spec(self): train_layer = LinearNetReturnLoss(8, 8) @@ -631,7 +660,8 @@ class TestJitSaveLoadConfig(unittest.TestCase): adam.minimize(loss) train_layer.clear_gradients() - model_path = "save_load_config.output_spec" + model_path = os.path.join(self.temp_dir.name, + "save_load_config.output_spec") output_spec = [out] paddle.jit.save(layer=train_layer, path=model_path, @@ -648,22 +678,22 @@ class TestJitSaveLoadConfig(unittest.TestCase): def test_save_no_support_config_error(self): layer = LinearNet(784, 1) - path = "no_support_config_test" + path = os.path.join(self.temp_dir.name, "no_support_config_test") with self.assertRaises(ValueError): paddle.jit.save(layer=layer, path=path, model_filename="") def test_load_empty_model_filename_error(self): - path = "error_model_filename_test" + path = os.path.join(self.temp_dir.name, "error_model_filename_test") with self.assertRaises(ValueError): paddle.jit.load(path, model_filename="") def test_load_empty_params_filename_error(self): - path = "error_params_filename_test" + path = os.path.join(self.temp_dir.name, "error_params_filename_test") with self.assertRaises(ValueError): paddle.jit.load(path, params_filename="") def test_load_with_no_support_config(self): - path = "no_support_config_test" + path = os.path.join(self.temp_dir.name, "no_support_config_test") with self.assertRaises(ValueError): paddle.jit.load(path, separate_params=True) @@ -672,7 +702,9 @@ class TestJitMultipleLoading(unittest.TestCase): def setUp(self): self.linear_size = 4 - self.model_path = "jit_multi_load/model" + self.temp_dir = tempfile.TemporaryDirectory() + self.model_path = os.path.join(self.temp_dir.name, + "jit_multi_load/model") # enable dygraph mode fluid.enable_dygraph() # config seed @@ -681,6 +713,9 @@ class TestJitMultipleLoading(unittest.TestCase): # train and save base model self.train_and_save_orig_model() + def tearDown(self): + self.temp_dir.cleanup() + def train_and_save_orig_model(self): layer = LinearNet(self.linear_size, self.linear_size) example_inputs, layer, _ = train(layer, self.linear_size, 1) @@ -702,13 +737,18 @@ class TestJitPruneModelAndLoad(unittest.TestCase): def setUp(self): self.linear_size = 4 - self.model_path = "jit_prune_model_and_load/model" + self.temp_dir = tempfile.TemporaryDirectory() + self.model_path = os.path.join(self.temp_dir.name, + "jit_prune_model_and_load/model") # enable dygraph mode fluid.enable_dygraph() # config seed paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) + def tearDown(self): + self.temp_dir.cleanup() + def train_and_save(self): train_layer = LinearNetReturnHidden(8, 8) adam = fluid.optimizer.AdamOptimizer( @@ -764,6 +804,10 @@ class TestJitSaveMultiCases(unittest.TestCase): # config seed paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def verify_inference_correctness(self, layer, @@ -796,7 +840,8 @@ class TestJitSaveMultiCases(unittest.TestCase): train(layer) - model_path = "test_no_prune_to_static_after_train/model" + model_path = os.path.join(self.temp_dir.name, + "test_no_prune_to_static_after_train/model") paddle.jit.save(layer, model_path) self.verify_inference_correctness(layer, model_path) @@ -804,7 +849,8 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_no_prune_to_static_no_train(self): layer = LinearNetWithInputSpec(784, 1) - model_path = "test_no_prune_to_static_no_train/model" + model_path = os.path.join(self.temp_dir.name, + "test_no_prune_to_static_no_train/model") paddle.jit.save(layer, model_path) self.verify_inference_correctness(layer, model_path) @@ -814,7 +860,8 @@ class TestJitSaveMultiCases(unittest.TestCase): train(layer) - model_path = "test_no_prune_no_to_static_after_train/model" + model_path = os.path.join( + self.temp_dir.name, "test_no_prune_no_to_static_after_train/model") paddle.jit.save( layer, model_path, @@ -827,7 +874,9 @@ class TestJitSaveMultiCases(unittest.TestCase): example_inputs, _, _ = train(layer) - model_path = "test_no_prune_no_to_static_after_train_with_examples/model" + model_path = os.path.join( + self.temp_dir.name, + "test_no_prune_no_to_static_after_train_with_examples/model") paddle.jit.save(layer=layer, path=model_path, input_spec=example_inputs) self.verify_inference_correctness(layer, model_path) @@ -835,7 +884,8 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_no_prune_no_to_static_no_train(self): layer = LinearNetNotDeclarative(784, 1) - model_path = "test_no_prune_no_to_static_no_train/model" + model_path = os.path.join(self.temp_dir.name, + "test_no_prune_no_to_static_no_train/model") paddle.jit.save( layer, model_path, @@ -848,7 +898,8 @@ class TestJitSaveMultiCases(unittest.TestCase): out = train_with_label(layer) - model_path = "test_prune_to_static_after_train/model" + model_path = os.path.join(self.temp_dir.name, + "test_prune_to_static_after_train/model") paddle.jit.save(layer, model_path, input_spec=[ @@ -865,7 +916,8 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_prune_to_static_no_train(self): layer = LinerNetWithLabel(784, 1) - model_path = "test_prune_to_static_no_train/model" + model_path = os.path.join(self.temp_dir.name, + "test_prune_to_static_no_train/model") # TODO: no train, cannot get output_spec var here # now only can use index output_spec = layer.forward.outputs[:1] @@ -885,7 +937,8 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_prune_input_to_static_no_train(self): layer = LinerNetWithPruneInput(784, 1) - model_path = "test_prune_input_to_static_no_train/model" + model_path = os.path.join(self.temp_dir.name, + "test_prune_input_to_static_no_train/model") paddle.jit.save(layer, model_path, input_spec=[ @@ -899,7 +952,9 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_prune_useless_input_to_static_no_train(self): layer = LinerNetWithUselessInput(784, 1) - model_path = "test_prune_useless_input_to_static_no_train/model" + model_path = os.path.join( + self.temp_dir.name, + "test_prune_useless_input_to_static_no_train/model") paddle.jit.save(layer, model_path, input_spec=[ @@ -915,7 +970,8 @@ class TestJitSaveMultiCases(unittest.TestCase): train(layer) - model_path = "test_no_prune_input_spec_name_warning/model" + model_path = os.path.join( + self.temp_dir.name, "test_no_prune_input_spec_name_warning/model") paddle.jit.save( layer, model_path, @@ -935,7 +991,8 @@ class TestJitSaveMultiCases(unittest.TestCase): train(layer) - model_path = "test_not_prune_output_spec_name_warning/model" + model_path = os.path.join( + self.temp_dir.name, "test_not_prune_output_spec_name_warning/model") out = paddle.to_tensor(np.random.random((1, 1)).astype('float')) paddle.jit.save(layer, model_path, output_spec=[out]) @@ -944,7 +1001,8 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_prune_input_spec_name_error(self): layer = LinerNetWithLabel(784, 1) - model_path = "test_prune_input_spec_name_error/model" + model_path = os.path.join(self.temp_dir.name, + "test_prune_input_spec_name_error/model") with self.assertRaises(ValueError): paddle.jit.save( layer, @@ -964,7 +1022,8 @@ class TestJitSaveMultiCases(unittest.TestCase): train_with_label(layer) - model_path = "test_prune_to_static_after_train/model" + model_path = os.path.join(self.temp_dir.name, + "test_prune_to_static_after_train/model") out = paddle.to_tensor(np.random.random((1, 1)).astype('float')) with self.assertRaises(ValueError): paddle.jit.save(layer, @@ -980,10 +1039,15 @@ class TestJitSaveMultiCases(unittest.TestCase): class TestJitSaveLoadEmptyLayer(unittest.TestCase): def setUp(self): - self.model_path = "jit_save_load_empty_layer/model" + self.temp_dir = tempfile.TemporaryDirectory() + self.model_path = os.path.join(self.temp_dir.name, + "jit_save_load_empty_layer/model") # enable dygraph mode paddle.disable_static() + def tearDown(self): + self.temp_dir.cleanup() + def test_save_load_empty_layer(self): layer = EmptyLayer() x = paddle.to_tensor(np.random.random((10)).astype('float32')) @@ -997,10 +1061,15 @@ class TestJitSaveLoadEmptyLayer(unittest.TestCase): class TestJitSaveLoadNoParamLayer(unittest.TestCase): def setUp(self): - self.model_path = "jit_save_load_no_param_layer/model" + self.temp_dir = tempfile.TemporaryDirectory() + self.model_path = os.path.join(self.temp_dir.name, + "jit_save_load_no_param_layer/model") # enable dygraph mode paddle.disable_static() + def tearDown(self): + self.temp_dir.cleanup() + def test_save_load_no_param_layer(self): layer = NoParamLayer() x = paddle.to_tensor(np.random.random((5)).astype('float32')) @@ -1017,9 +1086,14 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase): def setUp(self): # enable dygraph mode paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_jit_save_load_inference(self): - model_path_inference = "jit_save_load_multi_methods/model" + model_path_inference = os.path.join( + self.temp_dir.name, "jit_save_load_multi_methods/model") IMAGE_SIZE = 224 layer = LinearNetWithMultiStaticFunc(IMAGE_SIZE, 10) inps = paddle.randn([1, IMAGE_SIZE]) @@ -1035,7 +1109,8 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase): getattr(load_net, func, None)(inps)).abs().max()) < 1e-5) def test_jit_save_load_multi_methods_inputspec(self): - model_path = 'jit_save_load_multi_methods/model' + model_path = os.path.join(self.temp_dir.name, + 'jit_save_load_multi_methods/model') layer = LinearNetWithMultiStaticFunc(784, 1) with self.assertRaises(ValueError): paddle.jit.save(layer, @@ -1043,7 +1118,8 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase): input_spec=[InputSpec(shape=[None, 784])]) def test_parse_name(self): - model_path_inference = "jit_save_load_parse_name/model" + model_path_inference = os.path.join(self.temp_dir.name, + "jit_save_load_parse_name/model") IMAGE_SIZE = 224 layer = LinearNet(IMAGE_SIZE, 1) inps = paddle.randn([1, IMAGE_SIZE]) @@ -1116,9 +1192,14 @@ class TestJitSaveLoadSaveWithoutRunning(unittest.TestCase): def setUp(self): # enable dygraph mode paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_save_load_finetune_load(self): - model_path = "test_jit_save_load_save_without_running/model" + model_path = os.path.join( + self.temp_dir.name, "test_jit_save_load_save_without_running/model") IMAGE_SIZE = 224 inps0 = paddle.randn([1, IMAGE_SIZE]) inps1 = paddle.randn([2, IMAGE_SIZE]) @@ -1157,9 +1238,14 @@ class TestJitSaveLoadFinetuneLoad(unittest.TestCase): def setUp(self): # enable dygraph mode paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_save_load_finetune_load(self): - model_path = "test_jit_save_load_finetune_load/model" + model_path = os.path.join(self.temp_dir.name, + "test_jit_save_load_finetune_load/model") IMAGE_SIZE = 224 inps0 = paddle.randn([1, IMAGE_SIZE]) inps1 = paddle.randn([2, IMAGE_SIZE]) @@ -1196,6 +1282,10 @@ class TestJitSaveLoadFunctionCase1(unittest.TestCase): def setUp(self): paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_jit_save_load_static_function(self): @@ -1203,7 +1293,8 @@ class TestJitSaveLoadFunctionCase1(unittest.TestCase): def fun(inputs): return paddle.tanh(inputs) - path = 'test_jit_save_load_function_1/func' + path = os.path.join(self.temp_dir.name, + 'test_jit_save_load_function_1/func') inps = paddle.rand([3, 6]) origin = fun(inps) @@ -1218,6 +1309,10 @@ class TestJitSaveLoadFunctionCase2(unittest.TestCase): def setUp(self): paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_jit_save_load_function_input_spec(self): @@ -1227,7 +1322,8 @@ class TestJitSaveLoadFunctionCase2(unittest.TestCase): def fun(inputs): return paddle.nn.functional.relu(inputs) - path = 'test_jit_save_load_function_2/func' + path = os.path.join(self.temp_dir.name, + 'test_jit_save_load_function_2/func') inps = paddle.rand([3, 6]) origin = fun(inps) @@ -1241,13 +1337,18 @@ class TestJitSaveLoadFunctionCase3(unittest.TestCase): def setUp(self): paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_jit_save_load_function_function(self): def fun(inputs): return paddle.tanh(inputs) - path = 'test_jit_save_load_function_3/func' + path = os.path.join(self.temp_dir.name, + 'test_jit_save_load_function_3/func') inps = paddle.rand([3, 6]) origin = fun(inps) @@ -1268,6 +1369,10 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase): def setUp(self): paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_jit_save_load_function(self): @@ -1290,7 +1395,9 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase): func = paddle.jit.to_static(layer.anothor_forward, [paddle.static.InputSpec(shape=[-1, 5])]) - path = 'test_jit_save_load_function_with_params_case1/func' + path = os.path.join( + self.temp_dir.name, + 'test_jit_save_load_function_with_params_case1/func') paddle.jit.save(func, path) load_func = paddle.jit.load(path) @@ -1302,6 +1409,10 @@ class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase): def setUp(self): paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_jit_save_load_function(self): @@ -1322,7 +1433,9 @@ class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase): inps = paddle.rand([3, 5]) - path = 'test_jit_save_load_function_with_params_case2/func' + path = os.path.join( + self.temp_dir.name, + 'test_jit_save_load_function_with_params_case2/func') paddle.jit.save(layer.anothor_forward, path) origin_result = layer.anothor_forward(inps) load_func = paddle.jit.load(path) @@ -1337,6 +1450,10 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase): def setUp(self): paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_jit_save_load_function(self): @@ -1358,7 +1475,9 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase): inps = paddle.rand([3, 5]) origin = layer.anothor_forward(inps) - path = 'test_jit_save_load_function_with_params_case3/func' + path = os.path.join( + self.temp_dir.name, + 'test_jit_save_load_function_with_params_case3/func') paddle.jit.save(layer.anothor_forward, path) load_func = paddle.jit.load(path) @@ -1368,6 +1487,12 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase): class TestJitSaveLoadDataParallel(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def verify_inference_correctness(self, layer, path): layer.eval() loaded_layer = paddle.jit.load(path) @@ -1384,8 +1509,8 @@ class TestJitSaveLoadDataParallel(unittest.TestCase): def test_jit_save_data_parallel_with_inputspec(self): layer = LinearNetNotDeclarative(784, 1) layer = paddle.DataParallel(layer) - - path = "jit_save_data_parallel_with_inputspec/model" + path = os.path.join(self.temp_dir.name, + "jit_save_data_parallel_with_inputspec/model") paddle.jit.save(layer=layer, path=path, input_spec=[InputSpec(shape=[None, 784])]) @@ -1396,7 +1521,8 @@ class TestJitSaveLoadDataParallel(unittest.TestCase): layer = LinearNetWithInputSpec(784, 1) layer = paddle.DataParallel(layer) - path = "jit_save_data_parallel_with_to_static/model" + path = os.path.join(self.temp_dir.name, + "jit_save_data_parallel_with_to_static/model") paddle.jit.save(layer, path) self.verify_inference_correctness(layer, path) @@ -1417,6 +1543,12 @@ class InputSepcLayer(paddle.nn.Layer): class TestInputSpecCompatibility(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def _assert_input_spec_layer_return(self, expect_layer, test_layer): input_x = paddle.uniform([8, 8], dtype='float32') input_y = paddle.uniform([8, 1], dtype='float64') @@ -1429,7 +1561,8 @@ class TestInputSpecCompatibility(unittest.TestCase): def test_jit_save_compatible_input_sepc(self): layer = InputSepcLayer() - save_dir = "jit_save_compatible_input_spec" + save_dir = os.path.join(self.temp_dir.name, + "jit_save_compatible_input_spec") path = save_dir + "/model" paddle.jit.save(layer=layer, path=path) @@ -1463,7 +1596,8 @@ class TestInputSpecCompatibility(unittest.TestCase): def test_jit_save_incompatible_input_sepc(self): layer = InputSepcLayer() - save_dir = "jit_save_compatible_input_spec" + save_dir = os.path.join(self.temp_dir.name, + "jit_save_compatible_input_spec") path = save_dir + "/model" with self.assertRaises(ValueError): diff --git a/python/paddle/fluid/tests/unittests/test_load_op.py b/python/paddle/fluid/tests/unittests/test_load_op.py index 2896ff218c7..a9865251355 100644 --- a/python/paddle/fluid/tests/unittests/test_load_op.py +++ b/python/paddle/fluid/tests/unittests/test_load_op.py @@ -19,6 +19,8 @@ import numpy as np from op_test import OpTest, randomize_probability import paddle.fluid as fluid import paddle.fluid.layers as layers +import os +import tempfile class TestLoadOp(unittest.TestCase): @@ -26,6 +28,7 @@ class TestLoadOp(unittest.TestCase): """ def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() self.ones = np.ones((4, 4)).astype('float32') main_prog = fluid.Program() start_prog = fluid.Program() @@ -41,15 +44,20 @@ class TestLoadOp(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(start_prog) fluid.io.save_persistables(exe, - dirname="./model", + dirname=os.path.join(self.temp_dir.name, + "./model"), main_program=main_prog) + def tearDown(self): + self.temp_dir.cleanup() + def test_load(self): main_prog = fluid.Program() start_prog = fluid.Program() with fluid.program_guard(main_prog, start_prog): var = layers.create_tensor(dtype='float32') - layers.load(var, file_path='./model/w') + layers.load(var, + file_path=os.path.join(self.temp_dir.name, './model/w')) exe = fluid.Executor(fluid.CPUPlace()) exe.run(start_prog) diff --git a/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py b/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py index ac88b796054..32029e561d0 100644 --- a/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py +++ b/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py @@ -23,6 +23,7 @@ import paddle import paddle.fluid as fluid from paddle.fluid import core from test_imperative_base import new_program_scope +import tempfile def convolutional_neural_network(img): @@ -58,6 +59,7 @@ def static_train_net(img, label): class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() self.seed = 90 self.epoch_num = 1 self.batch_size = 128 @@ -65,6 +67,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): # enable static mode paddle.enable_static() + def tearDown(self): + self.temp_dir.cleanup() + def train_and_save_model(self, only_params=False): with new_program_scope(): startup_program = fluid.default_startup_program() @@ -121,7 +126,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): self.assertTrue(np.array_equal(value, load_dict[var_name])) def test_load_default(self): - self.save_dirname = "static_mnist.load_state_dict.default" + self.save_dirname = os.path.join( + self.temp_dir.name, "static_mnist.load_state_dict.default") self.model_filename = None self.params_filename = None orig_param_dict = self.train_and_save_model() @@ -133,7 +139,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): self.check_load_state_dict(orig_param_dict, new_load_param_dict) def test_load_with_model_filename(self): - self.save_dirname = "static_mnist.load_state_dict.model_filename" + self.save_dirname = os.path.join( + self.temp_dir.name, "static_mnist.load_state_dict.model_filename") self.model_filename = "static_mnist.model" self.params_filename = None orig_param_dict = self.train_and_save_model() @@ -147,7 +154,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): self.check_load_state_dict(orig_param_dict, new_load_param_dict) def test_load_with_param_filename(self): - self.save_dirname = "static_mnist.load_state_dict.param_filename" + self.save_dirname = os.path.join( + self.temp_dir.name, "static_mnist.load_state_dict.param_filename") self.model_filename = None self.params_filename = "static_mnist.params" orig_param_dict = self.train_and_save_model() @@ -161,7 +169,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): self.check_load_state_dict(orig_param_dict, new_load_param_dict) def test_load_with_model_and_param_filename(self): - self.save_dirname = "static_mnist.load_state_dict.model_and_param_filename" + self.save_dirname = os.path.join( + self.temp_dir.name, + "static_mnist.load_state_dict.model_and_param_filename") self.model_filename = "static_mnist.model" self.params_filename = "static_mnist.params" orig_param_dict = self.train_and_save_model() @@ -178,7 +188,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): self.check_load_state_dict(orig_param_dict, new_load_param_dict) def test_load_state_dict_from_save_params(self): - self.save_dirname = "static_mnist.load_state_dict.save_params" + self.save_dirname = os.path.join( + self.temp_dir.name, "static_mnist.load_state_dict.save_params") self.params_filename = None orig_param_dict = self.train_and_save_model(True) diff --git a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py index 3cf35550c58..d3052b719ae 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py @@ -19,6 +19,7 @@ import numpy as np import os import sys from io import BytesIO +import tempfile import paddle import paddle.nn as nn @@ -95,7 +96,10 @@ def train(layer, loader, loss_fn, opt): class TestSaveLoadLargeParameters(unittest.TestCase): def setUp(self): - pass + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_large_parameters_paddle_save(self): # enable dygraph mode @@ -105,7 +109,8 @@ class TestSaveLoadLargeParameters(unittest.TestCase): layer = LayerWithLargeParameters() save_dict = layer.state_dict() - path = os.path.join("test_paddle_save_load_large_param_save", + path = os.path.join(self.temp_dir.name, + "test_paddle_save_load_large_param_save", "layer.pdparams") protocol = 4 paddle.save(save_dict, path, protocol=protocol) @@ -117,6 +122,12 @@ class TestSaveLoadLargeParameters(unittest.TestCase): class TestSaveLoadPickle(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def test_pickle_protocol(self): # enable dygraph mode paddle.disable_static() @@ -124,7 +135,8 @@ class TestSaveLoadPickle(unittest.TestCase): layer = LinearNet() save_dict = layer.state_dict() - path = os.path.join("test_paddle_save_load_pickle_protocol", + path = os.path.join(self.temp_dir.name, + "test_paddle_save_load_pickle_protocol", "layer.pdparams") with self.assertRaises(ValueError): @@ -152,6 +164,12 @@ class TestSaveLoadPickle(unittest.TestCase): class TestSaveLoadAny(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def set_zero(self, prog, place, scope=None): if scope is None: scope = fluid.global_scope() @@ -213,7 +231,8 @@ class TestSaveLoadAny(unittest.TestCase): t = np.array(fluid.global_scope().find_var( var.name).get_tensor()) base_map[var.name] = t - path = os.path.join("test_replace_static_save_load", "model") + path = os.path.join(self.temp_dir.name, + "test_replace_static_save_load", "model") # paddle.save, legacy paddle.fluid.load self.replace_static_save(prog, path) self.set_zero(prog, place) @@ -239,7 +258,9 @@ class TestSaveLoadAny(unittest.TestCase): for var in prog.list_vars(): if var.persistable: tensor = var.get_value(fluid.global_scope()) - paddle.save(tensor, os.path.join(path_vars, var.name)) + paddle.save( + tensor, + os.path.join(self.temp_dir.name, path_vars, var.name)) with self.assertRaises(TypeError): var.get_value('fluid.global_scope()') with self.assertRaises(ValueError): @@ -257,7 +278,8 @@ class TestSaveLoadAny(unittest.TestCase): self.set_zero(prog, place) for var in prog.list_vars(): if var.persistable: - tensor = paddle.load(os.path.join(path_vars, var.name), + tensor = paddle.load(os.path.join(self.temp_dir.name, + path_vars, var.name), return_numpy=False) var.set_value(tensor) new_t = np.array(fluid.global_scope().find_var( @@ -293,7 +315,8 @@ class TestSaveLoadAny(unittest.TestCase): y.mean().backward() adam.step() state_dict = adam.state_dict() - path = 'paddle_save_load_v2/model.pdparams' + path = os.path.join(self.temp_dir.name, + 'paddle_save_load_v2/model.pdparams') with self.assertRaises(TypeError): paddle.save(state_dict, path, use_binary_format='False') # legacy paddle.save, paddle.load @@ -317,7 +340,8 @@ class TestSaveLoadAny(unittest.TestCase): # enable dygraph mode paddle.disable_static() layer = LinearNet() - path = 'paddle_save_load_v2/var_dygraph' + path = os.path.join(self.temp_dir.name, + 'paddle_save_load_v2/var_dygraph') tensor = layer._linear.weight with self.assertRaises(ValueError): paddle.save(tensor, path, pickle_protocol='3') @@ -361,7 +385,8 @@ class TestSaveLoadAny(unittest.TestCase): break scope = fluid.global_scope() origin_tensor = np.array(tensor) - path = 'test_single_pickle_var_static/var' + path = os.path.join(self.temp_dir.name, + 'test_single_pickle_var_static/var') paddle.save(tensor, path) self.set_zero(prog, place, scope) # static load @@ -383,7 +408,8 @@ class TestSaveLoadAny(unittest.TestCase): def test_dygraph_save_static_load(self): inps = np.random.randn(1, IMAGE_SIZE).astype('float32') - path = 'test_dygraph_save_static_load/dy-static.pdparams' + path = os.path.join(self.temp_dir.name, + 'test_dygraph_save_static_load/dy-static.pdparams') paddle.disable_static() with paddle.utils.unique_name.guard(): layer = LinearNet() @@ -427,10 +453,14 @@ class TestSaveLoadAny(unittest.TestCase): }) obj4 = (np.random.randn(5, 6), (123, )) - path1 = "test_save_load_any_complex_object_dygraph/obj1" - path2 = "test_save_load_any_complex_object_dygraph/obj2" - path3 = "test_save_load_any_complex_object_dygraph/obj3" - path4 = "test_save_load_any_complex_object_dygraph/obj4" + path1 = os.path.join(self.temp_dir.name, + "test_save_load_any_complex_object_dygraph/obj1") + path2 = os.path.join(self.temp_dir.name, + "test_save_load_any_complex_object_dygraph/obj2") + path3 = os.path.join(self.temp_dir.name, + "test_save_load_any_complex_object_dygraph/obj3") + path4 = os.path.join(self.temp_dir.name, + "test_save_load_any_complex_object_dygraph/obj4") paddle.save(obj1, path1) paddle.save(obj2, path2) paddle.save(obj3, path3) @@ -597,10 +627,18 @@ class TestSaveLoadAny(unittest.TestCase): }) obj4 = (np.ndarray([3, 4], dtype="float32"), ) - path1 = "test_save_load_any_complex_object_static/obj1" - path2 = "test_save_load_any_complex_object_static/obj2" - path3 = "test_save_load_any_complex_object_static/obj3" - path4 = "test_save_load_any_complex_object_static/obj4" + path1 = os.path.join( + self.temp_dir.name, + "test_save_load_any_complex_object_static/obj1") + path2 = os.path.join( + self.temp_dir.name, + "test_save_load_any_complex_object_static/obj2") + path3 = os.path.join( + self.temp_dir.name, + "test_save_load_any_complex_object_static/obj3") + path4 = os.path.join( + self.temp_dir.name, + "test_save_load_any_complex_object_static/obj4") paddle.save(obj1, path1) paddle.save(obj2, path2) paddle.save(obj3, path3) @@ -763,7 +801,8 @@ class TestSaveLoadAny(unittest.TestCase): def test_varbase_binary_var(self): paddle.disable_static() varbase = paddle.randn([3, 2], dtype='float32') - path = 'test_paddle_save_load_varbase_binary_var/varbase' + path = os.path.join(self.temp_dir.name, + 'test_paddle_save_load_varbase_binary_var/varbase') paddle.save(varbase, path, use_binary_format=True) load_array = paddle.load(path, return_numpy=True) load_tensor = paddle.load(path, return_numpy=False) @@ -851,6 +890,10 @@ class TestSaveLoad(unittest.TestCase): # config seed paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def build_and_train_model(self): # create network @@ -878,8 +921,10 @@ class TestSaveLoad(unittest.TestCase): layer, opt = self.build_and_train_model() # save - layer_save_path = "test_paddle_save_load.linear.pdparams" - opt_save_path = "test_paddle_save_load.linear.pdopt" + layer_save_path = os.path.join(self.temp_dir.name, + "test_paddle_save_load.linear.pdparams") + opt_save_path = os.path.join(self.temp_dir.name, + "test_paddle_save_load.linear.pdopt") layer_state_dict = layer.state_dict() opt_state_dict = opt.state_dict() @@ -895,7 +940,9 @@ class TestSaveLoad(unittest.TestCase): # test save load in static mode paddle.enable_static() - static_save_path = "static_mode_test/test_paddle_save_load.linear.pdparams" + static_save_path = os.path.join( + self.temp_dir.name, + "static_mode_test/test_paddle_save_load.linear.pdparams") paddle.save(layer_state_dict, static_save_path) load_static_state_dict = paddle.load(static_save_path) self.check_load_state_dict(layer_state_dict, load_static_state_dict) @@ -906,21 +953,30 @@ class TestSaveLoad(unittest.TestCase): # 2. test save path format error with self.assertRaises(ValueError): - paddle.save(layer_state_dict, "test_paddle_save_load.linear.model/") + paddle.save( + layer_state_dict, + os.path.join(self.temp_dir.name, + "test_paddle_save_load.linear.model/")) # 3. test load path not exist error with self.assertRaises(ValueError): - paddle.load("test_paddle_save_load.linear.params") + paddle.load( + os.path.join(self.temp_dir.name, + "test_paddle_save_load.linear.params")) # 4. test load old save path error with self.assertRaises(ValueError): - paddle.load("test_paddle_save_load.linear") + paddle.load( + os.path.join(self.temp_dir.name, + "test_paddle_save_load.linear")) class TestSaveLoadProgram(unittest.TestCase): def test_save_load_program(self): paddle.enable_static() + temp_dir = tempfile.TemporaryDirectory() + with new_program_scope(): layer = LinearNet() data = paddle.static.data(name='x_static_save', @@ -931,8 +987,12 @@ class TestSaveLoadProgram(unittest.TestCase): startup_program = paddle.static.default_startup_program() origin_main = main_program.desc.serialize_to_string() origin_startup = startup_program.desc.serialize_to_string() - path1 = "test_paddle_save_load_program/main_program.pdmodel" - path2 = "test_paddle_save_load_program/startup_program.pdmodel" + path1 = os.path.join( + temp_dir.name, + "test_paddle_save_load_program/main_program.pdmodel") + path2 = os.path.join( + temp_dir.name, + "test_paddle_save_load_program/startup_program.pdmodel") paddle.save(main_program, path1) paddle.save(startup_program, path2) @@ -941,12 +1001,14 @@ class TestSaveLoadProgram(unittest.TestCase): load_startup = paddle.load(path2).desc.serialize_to_string() self.assertTrue(origin_main == load_main) self.assertTrue(origin_startup == load_startup) + temp_dir.cleanup() class TestSaveLoadLayer(unittest.TestCase): def test_save_load_layer(self): paddle.disable_static() + temp_dir = tempfile.TemporaryDirectory() inps = paddle.randn([1, IMAGE_SIZE], dtype='float32') layer1 = LinearNet() layer2 = LinearNet() @@ -954,9 +1016,11 @@ class TestSaveLoadLayer(unittest.TestCase): layer2.eval() origin_layer = (layer1, layer2) origin = (layer1(inps), layer2(inps)) - path = "test_save_load_layer_/layer.pdmodel" + path = os.path.join(temp_dir.name, + "test_save_load_layer_/layer.pdmodel") with self.assertRaises(ValueError): paddle.save(origin_layer, path) + temp_dir.cleanup() if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py b/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py index bba65e469ab..c7ac11546e1 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py @@ -21,6 +21,7 @@ import os import sys import six import platform +import tempfile import paddle import paddle.nn as nn @@ -38,6 +39,10 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): def setUp(self): # enable static graph mode paddle.enable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def set_zero(self, prog, place, scope=None): if scope is None: @@ -97,7 +102,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t # test for replace_save_vars/io.load_vars - path_vars1 = 'test_replace_save_load_vars_binary1/model' + path_vars1 = os.path.join( + self.temp_dir.name, 'test_replace_save_load_vars_binary1/model') self.replace_save_vars(prog, path_vars1) # set var to zero self.set_zero(prog, place) @@ -116,7 +122,9 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): self.assertTrue(np.array_equal(new_t, base_t)) # test for io.save_vars/replace_load_vars - path_vars2 = 'test_replace_save_load_vars_binary2/model/' + path_vars2 = os.path.join( + self.temp_dir.name, + 'test_replace_save_load_vars_binary2/model/') fluid.io.save_vars(exe, path_vars2, main_program=prog, @@ -149,7 +157,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): prog = paddle.static.default_main_program() exe.run(fluid.default_startup_program()) - dirname = 'test_save_load_lod_tensor1/tensor_' + dirname = os.path.join(self.temp_dir.name, + 'test_save_load_lod_tensor1/tensor_') for var in prog.list_vars(): if var.persistable and list( var.shape) == [IMAGE_SIZE, OUTPUT_NUM]: @@ -171,12 +180,13 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): self.assertTrue(np.array_equal(origin, to_array)) with self.assertRaises(NotImplementedError): - path = 'test_save_load_error/temp' + path = os.path.join(self.temp_dir.name, 'test_save_load_error/temp') paddle.save({}, path, use_binary_format=True) # On the Windows platform, when parsing a string that can't be parsed as a `Program`, `desc_.ParseFromString` has a timeout risk. if 'Windows' != platform.system(): with self.assertRaises(ValueError): - path = 'test_save_load_error/temp' + path = os.path.join(self.temp_dir.name, + 'test_save_load_error/temp') with open(path, "w") as f: f.write('\0') paddle.load(path) @@ -187,11 +197,17 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): with self.assertRaises(RuntimeError): fluid.core.save_lod_tensor( - temp_lod, 'test_save_load_error_not_exist_file/not_exist_file') + temp_lod, + os.path.join( + self.temp_dir.name, + 'test_save_load_error_not_exist_file/not_exist_file')) with self.assertRaises(RuntimeError): fluid.core.load_lod_tensor( - temp_lod, 'test_save_load_error_not_exist_file/not_exist_file') + temp_lod, + os.path.join( + self.temp_dir.name, + 'test_save_load_error_not_exist_file/not_exist_file')) # save to memory byio = BytesIO() @@ -215,7 +231,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): rows = [0, 4, 7] row_numel = 12 selected_rows = fluid.core.SelectedRows(rows, height) - path = 'test_paddle_save_load_selected_rows/sr.pdsr' + path = os.path.join(self.temp_dir.name, + 'test_paddle_save_load_selected_rows/sr.pdsr') with self.assertRaises(ValueError): paddle.save(selected_rows, path, use_binary_format=True) @@ -236,11 +253,15 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): with self.assertRaises(RuntimeError): fluid.core.save_selected_rows( selected_rows, - 'test_paddle_save_load_selected_rows_not_exist_file/temp') + os.path.join( + self.temp_dir.name, + 'test_paddle_save_load_selected_rows_not_exist_file/temp')) with self.assertRaises(RuntimeError): fluid.core.load_selected_rows( selected_rows, - 'test_paddle_save_load_selected_rows_not_exist_file/temp') + os.path.join( + self.temp_dir.name, + 'test_paddle_save_load_selected_rows_not_exist_file/temp')) # save to memory byio = BytesIO() diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load.py b/python/paddle/fluid/tests/unittests/test_static_save_load.py index 9c44785d1c4..8a4f8f92013 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load.py @@ -30,6 +30,7 @@ import six import pickle import os import errno +import tempfile paddle.enable_static() @@ -240,6 +241,7 @@ class TestSaveLoadBase(unittest.TestCase): init_scale = 0.1 batch_size = 4 batch_num = 200 + temp_dir = tempfile.TemporaryDirectory() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -309,7 +311,7 @@ class TestSaveLoadBase(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - fluid.save(main_program, "./test_1") + fluid.save(main_program, os.path.join(temp_dir.name, "test_1")) # set var to zero for var in main_program.list_vars(): @@ -322,7 +324,8 @@ class TestSaveLoadBase(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, "./test_1.pdparams", exe) + fluid.load(main_program, + os.path.join(temp_dir.name, "test_1.pdparams"), exe) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -330,6 +333,7 @@ class TestSaveLoadBase(unittest.TestCase): var.name).get_tensor()) base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t)) + temp_dir.cleanup() class TestSaveLoadPartial(unittest.TestCase): @@ -347,6 +351,7 @@ class TestSaveLoadPartial(unittest.TestCase): init_scale = 0.1 batch_size = 4 batch_num = 200 + temp_dir = tempfile.TemporaryDirectory() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -424,7 +429,7 @@ class TestSaveLoadPartial(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - fluid.save(main_program, "./test_1") + fluid.save(main_program, os.path.join(temp_dir.name, "test_1")) # set var to zero for var in main_program.list_vars(): @@ -437,7 +442,8 @@ class TestSaveLoadPartial(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(test_program, "./test_1.pdopt", None) + fluid.load(test_program, os.path.join(temp_dir.name, + "test_1.pdopt"), None) for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -445,7 +451,9 @@ class TestSaveLoadPartial(unittest.TestCase): var.name).get_tensor()) base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t)) - fluid.load(test_program, "./test_1.pdmodel", None) + fluid.load(test_program, + os.path.join(temp_dir.name, "test_1.pdmodel"), None) + temp_dir.cleanup() class TestSaveLoadSetStateDict(unittest.TestCase): @@ -463,6 +471,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): init_scale = 0.1 batch_size = 4 batch_num = 200 + temp_dir = tempfile.TemporaryDirectory() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -532,7 +541,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - fluid.save(main_program, "./test_1") + fluid.save(main_program, os.path.join(temp_dir.name, "test_1")) # set var to zero for var in main_program.list_vars(): @@ -545,7 +554,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, "./test_1", exe) + fluid.load(main_program, os.path.join(temp_dir.name, "test_1"), exe) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -553,6 +562,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): var.name).get_tensor()) base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t)) + temp_dir.cleanup() class TestProgramStatePartial(unittest.TestCase): @@ -570,6 +580,7 @@ class TestProgramStatePartial(unittest.TestCase): init_scale = 0.1 batch_size = 4 batch_num = 200 + temp_dir = tempfile.TemporaryDirectory() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -647,7 +658,7 @@ class TestProgramStatePartial(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - fluid.save(main_program, os.path.join('some_dir', 'test_1')) + fluid.save(main_program, os.path.join(temp_dir.name, 'test_1')) # set var to zero for var in main_program.list_vars(): @@ -662,16 +673,16 @@ class TestProgramStatePartial(unittest.TestCase): #fluid.load(test_program, "./test_1", None ) program_state = fluid.load_program_state( - os.path.join('some_dir', 'test_1')) + os.path.join(temp_dir.name, 'test_1')) program_state_1 = fluid.load_program_state( - os.path.join('some_dir', 'test_1.pdparams')) + os.path.join(temp_dir.name, 'test_1.pdparams')) program_state_2 = fluid.load_program_state( - os.path.join('some_dir', 'test_1.pdopt')) + os.path.join(temp_dir.name, 'test_1.pdopt')) program_state_3 = fluid.load_program_state( - os.path.join('some_dir', 'test_1.pdmodel')) + os.path.join(temp_dir.name, 'test_1.pdmodel')) fluid.set_program_state(test_program, program_state) @@ -741,6 +752,7 @@ class TestProgramStatePartial(unittest.TestCase): var.name).get_tensor()) base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t)) + temp_dir.cleanup() class TestVariableInit(unittest.TestCase): @@ -759,7 +771,9 @@ class TestVariableInit(unittest.TestCase): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - fluid.save(fluid.default_main_program(), "./test_path") + temp_dir = tempfile.TemporaryDirectory() + fluid.save(fluid.default_main_program(), + os.path.join(temp_dir.name, "test_path")) def set_var(var, ndarray): t = var.get_tensor() @@ -785,7 +799,7 @@ class TestVariableInit(unittest.TestCase): fluid.core._create_loaded_parameter(parameter_list, new_scope, exe._default_executor) - parameter_file_name = "./test_path.pdparams" + parameter_file_name = os.path.join(temp_dir.name, "test_path.pdparams") with open(parameter_file_name, 'rb') as f: load_dict = pickle.load(f) @@ -801,7 +815,7 @@ class TestVariableInit(unittest.TestCase): fluid.core._create_loaded_parameter(opt_list, new_scope, exe._default_executor) - opt_file_name = "./test_path.pdopt" + opt_file_name = os.path.join(temp_dir.name, "test_path.pdopt") with open(opt_file_name, 'rb') as f: load_dict = pickle.load(f) @@ -827,6 +841,7 @@ class TestVariableInit(unittest.TestCase): base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t)) + temp_dir.cleanup() class TestLoadFromOldInterface(unittest.TestCase): @@ -838,10 +853,15 @@ class TestLoadFromOldInterface(unittest.TestCase): if os.path.exists("test_static_load_var_list.pdparams"): os.remove("test_static_load_var_list.pdparams") + self.temp_dir = tempfile.TemporaryDirectory() + def set_place(self): return fluid.CPUPlace( ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + def tearDown(self): + self.temp_dir.cleanup() + def test_load_from_old_interface(self): seed = 90 hidden_size = 10 @@ -923,7 +943,9 @@ class TestLoadFromOldInterface(unittest.TestCase): base_map[var.name] = t #fluid.save(main_program, "./test_1") - fluid.io.save_persistables(exe, "test_path", main_program) + fluid.io.save_persistables( + exe, os.path.join(self.temp_dir.name, "test_path"), + main_program) # set var to zero for var in main_program.list_vars(): @@ -936,7 +958,8 @@ class TestLoadFromOldInterface(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, "test_path", exe) + fluid.load(main_program, + os.path.join(self.temp_dir.name, "test_path"), exe) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -953,11 +976,13 @@ class TestLoadFromOldInterface(unittest.TestCase): var.desc.set_shape(new_shape) with self.assertRaises(RuntimeError): - fluid.load(main_program, "test_path", exe) + fluid.load(main_program, + os.path.join(self.temp_dir.name, "test_path"), exe) # check unused parameter - fluid.load(test_clone_program, "test_path", exe) + fluid.load(test_clone_program, + os.path.join(self.temp_dir.name, "test_path"), exe) def test_load_from_old_interface_var_list(self): seed = 90 @@ -1040,8 +1065,10 @@ class TestLoadFromOldInterface(unittest.TestCase): base_map[var.name] = t #fluid.save(main_program, "./test_1") - fluid.io.save_persistables(exe, "test_static_load_var_list", - main_program) + fluid.io.save_persistables( + exe, + os.path.join(self.temp_dir.name, "test_static_load_var_list"), + main_program) # set var to zero var_list = [] @@ -1057,7 +1084,10 @@ class TestLoadFromOldInterface(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, "test_static_load_var_list", exe, var_list) + fluid.load( + main_program, + os.path.join(self.temp_dir.name, "test_static_load_var_list"), + exe, var_list) var_list_names = [var.name for var in var_list] for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -1087,6 +1117,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): init_scale = 0.1 batch_size = 4 batch_num = 200 + temp_dir = tempfile.TemporaryDirectory() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -1155,10 +1186,10 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - + save_dir = os.path.join(temp_dir.name, "test_path") #fluid.save(main_program, "./test_1") fluid.io.save_persistables(exe, - "test_path", + save_dir, main_program, filename="model_single") @@ -1173,7 +1204,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - file_model_path = os.path.join("test_path", "model_single") + file_model_path = os.path.join(save_dir, "model_single") fluid.load(main_program, file_model_path, exe, fluid.io.get_program_persistable_vars(main_program)) @@ -1223,12 +1254,17 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): all_var_list = list(main_program.list_vars()) fluid.load(main_program, file_model_path, exe, all_var_list + [temp_var]) + temp_dir.cleanup() class TestProgramStateOldSave(unittest.TestCase): def setUp(self): self.test_dygraph = True + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def set_place(self): return fluid.CPUPlace( @@ -1319,8 +1355,8 @@ class TestProgramStateOldSave(unittest.TestCase): # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - - fluid.io.save_persistables(exe, "test_program_1", main_program) + save_dir = os.path.join(self.temp_dir.name, "test_program_1") + fluid.io.save_persistables(exe, save_dir, main_program) # set var to zero for var in main_program.list_vars(): @@ -1334,7 +1370,7 @@ class TestProgramStateOldSave(unittest.TestCase): self.assertTrue(np.sum(np.abs(new_t)) == 0) # case 1: load basic - program_state = fluid.load_program_state("test_program_1") + program_state = fluid.load_program_state(save_dir) fluid.set_program_state(main_program, program_state) self.check_in_static(main_program, base_map) @@ -1349,24 +1385,20 @@ class TestProgramStateOldSave(unittest.TestCase): else: raise e - orig_filepath = './test_program_1/fc_0.w_0' - symlink_filepath = './test_program_1/link_fc_0.w_0' - # create a needless link file for coverage - symlink_force(orig_filepath, symlink_filepath) - program_state = fluid.load_program_state("test_program_1") + program_state = fluid.load_program_state(save_dir) fluid.set_program_state(main_program, program_state) self.check_in_static(main_program, base_map) # case 3: load with var_list program_state = fluid.load_program_state( - "test_program_1", main_program.all_parameters()) + save_dir, main_program.all_parameters()) fluid.set_program_state(main_program, program_state) self.check_in_static(main_program, base_map) if self.test_dygraph: # make sure `load_program_state` can be used in dynamic graph mode with fluid.dygraph.guard(place): - load_state = fluid.load_program_state("test_program_1") + load_state = fluid.load_program_state(save_dir) for k, v in load_state.items(): self.assertTrue(np.array_equal(base_map[k], v)) @@ -1402,6 +1434,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): init_scale = 0.1 batch_size = 4 batch_num = 200 + temp_dir = tempfile.TemporaryDirectory() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -1479,8 +1512,9 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t + save_dir = os.path.join(temp_dir.name, "test_program_2") fluid.io.save_persistables(exe, - "test_program_2", + save_dir, main_program, filename="model_1") @@ -1497,7 +1531,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): #fluid.load(test_program, "./test_1", None ) program_state = fluid.load_program_state( - os.path.join("test_program_2", "model_1"), + os.path.join(save_dir, "model_1"), var_list=fluid.io.get_program_persistable_vars(main_program)) fluid.set_program_state(main_program, program_state) @@ -1509,21 +1543,20 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): self.assertTrue(np.array_equal(new_t, base_t)) with self.assertRaises(ValueError): - fluid.load_program_state( - os.path.join("test_program_2", "model_1")) + fluid.load_program_state(os.path.join(save_dir, "model_1")) with self.assertRaises(TypeError): - fluid.load_program_state(os.path.join("test_program_2", - "model_1"), + fluid.load_program_state(os.path.join(save_dir, "model_1"), var_list=["str"]) with self.assertRaises(RuntimeError): fluid.load_program_state( - os.path.join("test_program_2", "model_1"), + os.path.join(save_dir, "model_1"), var_list=[ main_program.global_block().create_var( name="fake_var_name", persistable=True) ]) + temp_dir.cleanup() class TestStaticSaveLoadPickle(unittest.TestCase): @@ -1552,7 +1585,8 @@ class TestStaticSaveLoadPickle(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - path = os.path.join("test_static_save_load_pickle", + temp_dir = tempfile.TemporaryDirectory() + path = os.path.join(temp_dir.name, "test_static_save_load_pickle", "pickle_protocol") with self.assertRaises(ValueError): diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py b/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py index 25619aa4a5c..6da849a44bd 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py @@ -23,12 +23,20 @@ from paddle.fluid.optimizer import SGDOptimizer from paddle.fluid.tests.unittests.test_imperative_base import new_program_scope from paddle.fluid.tests.unittests.test_static_save_load import PtbModel import numpy as np +import tempfile +import os @unittest.skipIf(not core.supports_bfloat16(), "place does not support BF16 evaluation") class TestSaveLoadBF16(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def set_place(self): return fluid.CPUPlace() @@ -111,8 +119,8 @@ class TestSaveLoadBF16(unittest.TestCase): # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - - fluid.save(main_program, "./test_1") + save_dir = os.path.join(self.temp_dir.name, "test_1") + fluid.save(main_program, save_dir) # set var to zero for var in main_program.list_vars(): @@ -125,7 +133,8 @@ class TestSaveLoadBF16(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, "./test_1.pdparams", exe) + fluid.load(main_program, + os.path.join(self.temp_dir.name, "test_1.pdparams"), exe) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load_large.py b/python/paddle/fluid/tests/unittests/test_static_save_load_large.py index fdb6a1f2f05..e45cd59b444 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load_large.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load_large.py @@ -23,6 +23,7 @@ from test_imperative_base import new_program_scope import numpy as np import pickle import os +import tempfile LARGE_PARAM = 2**26 @@ -51,9 +52,10 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - - path = os.path.join("test_static_save_load_large_param", - "static_save") + temp_dir = tempfile.TemporaryDirectory() + path = os.path.join(temp_dir.name, + "test_static_save_load_large_param") + path = os.path.join(path, "static_save") protocol = 4 paddle.fluid.save(prog, path, pickle_protocol=protocol) # set var to zero @@ -93,6 +95,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): var.name).get_tensor()) base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t)) + temp_dir.cleanup() if __name__ == '__main__': -- GitLab