From 2eb60ddb2932fbfc3b937a063bd6c57e7ad65c54 Mon Sep 17 00:00:00 2001 From: YuanRisheng Date: Fri, 17 Jun 2022 14:10:53 +0800 Subject: [PATCH] cherry pick 43581 (#43596) --- .../unittests/test_faster_tokenizer_op.py | 8 +- .../test_imperative_load_static_param.py | 10 +- .../unittests/test_imperative_save_load_v2.py | 43 +++- .../tests/unittests/test_io_save_load.py | 30 ++- .../tests/unittests/test_jit_save_load.py | 236 ++++++++++++++---- .../fluid/tests/unittests/test_load_op.py | 13 +- .../test_load_state_dict_from_old_format.py | 21 +- .../tests/unittests/test_paddle_save_load.py | 119 ++++++--- .../unittests/test_paddle_save_load_binary.py | 41 ++- .../tests/unittests/test_static_save_load.py | 120 +++++---- .../unittests/test_static_save_load_bf16.py | 15 +- .../unittests/test_static_save_load_large.py | 9 +- 12 files changed, 501 insertions(+), 164 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py b/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py index 190345958e..504ad7bfe6 100755 --- a/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py +++ b/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py @@ -27,6 +27,8 @@ from paddle.fluid.layer_helper import LayerHelper from paddle import _C_ops import sys +import tempfile + sys.path.append("./tokenizer") from tokenizer.bert_tokenizer import BertTokenizer @@ -149,13 +151,17 @@ class Predictor(object): class TestBertTokenizerOp(unittest.TestCase): def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() self.bert_tokenizer = BertTokenizer.from_pretrained("bert-base-chinese") self.faster_tokenizer = FasterTokenizer(self.bert_tokenizer.vocab) self.init_data() - self.save_path = os.path.join(DATA_HOME, "fast_tokenizer") + self.save_path = os.path.join(self.temp_dir.name, "fast_tokenizer") self.param_path = os.path.join(self.save_path, "model.pdparams") self.inference_path = os.path.join(self.save_path, "inference") + def tearDown(self): + self.temp_dir.cleanup() + def init_data(self): self.text = [ '选择珠江花园的原因就是方便，有电动扶梯直接到达海边，周围餐馆、食廊、商场、超市、摊位一应俱全。' diff --git a/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py b/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py index e4faa7e259..27ddff262c 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py @@ -17,11 +17,14 @@ import paddle.fluid as fluid import paddle.fluid.framework as framework from paddle.fluid.dygraph.nn import * import numpy as np +import os +import tempfile class TestDygraphLoadStatic(unittest.TestCase): def testLoadStaticModel(self): # static mode + temp_dir = tempfile.TemporaryDirectory() a = fluid.data(name="a", shape=[10, 10]) conv_in = fluid.data(name="conv_in", shape=[None, 10, 10, 10]) @@ -121,9 +124,11 @@ class TestDygraphLoadStatic(unittest.TestCase): ) if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0)) out = exe.run(framework.default_startup_program()) - fluid.save(framework.default_main_program(), "./test_1") + fluid.save(framework.default_main_program(), + os.path.join(temp_dir.name, "test_1")) - para_dict = fluid.load_program_state("./test_1") + para_dict = fluid.load_program_state( + os.path.join(temp_dir.name, "test_1")) new_dict = {} for k, v in para_dict.items(): @@ -192,6 +197,7 @@ class TestDygraphLoadStatic(unittest.TestCase): my_test.set_dict(new_dict, use_structured_name=False) for k, v in my_test.state_dict().items(): self.assertTrue(np.array_equal(v.numpy(), new_dict[v.name])) + temp_dir.cleanup() if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py b/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py index 7e7b2e2fd5..9ed2b90424 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py @@ -29,6 +29,8 @@ import six import paddle from paddle.fluid.framework import _test_eager_guard +import tempfile + class SimpleLSTMRNN(fluid.Layer): def __init__(self, @@ -209,6 +211,12 @@ class PtbModel(fluid.Layer): class TestDygraphPtbRnn(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def func_setUp(self): seed = 90 hidden_size = 10 @@ -286,7 +294,8 @@ class TestDygraphPtbRnn(unittest.TestCase): else: self.base_opti[k] = v - paddle.save(self.opti_dict, "./test_dy_v2.pdopt") + paddle.save(self.opti_dict, + os.path.join(self.temp_dir.name, "test_dy_v2.pdopt")) self.state_dict = ptb_model.state_dict() @@ -295,10 +304,10 @@ class TestDygraphPtbRnn(unittest.TestCase): np_t = v.numpy() self.model_base[k] = np_t - paddle.save(self.state_dict, "./test_dy_v2.pdparams") + paddle.save(self.state_dict, + os.path.join(self.temp_dir.name, "test_dy_v2.pdparams")) def func_testLoadAndSetVarBase(self): - self.setUp() seed = 90 hidden_size = 10 vocab_size = 1000 @@ -375,8 +384,10 @@ class TestDygraphPtbRnn(unittest.TestCase): self.assertTrue(np.sum(np.abs(v.numpy())) == 0) - para_state_dict = paddle.load("./test_dy_v2.pdparams") - opti_state_dict = paddle.load("./test_dy_v2.pdopt") + para_state_dict = paddle.load( + os.path.join(self.temp_dir.name, "test_dy_v2.pdparams")) + opti_state_dict = paddle.load( + os.path.join(self.temp_dir.name, "test_dy_v2.pdopt")) adam.set_state_dict(opti_state_dict) opti_dict = adam.state_dict() @@ -750,7 +761,8 @@ class TestDygraphPtbRnn(unittest.TestCase): last_hidden = None last_cell = None - state_dict, opti_dict = fluid.load_dygraph("./test_dy_v2") + state_dict, opti_dict = fluid.load_dygraph( + os.path.join(self.temp_dir.name, "test_dy_v2")) adam.set_state_dict(opti_dict) ptb_model.set_dict(state_dict) @@ -907,19 +919,23 @@ class TestDygraphPtbRnn(unittest.TestCase): with fluid.dygraph.guard(): emb = fluid.dygraph.Embedding([10, 10]) state_dict = emb.state_dict() - paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams')) + paddle.save( + state_dict, + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) para_state_dict = paddle.load( - os.path.join('saved_dy', 'emb_dy.pdparams')) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) def func_test_no_state_in_input_dict(self): with fluid.dygraph.guard(): emb = fluid.dygraph.Embedding([10, 10]) state_dict = emb.state_dict() - paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams')) + paddle.save( + state_dict, + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) para_state_dict = paddle.load( - os.path.join('saved_dy', 'emb_dy.pdparams')) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) para_state_dict.pop('weight') emb.set_state_dict(para_state_dict) @@ -928,10 +944,13 @@ class TestDygraphPtbRnn(unittest.TestCase): with fluid.dygraph.guard(): emb = fluid.dygraph.Embedding([10, 10]) state_dict = emb.state_dict() - paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams')) + paddle.save( + state_dict, + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) para_state_dict = paddle.load( - os.path.join('saved_dy', 'emb_dy.pdparams'), return_numpy=True) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'), + return_numpy=True) para_state_dict['weight'] = np.expand_dims( para_state_dict['weight'], axis=-1) diff --git a/python/paddle/fluid/tests/unittests/test_io_save_load.py b/python/paddle/fluid/tests/unittests/test_io_save_load.py index a9a223f8f9..efdc93c10f 100644 --- a/python/paddle/fluid/tests/unittests/test_io_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_io_save_load.py @@ -19,9 +19,18 @@ import paddle import paddle.fluid as fluid from paddle.fluid import core from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph +import tempfile +import os class TestSaveLoadAPIError(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.save_dir = os.path.join(self.temp_dir.name, "fake_dir") + + def tearDown(self): + self.temp_dir.cleanup() + def func_test_get_valid_program_error(self): # case 1: CompiledProgram no program graph = core.Graph(core.ProgramDesc()) @@ -44,13 +53,13 @@ class TestSaveLoadAPIError(unittest.TestCase): # case 1: main_program type error when vars None with self.assertRaises(TypeError): fluid.io.load_vars( - executor=exe, dirname="./fake_dir", main_program="program") + executor=exe, dirname=self.save_dir, main_program="program") # case 2: main_program type error when vars not None with self.assertRaises(TypeError): fluid.io.load_vars( executor=exe, - dirname="./fake_dir", + dirname=self.save_dir, main_program="program", vars="vars") @@ -61,6 +70,12 @@ class TestSaveLoadAPIError(unittest.TestCase): class TestSaveInferenceModelAPIError(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def func_test_useless_feeded_var_names(self): start_prog = fluid.Program() main_prog = fluid.Program() @@ -74,7 +89,7 @@ class TestSaveInferenceModelAPIError(unittest.TestCase): with self.assertRaisesRegexp( ValueError, "not involved in the target_vars calculation"): fluid.io.save_inference_model( - dirname='./model', + dirname=os.path.join(self.temp_dir.name, 'model'), feeded_var_names=['x', 'y'], target_vars=[z], executor=exe, @@ -87,13 +102,20 @@ class TestSaveInferenceModelAPIError(unittest.TestCase): class TestWhenTrainWithNoGrad(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def func_test_when_train_with_no_grad(self): paddle.disable_static() net = paddle.nn.Linear(1024, 1) net = paddle.jit.to_static(net) x = paddle.rand([1024], 'float32') net(x) - save_path = './train_with_no_grad' + save_path = os.path.join(self.temp_dir.name, 'train_with_no_grad') + paddle.jit.save(net, save_path) net = paddle.jit.load(save_path) net.train() diff --git a/python/paddle/fluid/tests/unittests/test_jit_save_load.py b/python/paddle/fluid/tests/unittests/test_jit_save_load.py index 5dabf85473..e0ed00cb1c 100644 --- a/python/paddle/fluid/tests/unittests/test_jit_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_jit_save_load.py @@ -18,6 +18,7 @@ import os import pickle import shutil import unittest +import tempfile import numpy as np import paddle from paddle.static import InputSpec @@ -333,13 +334,18 @@ def train_with_label(layer, input_size=784, label_size=1): class TestJitSaveLoad(unittest.TestCase): def setUp(self): - self.model_path = "test_jit_save_load/model" + self.temp_dir = tempfile.TemporaryDirectory() + self.model_path = os.path.join(self.temp_dir.name, + "test_jit_save_load/model") # enable dygraph mode fluid.enable_dygraph() # config seed paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) + def tearDown(self): + self.temp_dir.cleanup() + def train_and_save_model(self, model_path=None): layer = LinearNet(784, 1) example_inputs, layer, _ = train(layer) @@ -395,12 +401,14 @@ class TestJitSaveLoad(unittest.TestCase): np.array_equal(train_layer(x).numpy(), new_layer(x).numpy())) def test_load_dygraph_no_path(self): - model_path = "test_jit_save_load.no_path/model_path" + model_path = os.path.join(self.temp_dir.name, + "test_jit_save_load.no_path/model_path") with self.assertRaises(ValueError): model_dict, _ = fluid.dygraph.load_dygraph(model_path) def test_jit_load_no_path(self): - path = "test_jit_save_load.no_path/model_path" + path = os.path.join(self.temp_dir.name, + "test_jit_save_load.no_path/model_path") with self.assertRaises(ValueError): loaded_layer = paddle.jit.load(path) @@ -409,6 +417,10 @@ class TestSaveLoadWithNestOut(unittest.TestCase): def setUp(self): # enable dygraph mode fluid.enable_dygraph() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_nest_output(self): x = fluid.dygraph.to_variable( @@ -418,7 +430,7 @@ class TestSaveLoadWithNestOut(unittest.TestCase): dy_outs = flatten(net(x)) net = declarative(net, input_spec=[InputSpec([None, 8], name='x')]) - model_path = "net_with_nest_out/model" + model_path = os.path.join(self.temp_dir.name, "net_with_nest_out/model") paddle.jit.save(net, model_path) load_net = paddle.jit.load(model_path) @@ -440,8 +452,9 @@ class TestSaveLoadWithDictInput(unittest.TestCase): # {'img': var img : fluid.VarType.LOD_TENSOR.shape(-1, 8).astype(VarType.FP32)}, # {'label': var label : fluid.VarType.LOD_TENSOR.shape(-1, 1).astype(VarType.INT64)}) self.assertEqual(len(net.forward.concrete_program.inputs), 3) - - path = "test_jit_save_load_with_dict_input/model" + temp_dir = tempfile.TemporaryDirectory() + path = os.path.join(temp_dir.name, + "test_jit_save_load_with_dict_input/model") # prune inputs paddle.jit.save( layer=net, @@ -458,13 +471,15 @@ class TestSaveLoadWithDictInput(unittest.TestCase): # loaded_net._input_spec(): # [InputSpec(shape=(-1, 8), dtype=VarType.FP32, name=img)] self.assertEqual(len(loaded_net._input_spec()), 1) + temp_dir.cleanup() class TestSaveLoadWithDictInputNoPrune(unittest.TestCase): def test_dict_input(self): net = LinearNetWithDictInputNoPrune(8, 8) - - path = "test_jit_save_load_with_dict_input_no_prune/model" + temp_dir = tempfile.TemporaryDirectory() + path = os.path.join(temp_dir.name, + "test_jit_save_load_with_dict_input_no_prune/model") # prune inputs paddle.jit.save( layer=net, @@ -482,12 +497,17 @@ class TestSaveLoadWithDictInputNoPrune(unittest.TestCase): loaded_out = loaded_net(img, img2) self.assertEqual(len(loaded_net._input_spec()), 2) + temp_dir.cleanup() class TestSaveLoadWithInputSpec(unittest.TestCase): def setUp(self): # enable dygraph mode fluid.enable_dygraph() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_with_input_spec(self): net = LinearNetReturnLoss(8, 8) @@ -496,7 +516,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): net.forward, input_spec=[InputSpec( [None, 8], name='x')]) - model_path = "input_spec.output_spec/model" + model_path = os.path.join(self.temp_dir.name, + "input_spec.output_spec/model") # check inputs and outputs self.assertTrue(len(net.forward.inputs) == 1) input_x = net.forward.inputs[0] @@ -516,7 +537,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): def test_multi_in_out(self): net = LinearNetMultiInput(8, 8) - model_path = "multi_inout.output_spec1/model" + model_path = os.path.join(self.temp_dir.name, + "multi_inout.output_spec1/model") # 1. check inputs and outputs self.assertTrue(len(net.forward.inputs) == 2) input_x = net.forward.inputs[0] @@ -538,7 +560,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): pred_x, pred_y = infer_layer(x, y) # 1. prune y and loss - model_path = "multi_inout.output_spec2/model" + model_path = os.path.join(self.temp_dir.name, + "multi_inout.output_spec2/model") output_spec = net.forward.outputs[:1] paddle.jit.save(net, model_path, [input_x], output_spec=output_spec) # 2. load again @@ -552,7 +575,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): def test_multi_in_out1(self): net = LinearNetMultiInput1(8, 8) - model_path = "multi_inout1.output_spec1/model" + model_path = os.path.join(self.temp_dir.name, + "multi_inout1.output_spec1/model") # 1. check inputs and outputs self.assertTrue(len(net.forward.inputs) == 2) input_x = net.forward.inputs[0] @@ -574,7 +598,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): pred_x, pred_y = infer_layer(x, y) # 1. prune y and loss - model_path = "multi_inout1.output_spec2/model" + model_path = os.path.join(self.temp_dir.name, + "multi_inout1.output_spec2/model") output_spec = net.forward.outputs[:1] paddle.jit.save(net, model_path, (input_x, ), output_spec=output_spec) # 2. load again @@ -593,6 +618,10 @@ class TestJitSaveLoadConfig(unittest.TestCase): # config seed paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_output_spec(self): train_layer = LinearNetReturnLoss(8, 8) @@ -606,7 +635,8 @@ class TestJitSaveLoadConfig(unittest.TestCase): adam.minimize(loss) train_layer.clear_gradients() - model_path = "save_load_config.output_spec" + model_path = os.path.join(self.temp_dir.name, + "save_load_config.output_spec") output_spec = [out] paddle.jit.save( layer=train_layer, @@ -623,22 +653,22 @@ class TestJitSaveLoadConfig(unittest.TestCase): def test_save_no_support_config_error(self): layer = LinearNet(784, 1) - path = "no_support_config_test" + path = os.path.join(self.temp_dir.name, "no_support_config_test") with self.assertRaises(ValueError): paddle.jit.save(layer=layer, path=path, model_filename="") def test_load_empty_model_filename_error(self): - path = "error_model_filename_test" + path = os.path.join(self.temp_dir.name, "error_model_filename_test") with self.assertRaises(ValueError): paddle.jit.load(path, model_filename="") def test_load_empty_params_filename_error(self): - path = "error_params_filename_test" + path = os.path.join(self.temp_dir.name, "error_params_filename_test") with self.assertRaises(ValueError): paddle.jit.load(path, params_filename="") def test_load_with_no_support_config(self): - path = "no_support_config_test" + path = os.path.join(self.temp_dir.name, "no_support_config_test") with self.assertRaises(ValueError): paddle.jit.load(path, separate_params=True) @@ -646,7 +676,9 @@ class TestJitSaveLoadConfig(unittest.TestCase): class TestJitMultipleLoading(unittest.TestCase): def setUp(self): self.linear_size = 4 - self.model_path = "jit_multi_load/model" + self.temp_dir = tempfile.TemporaryDirectory() + self.model_path = os.path.join(self.temp_dir.name, + "jit_multi_load/model") # enable dygraph mode fluid.enable_dygraph() # config seed @@ -655,6 +687,9 @@ class TestJitMultipleLoading(unittest.TestCase): # train and save base model self.train_and_save_orig_model() + def tearDown(self): + self.temp_dir.cleanup() + def train_and_save_orig_model(self): layer = LinearNet(self.linear_size, self.linear_size) example_inputs, layer, _ = train(layer, self.linear_size, 1) @@ -674,13 +709,18 @@ class TestJitMultipleLoading(unittest.TestCase): class TestJitPruneModelAndLoad(unittest.TestCase): def setUp(self): self.linear_size = 4 - self.model_path = "jit_prune_model_and_load/model" + self.temp_dir = tempfile.TemporaryDirectory() + self.model_path = os.path.join(self.temp_dir.name, + "jit_prune_model_and_load/model") # enable dygraph mode fluid.enable_dygraph() # config seed paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) + def tearDown(self): + self.temp_dir.cleanup() + def train_and_save(self): train_layer = LinearNetReturnHidden(8, 8) adam = fluid.optimizer.AdamOptimizer( @@ -735,6 +775,10 @@ class TestJitSaveMultiCases(unittest.TestCase): # config seed paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def verify_inference_correctness(self, layer, @@ -767,7 +811,8 @@ class TestJitSaveMultiCases(unittest.TestCase): train(layer) - model_path = "test_no_prune_to_static_after_train/model" + model_path = os.path.join(self.temp_dir.name, + "test_no_prune_to_static_after_train/model") paddle.jit.save(layer, model_path) self.verify_inference_correctness(layer, model_path) @@ -775,7 +820,8 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_no_prune_to_static_no_train(self): layer = LinearNetWithInputSpec(784, 1) - model_path = "test_no_prune_to_static_no_train/model" + model_path = os.path.join(self.temp_dir.name, + "test_no_prune_to_static_no_train/model") paddle.jit.save(layer, model_path) self.verify_inference_correctness(layer, model_path) @@ -785,7 +831,8 @@ class TestJitSaveMultiCases(unittest.TestCase): train(layer) - model_path = "test_no_prune_no_to_static_after_train/model" + model_path = os.path.join( + self.temp_dir.name, "test_no_prune_no_to_static_after_train/model") paddle.jit.save( layer, model_path, @@ -799,7 +846,9 @@ class TestJitSaveMultiCases(unittest.TestCase): example_inputs, _, _ = train(layer) - model_path = "test_no_prune_no_to_static_after_train_with_examples/model" + model_path = os.path.join( + self.temp_dir.name, + "test_no_prune_no_to_static_after_train_with_examples/model") paddle.jit.save(layer=layer, path=model_path, input_spec=example_inputs) self.verify_inference_correctness(layer, model_path) @@ -807,7 +856,8 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_no_prune_no_to_static_no_train(self): layer = LinearNetNotDeclarative(784, 1) - model_path = "test_no_prune_no_to_static_no_train/model" + model_path = os.path.join(self.temp_dir.name, + "test_no_prune_no_to_static_no_train/model") paddle.jit.save( layer, model_path, @@ -821,7 +871,8 @@ class TestJitSaveMultiCases(unittest.TestCase): out = train_with_label(layer) - model_path = "test_prune_to_static_after_train/model" + model_path = os.path.join(self.temp_dir.name, + "test_prune_to_static_after_train/model") paddle.jit.save( layer, model_path, @@ -837,7 +888,8 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_prune_to_static_no_train(self): layer = LinerNetWithLabel(784, 1) - model_path = "test_prune_to_static_no_train/model" + model_path = os.path.join(self.temp_dir.name, + "test_prune_to_static_no_train/model") # TODO: no train, cannot get output_spec var here # now only can use index output_spec = layer.forward.outputs[:1] @@ -856,7 +908,8 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_prune_input_to_static_no_train(self): layer = LinerNetWithPruneInput(784, 1) - model_path = "test_prune_input_to_static_no_train/model" + model_path = os.path.join(self.temp_dir.name, + "test_prune_input_to_static_no_train/model") paddle.jit.save( layer, model_path, @@ -870,7 +923,9 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_prune_useless_input_to_static_no_train(self): layer = LinerNetWithUselessInput(784, 1) - model_path = "test_prune_useless_input_to_static_no_train/model" + model_path = os.path.join( + self.temp_dir.name, + "test_prune_useless_input_to_static_no_train/model") paddle.jit.save( layer, model_path, @@ -886,7 +941,8 @@ class TestJitSaveMultiCases(unittest.TestCase): train(layer) - model_path = "test_no_prune_input_spec_name_warning/model" + model_path = os.path.join(self.temp_dir.name, + "test_no_prune_input_spec_name_warning/model") paddle.jit.save( layer, model_path, @@ -907,7 +963,8 @@ class TestJitSaveMultiCases(unittest.TestCase): train(layer) - model_path = "test_not_prune_output_spec_name_warning/model" + model_path = os.path.join( + self.temp_dir.name, "test_not_prune_output_spec_name_warning/model") out = paddle.to_tensor(np.random.random((1, 1)).astype('float')) paddle.jit.save(layer, model_path, output_spec=[out]) @@ -916,7 +973,8 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_prune_input_spec_name_error(self): layer = LinerNetWithLabel(784, 1) - model_path = "test_prune_input_spec_name_error/model" + model_path = os.path.join(self.temp_dir.name, + "test_prune_input_spec_name_error/model") with self.assertRaises(ValueError): paddle.jit.save( layer, @@ -937,7 +995,8 @@ class TestJitSaveMultiCases(unittest.TestCase): train_with_label(layer) - model_path = "test_prune_to_static_after_train/model" + model_path = os.path.join(self.temp_dir.name, + "test_prune_to_static_after_train/model") out = paddle.to_tensor(np.random.random((1, 1)).astype('float')) with self.assertRaises(ValueError): paddle.jit.save( @@ -952,10 +1011,15 @@ class TestJitSaveMultiCases(unittest.TestCase): class TestJitSaveLoadEmptyLayer(unittest.TestCase): def setUp(self): - self.model_path = "jit_save_load_empty_layer/model" + self.temp_dir = tempfile.TemporaryDirectory() + self.model_path = os.path.join(self.temp_dir.name, + "jit_save_load_empty_layer/model") # enable dygraph mode paddle.disable_static() + def tearDown(self): + self.temp_dir.cleanup() + def test_save_load_empty_layer(self): layer = EmptyLayer() x = paddle.to_tensor(np.random.random((10)).astype('float32')) @@ -968,10 +1032,15 @@ class TestJitSaveLoadEmptyLayer(unittest.TestCase): class TestJitSaveLoadNoParamLayer(unittest.TestCase): def setUp(self): - self.model_path = "jit_save_load_no_param_layer/model" + self.temp_dir = tempfile.TemporaryDirectory() + self.model_path = os.path.join(self.temp_dir.name, + "jit_save_load_no_param_layer/model") # enable dygraph mode paddle.disable_static() + def tearDown(self): + self.temp_dir.cleanup() + def test_save_load_no_param_layer(self): layer = NoParamLayer() x = paddle.to_tensor(np.random.random((5)).astype('float32')) @@ -987,9 +1056,14 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase): def setUp(self): # enable dygraph mode paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_jit_save_load_inference(self): - model_path_inference = "jit_save_load_multi_methods/model" + model_path_inference = os.path.join(self.temp_dir.name, + "jit_save_load_multi_methods/model") IMAGE_SIZE = 224 layer = LinearNetWithMultiStaticFunc(IMAGE_SIZE, 10) inps = paddle.randn([1, IMAGE_SIZE]) @@ -1005,14 +1079,16 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase): )) < 1e-5) def test_jit_save_load_multi_methods_inputspec(self): - model_path = 'jit_save_load_multi_methods/model' + model_path = os.path.join(self.temp_dir.name, + 'jit_save_load_multi_methods/model') layer = LinearNetWithMultiStaticFunc(784, 1) with self.assertRaises(ValueError): paddle.jit.save( layer, model_path, input_spec=[InputSpec(shape=[None, 784])]) def test_parse_name(self): - model_path_inference = "jit_save_load_parse_name/model" + model_path_inference = os.path.join(self.temp_dir.name, + "jit_save_load_parse_name/model") IMAGE_SIZE = 224 layer = LinearNet(IMAGE_SIZE, 1) inps = paddle.randn([1, IMAGE_SIZE]) @@ -1082,9 +1158,14 @@ class TestJitSaveLoadSaveWithoutRunning(unittest.TestCase): def setUp(self): # enable dygraph mode paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_save_load_finetune_load(self): - model_path = "test_jit_save_load_save_without_running/model" + model_path = os.path.join( + self.temp_dir.name, "test_jit_save_load_save_without_running/model") IMAGE_SIZE = 224 inps0 = paddle.randn([1, IMAGE_SIZE]) inps1 = paddle.randn([2, IMAGE_SIZE]) @@ -1124,9 +1205,14 @@ class TestJitSaveLoadFinetuneLoad(unittest.TestCase): def setUp(self): # enable dygraph mode paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_save_load_finetune_load(self): - model_path = "test_jit_save_load_finetune_load/model" + model_path = os.path.join(self.temp_dir.name, + "test_jit_save_load_finetune_load/model") IMAGE_SIZE = 224 inps0 = paddle.randn([1, IMAGE_SIZE]) inps1 = paddle.randn([2, IMAGE_SIZE]) @@ -1162,13 +1248,18 @@ class TestJitSaveLoadFinetuneLoad(unittest.TestCase): class TestJitSaveLoadFunctionCase1(unittest.TestCase): def setUp(self): paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_jit_save_load_static_function(self): @paddle.jit.to_static def fun(inputs): return paddle.tanh(inputs) - path = 'test_jit_save_load_function_1/func' + path = os.path.join(self.temp_dir.name, + 'test_jit_save_load_function_1/func') inps = paddle.rand([3, 6]) origin = fun(inps) @@ -1182,6 +1273,10 @@ class TestJitSaveLoadFunctionCase1(unittest.TestCase): class TestJitSaveLoadFunctionCase2(unittest.TestCase): def setUp(self): paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_jit_save_load_function_input_spec(self): @paddle.jit.to_static(input_spec=[ @@ -1191,7 +1286,8 @@ class TestJitSaveLoadFunctionCase2(unittest.TestCase): def fun(inputs): return paddle.nn.functional.relu(inputs) - path = 'test_jit_save_load_function_2/func' + path = os.path.join(self.temp_dir.name, + 'test_jit_save_load_function_2/func') inps = paddle.rand([3, 6]) origin = fun(inps) @@ -1204,12 +1300,17 @@ class TestJitSaveLoadFunctionCase2(unittest.TestCase): class TestJitSaveLoadFunctionCase3(unittest.TestCase): def setUp(self): paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_jit_save_load_function_function(self): def fun(inputs): return paddle.tanh(inputs) - path = 'test_jit_save_load_function_3/func' + path = os.path.join(self.temp_dir.name, + 'test_jit_save_load_function_3/func') inps = paddle.rand([3, 6]) origin = fun(inps) @@ -1229,6 +1330,10 @@ class TestJitSaveLoadFunctionCase3(unittest.TestCase): class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase): def setUp(self): paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_jit_save_load_function(self): class LinearNet(paddle.nn.Layer): @@ -1249,7 +1354,9 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase): func = paddle.jit.to_static( layer.anothor_forward, [paddle.static.InputSpec(shape=[-1, 5])]) - path = 'test_jit_save_load_function_with_params_case1/func' + path = os.path.join( + self.temp_dir.name, + 'test_jit_save_load_function_with_params_case1/func') paddle.jit.save(func, path) load_func = paddle.jit.load(path) @@ -1260,6 +1367,10 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase): class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase): def setUp(self): paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_jit_save_load_function(self): class LinearNet(paddle.nn.Layer): @@ -1278,7 +1389,9 @@ class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase): inps = paddle.rand([3, 5]) - path = 'test_jit_save_load_function_with_params_case2/func' + path = os.path.join( + self.temp_dir.name, + 'test_jit_save_load_function_with_params_case2/func') paddle.jit.save(layer.anothor_forward, path) origin_result = layer.anothor_forward(inps) load_func = paddle.jit.load(path) @@ -1292,6 +1405,10 @@ class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase): class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase): def setUp(self): paddle.disable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_jit_save_load_function(self): class LinearNet(paddle.nn.Layer): @@ -1311,7 +1428,9 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase): inps = paddle.rand([3, 5]) origin = layer.anothor_forward(inps) - path = 'test_jit_save_load_function_with_params_case3/func' + path = os.path.join( + self.temp_dir.name, + 'test_jit_save_load_function_with_params_case3/func') paddle.jit.save(layer.anothor_forward, path) load_func = paddle.jit.load(path) @@ -1320,6 +1439,12 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase): class TestJitSaveLoadDataParallel(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def verify_inference_correctness(self, layer, path): layer.eval() loaded_layer = paddle.jit.load(path) @@ -1336,8 +1461,8 @@ class TestJitSaveLoadDataParallel(unittest.TestCase): def test_jit_save_data_parallel_with_inputspec(self): layer = LinearNetNotDeclarative(784, 1) layer = paddle.DataParallel(layer) - - path = "jit_save_data_parallel_with_inputspec/model" + path = os.path.join(self.temp_dir.name, + "jit_save_data_parallel_with_inputspec/model") paddle.jit.save( layer=layer, path=path, input_spec=[InputSpec(shape=[None, 784])]) @@ -1347,7 +1472,8 @@ class TestJitSaveLoadDataParallel(unittest.TestCase): layer = LinearNetWithInputSpec(784, 1) layer = paddle.DataParallel(layer) - path = "jit_save_data_parallel_with_to_static/model" + path = os.path.join(self.temp_dir.name, + "jit_save_data_parallel_with_to_static/model") paddle.jit.save(layer, path) self.verify_inference_correctness(layer, path) @@ -1368,6 +1494,12 @@ class InputSepcLayer(paddle.nn.Layer): class TestInputSpecCompatibility(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def _assert_input_spec_layer_return(self, expect_layer, test_layer): input_x = paddle.uniform([8, 8], dtype='float32') input_y = paddle.uniform([8, 1], dtype='float64') @@ -1380,7 +1512,8 @@ class TestInputSpecCompatibility(unittest.TestCase): def test_jit_save_compatible_input_sepc(self): layer = InputSepcLayer() - save_dir = "jit_save_compatible_input_spec" + save_dir = os.path.join(self.temp_dir.name, + "jit_save_compatible_input_spec") path = save_dir + "/model" paddle.jit.save(layer=layer, path=path) @@ -1414,7 +1547,8 @@ class TestInputSpecCompatibility(unittest.TestCase): def test_jit_save_incompatible_input_sepc(self): layer = InputSepcLayer() - save_dir = "jit_save_compatible_input_spec" + save_dir = os.path.join(self.temp_dir.name, + "jit_save_compatible_input_spec") path = save_dir + "/model" with self.assertRaises(ValueError): diff --git a/python/paddle/fluid/tests/unittests/test_load_op.py b/python/paddle/fluid/tests/unittests/test_load_op.py index 885c26e2be..2e6e1b4bdf 100644 --- a/python/paddle/fluid/tests/unittests/test_load_op.py +++ b/python/paddle/fluid/tests/unittests/test_load_op.py @@ -19,6 +19,8 @@ import numpy as np from op_test import OpTest, randomize_probability import paddle.fluid as fluid import paddle.fluid.layers as layers +import os +import tempfile class TestLoadOp(unittest.TestCase): @@ -26,6 +28,7 @@ class TestLoadOp(unittest.TestCase): """ def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() self.ones = np.ones((4, 4)).astype('float32') main_prog = fluid.Program() start_prog = fluid.Program() @@ -41,14 +44,20 @@ class TestLoadOp(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(start_prog) fluid.io.save_persistables( - exe, dirname="./model", main_program=main_prog) + exe, + dirname=os.path.join(self.temp_dir.name, "./model"), + main_program=main_prog) + + def tearDown(self): + self.temp_dir.cleanup() def test_load(self): main_prog = fluid.Program() start_prog = fluid.Program() with fluid.program_guard(main_prog, start_prog): var = layers.create_tensor(dtype='float32') - layers.load(var, file_path='./model/w') + layers.load( + var, file_path=os.path.join(self.temp_dir.name, './model/w')) exe = fluid.Executor(fluid.CPUPlace()) exe.run(start_prog) diff --git a/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py b/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py index 35ad6fdb30..a5d56fc6c0 100644 --- a/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py +++ b/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py @@ -23,6 +23,7 @@ import paddle import paddle.fluid as fluid from paddle.fluid import core from test_imperative_base import new_program_scope +import tempfile def convolutional_neural_network(img): @@ -59,6 +60,7 @@ def static_train_net(img, label): class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() self.seed = 90 self.epoch_num = 1 self.batch_size = 128 @@ -66,6 +68,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): # enable static mode paddle.enable_static() + def tearDown(self): + self.temp_dir.cleanup() + def train_and_save_model(self, only_params=False): with new_program_scope(): startup_program = fluid.default_startup_program() @@ -121,7 +126,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): self.assertTrue(np.array_equal(value, load_dict[var_name])) def test_load_default(self): - self.save_dirname = "static_mnist.load_state_dict.default" + self.save_dirname = os.path.join(self.temp_dir.name, + "static_mnist.load_state_dict.default") self.model_filename = None self.params_filename = None orig_param_dict = self.train_and_save_model() @@ -133,7 +139,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): self.check_load_state_dict(orig_param_dict, new_load_param_dict) def test_load_with_model_filename(self): - self.save_dirname = "static_mnist.load_state_dict.model_filename" + self.save_dirname = os.path.join( + self.temp_dir.name, "static_mnist.load_state_dict.model_filename") self.model_filename = "static_mnist.model" self.params_filename = None orig_param_dict = self.train_and_save_model() @@ -147,7 +154,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): self.check_load_state_dict(orig_param_dict, new_load_param_dict) def test_load_with_param_filename(self): - self.save_dirname = "static_mnist.load_state_dict.param_filename" + self.save_dirname = os.path.join( + self.temp_dir.name, "static_mnist.load_state_dict.param_filename") self.model_filename = None self.params_filename = "static_mnist.params" orig_param_dict = self.train_and_save_model() @@ -161,7 +169,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): self.check_load_state_dict(orig_param_dict, new_load_param_dict) def test_load_with_model_and_param_filename(self): - self.save_dirname = "static_mnist.load_state_dict.model_and_param_filename" + self.save_dirname = os.path.join( + self.temp_dir.name, + "static_mnist.load_state_dict.model_and_param_filename") self.model_filename = "static_mnist.model" self.params_filename = "static_mnist.params" orig_param_dict = self.train_and_save_model() @@ -179,7 +189,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): self.check_load_state_dict(orig_param_dict, new_load_param_dict) def test_load_state_dict_from_save_params(self): - self.save_dirname = "static_mnist.load_state_dict.save_params" + self.save_dirname = os.path.join( + self.temp_dir.name, "static_mnist.load_state_dict.save_params") self.params_filename = None orig_param_dict = self.train_and_save_model(True) diff --git a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py index 8945d35c13..74c90dde02 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py @@ -19,6 +19,7 @@ import numpy as np import os import sys from io import BytesIO +import tempfile import paddle import paddle.nn as nn @@ -90,7 +91,10 @@ def train(layer, loader, loss_fn, opt): class TestSaveLoadLargeParameters(unittest.TestCase): def setUp(self): - pass + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_large_parameters_paddle_save(self): # enable dygraph mode @@ -100,7 +104,8 @@ class TestSaveLoadLargeParameters(unittest.TestCase): layer = LayerWithLargeParameters() save_dict = layer.state_dict() - path = os.path.join("test_paddle_save_load_large_param_save", + path = os.path.join(self.temp_dir.name, + "test_paddle_save_load_large_param_save", "layer.pdparams") protocol = 4 paddle.save(save_dict, path, protocol=protocol) @@ -111,6 +116,12 @@ class TestSaveLoadLargeParameters(unittest.TestCase): class TestSaveLoadPickle(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def test_pickle_protocol(self): # enable dygraph mode paddle.disable_static() @@ -118,7 +129,8 @@ class TestSaveLoadPickle(unittest.TestCase): layer = LinearNet() save_dict = layer.state_dict() - path = os.path.join("test_paddle_save_load_pickle_protocol", + path = os.path.join(self.temp_dir.name, + "test_paddle_save_load_pickle_protocol", "layer.pdparams") with self.assertRaises(ValueError): @@ -143,6 +155,12 @@ class TestSaveLoadPickle(unittest.TestCase): class TestSaveLoadAny(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def set_zero(self, prog, place, scope=None): if scope is None: scope = fluid.global_scope() @@ -203,7 +221,8 @@ class TestSaveLoadAny(unittest.TestCase): t = np.array(fluid.global_scope().find_var(var.name) .get_tensor()) base_map[var.name] = t - path = os.path.join("test_replace_static_save_load", "model") + path = os.path.join(self.temp_dir.name, + "test_replace_static_save_load", "model") # paddle.save, legacy paddle.fluid.load self.replace_static_save(prog, path) self.set_zero(prog, place) @@ -229,7 +248,9 @@ class TestSaveLoadAny(unittest.TestCase): for var in prog.list_vars(): if var.persistable: tensor = var.get_value(fluid.global_scope()) - paddle.save(tensor, os.path.join(path_vars, var.name)) + paddle.save( + tensor, + os.path.join(self.temp_dir.name, path_vars, var.name)) with self.assertRaises(TypeError): var.get_value('fluid.global_scope()') with self.assertRaises(ValueError): @@ -248,7 +269,8 @@ class TestSaveLoadAny(unittest.TestCase): for var in prog.list_vars(): if var.persistable: tensor = paddle.load( - os.path.join(path_vars, var.name), return_numpy=False) + os.path.join(self.temp_dir.name, path_vars, var.name), + return_numpy=False) var.set_value(tensor) new_t = np.array(fluid.global_scope().find_var(var.name) .get_tensor()) @@ -282,7 +304,8 @@ class TestSaveLoadAny(unittest.TestCase): y.mean().backward() adam.step() state_dict = adam.state_dict() - path = 'paddle_save_load_v2/model.pdparams' + path = os.path.join(self.temp_dir.name, + 'paddle_save_load_v2/model.pdparams') with self.assertRaises(TypeError): paddle.save(state_dict, path, use_binary_format='False') # legacy paddle.save, paddle.load @@ -306,7 +329,8 @@ class TestSaveLoadAny(unittest.TestCase): # enable dygraph mode paddle.disable_static() layer = LinearNet() - path = 'paddle_save_load_v2/var_dygraph' + path = os.path.join(self.temp_dir.name, + 'paddle_save_load_v2/var_dygraph') tensor = layer._linear.weight with self.assertRaises(ValueError): paddle.save(tensor, path, pickle_protocol='3') @@ -348,7 +372,8 @@ class TestSaveLoadAny(unittest.TestCase): break scope = fluid.global_scope() origin_tensor = np.array(tensor) - path = 'test_single_pickle_var_static/var' + path = os.path.join(self.temp_dir.name, + 'test_single_pickle_var_static/var') paddle.save(tensor, path) self.set_zero(prog, place, scope) # static load @@ -370,7 +395,8 @@ class TestSaveLoadAny(unittest.TestCase): def test_dygraph_save_static_load(self): inps = np.random.randn(1, IMAGE_SIZE).astype('float32') - path = 'test_dygraph_save_static_load/dy-static.pdparams' + path = os.path.join(self.temp_dir.name, + 'test_dygraph_save_static_load/dy-static.pdparams') paddle.disable_static() with paddle.utils.unique_name.guard(): layer = LinearNet() @@ -414,10 +440,14 @@ class TestSaveLoadAny(unittest.TestCase): }) obj4 = (np.random.randn(5, 6), (123, )) - path1 = "test_save_load_any_complex_object_dygraph/obj1" - path2 = "test_save_load_any_complex_object_dygraph/obj2" - path3 = "test_save_load_any_complex_object_dygraph/obj3" - path4 = "test_save_load_any_complex_object_dygraph/obj4" + path1 = os.path.join(self.temp_dir.name, + "test_save_load_any_complex_object_dygraph/obj1") + path2 = os.path.join(self.temp_dir.name, + "test_save_load_any_complex_object_dygraph/obj2") + path3 = os.path.join(self.temp_dir.name, + "test_save_load_any_complex_object_dygraph/obj3") + path4 = os.path.join(self.temp_dir.name, + "test_save_load_any_complex_object_dygraph/obj4") paddle.save(obj1, path1) paddle.save(obj2, path2) paddle.save(obj3, path3) @@ -584,10 +614,18 @@ class TestSaveLoadAny(unittest.TestCase): }) obj4 = (np.ndarray([3, 4], dtype="float32"), ) - path1 = "test_save_load_any_complex_object_static/obj1" - path2 = "test_save_load_any_complex_object_static/obj2" - path3 = "test_save_load_any_complex_object_static/obj3" - path4 = "test_save_load_any_complex_object_static/obj4" + path1 = os.path.join( + self.temp_dir.name, + "test_save_load_any_complex_object_static/obj1") + path2 = os.path.join( + self.temp_dir.name, + "test_save_load_any_complex_object_static/obj2") + path3 = os.path.join( + self.temp_dir.name, + "test_save_load_any_complex_object_static/obj3") + path4 = os.path.join( + self.temp_dir.name, + "test_save_load_any_complex_object_static/obj4") paddle.save(obj1, path1) paddle.save(obj2, path2) paddle.save(obj3, path3) @@ -751,7 +789,8 @@ class TestSaveLoadAny(unittest.TestCase): def test_varbase_binary_var(self): paddle.disable_static() varbase = paddle.randn([3, 2], dtype='float32') - path = 'test_paddle_save_load_varbase_binary_var/varbase' + path = os.path.join(self.temp_dir.name, + 'test_paddle_save_load_varbase_binary_var/varbase') paddle.save(varbase, path, use_binary_format=True) load_array = paddle.load(path, return_numpy=True) load_tensor = paddle.load(path, return_numpy=False) @@ -836,6 +875,10 @@ class TestSaveLoad(unittest.TestCase): # config seed paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def build_and_train_model(self): # create network @@ -863,8 +906,10 @@ class TestSaveLoad(unittest.TestCase): layer, opt = self.build_and_train_model() # save - layer_save_path = "test_paddle_save_load.linear.pdparams" - opt_save_path = "test_paddle_save_load.linear.pdopt" + layer_save_path = os.path.join(self.temp_dir.name, + "test_paddle_save_load.linear.pdparams") + opt_save_path = os.path.join(self.temp_dir.name, + "test_paddle_save_load.linear.pdopt") layer_state_dict = layer.state_dict() opt_state_dict = opt.state_dict() @@ -880,7 +925,9 @@ class TestSaveLoad(unittest.TestCase): # test save load in static mode paddle.enable_static() - static_save_path = "static_mode_test/test_paddle_save_load.linear.pdparams" + static_save_path = os.path.join( + self.temp_dir.name, + "static_mode_test/test_paddle_save_load.linear.pdparams") paddle.save(layer_state_dict, static_save_path) load_static_state_dict = paddle.load(static_save_path) self.check_load_state_dict(layer_state_dict, load_static_state_dict) @@ -891,20 +938,28 @@ class TestSaveLoad(unittest.TestCase): # 2. test save path format error with self.assertRaises(ValueError): - paddle.save(layer_state_dict, "test_paddle_save_load.linear.model/") + paddle.save(layer_state_dict, + os.path.join(self.temp_dir.name, + "test_paddle_save_load.linear.model/")) # 3. test load path not exist error with self.assertRaises(ValueError): - paddle.load("test_paddle_save_load.linear.params") + paddle.load( + os.path.join(self.temp_dir.name, + "test_paddle_save_load.linear.params")) # 4. test load old save path error with self.assertRaises(ValueError): - paddle.load("test_paddle_save_load.linear") + paddle.load( + os.path.join(self.temp_dir.name, + "test_paddle_save_load.linear")) class TestSaveLoadProgram(unittest.TestCase): def test_save_load_program(self): paddle.enable_static() + temp_dir = tempfile.TemporaryDirectory() + with new_program_scope(): layer = LinearNet() data = paddle.static.data( @@ -914,8 +969,12 @@ class TestSaveLoadProgram(unittest.TestCase): startup_program = paddle.static.default_startup_program() origin_main = main_program.desc.serialize_to_string() origin_startup = startup_program.desc.serialize_to_string() - path1 = "test_paddle_save_load_program/main_program.pdmodel" - path2 = "test_paddle_save_load_program/startup_program.pdmodel" + path1 = os.path.join( + temp_dir.name, + "test_paddle_save_load_program/main_program.pdmodel") + path2 = os.path.join( + temp_dir.name, + "test_paddle_save_load_program/startup_program.pdmodel") paddle.save(main_program, path1) paddle.save(startup_program, path2) @@ -924,11 +983,13 @@ class TestSaveLoadProgram(unittest.TestCase): load_startup = paddle.load(path2).desc.serialize_to_string() self.assertTrue(origin_main == load_main) self.assertTrue(origin_startup == load_startup) + temp_dir.cleanup() class TestSaveLoadLayer(unittest.TestCase): def test_save_load_layer(self): paddle.disable_static() + temp_dir = tempfile.TemporaryDirectory() inps = paddle.randn([1, IMAGE_SIZE], dtype='float32') layer1 = LinearNet() layer2 = LinearNet() @@ -936,9 +997,11 @@ class TestSaveLoadLayer(unittest.TestCase): layer2.eval() origin_layer = (layer1, layer2) origin = (layer1(inps), layer2(inps)) - path = "test_save_load_layer_/layer.pdmodel" + path = os.path.join(temp_dir.name, + "test_save_load_layer_/layer.pdmodel") with self.assertRaises(ValueError): paddle.save(origin_layer, path) + temp_dir.cleanup() if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py b/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py index 0b9e038f7c..d5efd71a3c 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py @@ -21,6 +21,7 @@ import os import sys import six import platform +import tempfile import paddle import paddle.nn as nn @@ -37,6 +38,10 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): def setUp(self): # enable static graph mode paddle.enable_static() + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def set_zero(self, prog, place, scope=None): if scope is None: @@ -94,7 +99,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t # test for replace_save_vars/io.load_vars - path_vars1 = 'test_replace_save_load_vars_binary1/model' + path_vars1 = os.path.join( + self.temp_dir.name, 'test_replace_save_load_vars_binary1/model') self.replace_save_vars(prog, path_vars1) # set var to zero self.set_zero(prog, place) @@ -111,7 +117,9 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): self.assertTrue(np.array_equal(new_t, base_t)) # test for io.save_vars/replace_load_vars - path_vars2 = 'test_replace_save_load_vars_binary2/model/' + path_vars2 = os.path.join( + self.temp_dir.name, + 'test_replace_save_load_vars_binary2/model/') fluid.io.save_vars( exe, path_vars2, main_program=prog, vars=var_list) self.set_zero(prog, place) @@ -141,7 +149,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): prog = paddle.static.default_main_program() exe.run(fluid.default_startup_program()) - dirname = 'test_save_load_lod_tensor1/tensor_' + dirname = os.path.join(self.temp_dir.name, + 'test_save_load_lod_tensor1/tensor_') for var in prog.list_vars(): if var.persistable and list( var.shape) == [IMAGE_SIZE, OUTPUT_NUM]: @@ -162,12 +171,13 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): self.assertTrue(np.array_equal(origin, to_array)) with self.assertRaises(NotImplementedError): - path = 'test_save_load_error/temp' + path = os.path.join(self.temp_dir.name, 'test_save_load_error/temp') paddle.save({}, path, use_binary_format=True) # On the Windows platform, when parsing a string that can't be parsed as a `Program`, `desc_.ParseFromString` has a timeout risk. if 'Windows' != platform.system(): with self.assertRaises(ValueError): - path = 'test_save_load_error/temp' + path = os.path.join(self.temp_dir.name, + 'test_save_load_error/temp') with open(path, "w") as f: f.write('\0') paddle.load(path) @@ -178,11 +188,17 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): with self.assertRaises(RuntimeError): fluid.core.save_lod_tensor( - temp_lod, 'test_save_load_error_not_exist_file/not_exist_file') + temp_lod, + os.path.join( + self.temp_dir.name, + 'test_save_load_error_not_exist_file/not_exist_file')) with self.assertRaises(RuntimeError): fluid.core.load_lod_tensor( - temp_lod, 'test_save_load_error_not_exist_file/not_exist_file') + temp_lod, + os.path.join( + self.temp_dir.name, + 'test_save_load_error_not_exist_file/not_exist_file')) # save to memory byio = BytesIO() @@ -206,7 +222,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): rows = [0, 4, 7] row_numel = 12 selected_rows = fluid.core.SelectedRows(rows, height) - path = 'test_paddle_save_load_selected_rows/sr.pdsr' + path = os.path.join(self.temp_dir.name, + 'test_paddle_save_load_selected_rows/sr.pdsr') with self.assertRaises(ValueError): paddle.save(selected_rows, path, use_binary_format=True) @@ -227,11 +244,15 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): with self.assertRaises(RuntimeError): fluid.core.save_selected_rows( selected_rows, - 'test_paddle_save_load_selected_rows_not_exist_file/temp') + os.path.join( + self.temp_dir.name, + 'test_paddle_save_load_selected_rows_not_exist_file/temp')) with self.assertRaises(RuntimeError): fluid.core.load_selected_rows( selected_rows, - 'test_paddle_save_load_selected_rows_not_exist_file/temp') + os.path.join( + self.temp_dir.name, + 'test_paddle_save_load_selected_rows_not_exist_file/temp')) # save to memory byio = BytesIO() diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load.py b/python/paddle/fluid/tests/unittests/test_static_save_load.py index cfce0bb7d3..9370af06d5 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load.py @@ -30,6 +30,7 @@ import six import pickle import os import errno +import tempfile paddle.enable_static() @@ -230,6 +231,7 @@ class TestSaveLoadBase(unittest.TestCase): init_scale = 0.1 batch_size = 4 batch_num = 200 + temp_dir = tempfile.TemporaryDirectory() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -297,7 +299,7 @@ class TestSaveLoadBase(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - fluid.save(main_program, "./test_1") + fluid.save(main_program, os.path.join(temp_dir.name, "test_1")) # set var to zero for var in main_program.list_vars(): @@ -310,7 +312,8 @@ class TestSaveLoadBase(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, "./test_1.pdparams", exe) + fluid.load(main_program, + os.path.join(temp_dir.name, "test_1.pdparams"), exe) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -318,6 +321,7 @@ class TestSaveLoadBase(unittest.TestCase): .get_tensor()) base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t)) + temp_dir.cleanup() class TestSaveLoadPartial(unittest.TestCase): @@ -334,6 +338,7 @@ class TestSaveLoadPartial(unittest.TestCase): init_scale = 0.1 batch_size = 4 batch_num = 200 + temp_dir = tempfile.TemporaryDirectory() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -409,7 +414,7 @@ class TestSaveLoadPartial(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - fluid.save(main_program, "./test_1") + fluid.save(main_program, os.path.join(temp_dir.name, "test_1")) # set var to zero for var in main_program.list_vars(): @@ -422,7 +427,8 @@ class TestSaveLoadPartial(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(test_program, "./test_1.pdopt", None) + fluid.load(test_program, + os.path.join(temp_dir.name, "test_1.pdopt"), None) for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -430,7 +436,9 @@ class TestSaveLoadPartial(unittest.TestCase): .get_tensor()) base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t)) - fluid.load(test_program, "./test_1.pdmodel", None) + fluid.load(test_program, + os.path.join(temp_dir.name, "test_1.pdmodel"), None) + temp_dir.cleanup() class TestSaveLoadSetStateDict(unittest.TestCase): @@ -447,6 +455,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): init_scale = 0.1 batch_size = 4 batch_num = 200 + temp_dir = tempfile.TemporaryDirectory() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -514,7 +523,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - fluid.save(main_program, "./test_1") + fluid.save(main_program, os.path.join(temp_dir.name, "test_1")) # set var to zero for var in main_program.list_vars(): @@ -527,7 +536,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, "./test_1", exe) + fluid.load(main_program, os.path.join(temp_dir.name, "test_1"), exe) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -535,6 +544,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): .get_tensor()) base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t)) + temp_dir.cleanup() class TestProgramStatePartial(unittest.TestCase): @@ -551,6 +561,7 @@ class TestProgramStatePartial(unittest.TestCase): init_scale = 0.1 batch_size = 4 batch_num = 200 + temp_dir = tempfile.TemporaryDirectory() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -626,7 +637,7 @@ class TestProgramStatePartial(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - fluid.save(main_program, os.path.join('some_dir', 'test_1')) + fluid.save(main_program, os.path.join(temp_dir.name, 'test_1')) # set var to zero for var in main_program.list_vars(): @@ -641,16 +652,16 @@ class TestProgramStatePartial(unittest.TestCase): #fluid.load(test_program, "./test_1", None ) program_state = fluid.load_program_state( - os.path.join('some_dir', 'test_1')) + os.path.join(temp_dir.name, 'test_1')) program_state_1 = fluid.load_program_state( - os.path.join('some_dir', 'test_1.pdparams')) + os.path.join(temp_dir.name, 'test_1.pdparams')) program_state_2 = fluid.load_program_state( - os.path.join('some_dir', 'test_1.pdopt')) + os.path.join(temp_dir.name, 'test_1.pdopt')) program_state_3 = fluid.load_program_state( - os.path.join('some_dir', 'test_1.pdmodel')) + os.path.join(temp_dir.name, 'test_1.pdmodel')) fluid.set_program_state(test_program, program_state) @@ -720,6 +731,7 @@ class TestProgramStatePartial(unittest.TestCase): .get_tensor()) base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t)) + temp_dir.cleanup() class TestVariableInit(unittest.TestCase): @@ -737,7 +749,9 @@ class TestVariableInit(unittest.TestCase): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - fluid.save(fluid.default_main_program(), "./test_path") + temp_dir = tempfile.TemporaryDirectory() + fluid.save(fluid.default_main_program(), + os.path.join(temp_dir.name, "test_path")) def set_var(var, ndarray): t = var.get_tensor() @@ -763,7 +777,7 @@ class TestVariableInit(unittest.TestCase): fluid.core._create_loaded_parameter(parameter_list, new_scope, exe._default_executor) - parameter_file_name = "./test_path.pdparams" + parameter_file_name = os.path.join(temp_dir.name, "test_path.pdparams") with open(parameter_file_name, 'rb') as f: load_dict = pickle.load(f) @@ -779,7 +793,7 @@ class TestVariableInit(unittest.TestCase): fluid.core._create_loaded_parameter(opt_list, new_scope, exe._default_executor) - opt_file_name = "./test_path.pdopt" + opt_file_name = os.path.join(temp_dir.name, "test_path.pdopt") with open(opt_file_name, 'rb') as f: load_dict = pickle.load(f) @@ -805,6 +819,7 @@ class TestVariableInit(unittest.TestCase): base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t)) + temp_dir.cleanup() class TestLoadFromOldInterface(unittest.TestCase): @@ -815,10 +830,15 @@ class TestLoadFromOldInterface(unittest.TestCase): if os.path.exists("test_static_load_var_list.pdparams"): os.remove("test_static_load_var_list.pdparams") + self.temp_dir = tempfile.TemporaryDirectory() + def set_place(self): return fluid.CPUPlace() if not core.is_compiled_with_cuda( ) else fluid.CUDAPlace(0) + def tearDown(self): + self.temp_dir.cleanup() + def test_load_from_old_interface(self): seed = 90 hidden_size = 10 @@ -898,7 +918,9 @@ class TestLoadFromOldInterface(unittest.TestCase): base_map[var.name] = t #fluid.save(main_program, "./test_1") - fluid.io.save_persistables(exe, "test_path", main_program) + fluid.io.save_persistables( + exe, + os.path.join(self.temp_dir.name, "test_path"), main_program) # set var to zero for var in main_program.list_vars(): @@ -911,7 +933,8 @@ class TestLoadFromOldInterface(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, "test_path", exe) + fluid.load(main_program, + os.path.join(self.temp_dir.name, "test_path"), exe) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -928,11 +951,13 @@ class TestLoadFromOldInterface(unittest.TestCase): var.desc.set_shape(new_shape) with self.assertRaises(RuntimeError): - fluid.load(main_program, "test_path", exe) + fluid.load(main_program, + os.path.join(self.temp_dir.name, "test_path"), exe) # check unused parameter - fluid.load(test_clone_program, "test_path", exe) + fluid.load(test_clone_program, + os.path.join(self.temp_dir.name, "test_path"), exe) def test_load_from_old_interface_var_list(self): seed = 90 @@ -1013,8 +1038,10 @@ class TestLoadFromOldInterface(unittest.TestCase): base_map[var.name] = t #fluid.save(main_program, "./test_1") - fluid.io.save_persistables(exe, "test_static_load_var_list", - main_program) + fluid.io.save_persistables( + exe, + os.path.join(self.temp_dir.name, "test_static_load_var_list"), + main_program) # set var to zero var_list = [] @@ -1030,7 +1057,10 @@ class TestLoadFromOldInterface(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, "test_static_load_var_list", exe, var_list) + fluid.load( + main_program, + os.path.join(self.temp_dir.name, "test_static_load_var_list"), + exe, var_list) var_list_names = [var.name for var in var_list] for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: @@ -1059,6 +1089,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): init_scale = 0.1 batch_size = 4 batch_num = 200 + temp_dir = tempfile.TemporaryDirectory() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -1125,10 +1156,10 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - + save_dir = os.path.join(temp_dir.name, "test_path") #fluid.save(main_program, "./test_1") fluid.io.save_persistables( - exe, "test_path", main_program, filename="model_single") + exe, save_dir, main_program, filename="model_single") # set var to zero for var in main_program.list_vars(): @@ -1141,7 +1172,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - file_model_path = os.path.join("test_path", "model_single") + file_model_path = os.path.join(save_dir, "model_single") fluid.load(main_program, file_model_path, exe, fluid.io.get_program_persistable_vars(main_program)) @@ -1190,11 +1221,16 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): all_var_list = list(main_program.list_vars()) fluid.load(main_program, file_model_path, exe, all_var_list + [temp_var]) + temp_dir.cleanup() class TestProgramStateOldSave(unittest.TestCase): def setUp(self): self.test_dygraph = True + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def set_place(self): return fluid.CPUPlace() if not core.is_compiled_with_cuda( @@ -1283,8 +1319,8 @@ class TestProgramStateOldSave(unittest.TestCase): # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - - fluid.io.save_persistables(exe, "test_program_1", main_program) + save_dir = os.path.join(self.temp_dir.name, "test_program_1") + fluid.io.save_persistables(exe, save_dir, main_program) # set var to zero for var in main_program.list_vars(): @@ -1298,7 +1334,7 @@ class TestProgramStateOldSave(unittest.TestCase): self.assertTrue(np.sum(np.abs(new_t)) == 0) # case 1: load basic - program_state = fluid.load_program_state("test_program_1") + program_state = fluid.load_program_state(save_dir) fluid.set_program_state(main_program, program_state) self.check_in_static(main_program, base_map) @@ -1313,24 +1349,20 @@ class TestProgramStateOldSave(unittest.TestCase): else: raise e - orig_filepath = './test_program_1/fc_0.w_0' - symlink_filepath = './test_program_1/link_fc_0.w_0' - # create a needless link file for coverage - symlink_force(orig_filepath, symlink_filepath) - program_state = fluid.load_program_state("test_program_1") + program_state = fluid.load_program_state(save_dir) fluid.set_program_state(main_program, program_state) self.check_in_static(main_program, base_map) # case 3: load with var_list program_state = fluid.load_program_state( - "test_program_1", main_program.all_parameters()) + save_dir, main_program.all_parameters()) fluid.set_program_state(main_program, program_state) self.check_in_static(main_program, base_map) if self.test_dygraph: # make sure `load_program_state` can be used in dynamic graph mode with fluid.dygraph.guard(place): - load_state = fluid.load_program_state("test_program_1") + load_state = fluid.load_program_state(save_dir) for k, v in load_state.items(): self.assertTrue(np.array_equal(base_map[k], v)) @@ -1365,6 +1397,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): init_scale = 0.1 batch_size = 4 batch_num = 200 + temp_dir = tempfile.TemporaryDirectory() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -1440,8 +1473,9 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t + save_dir = os.path.join(temp_dir.name, "test_program_2") fluid.io.save_persistables( - exe, "test_program_2", main_program, filename="model_1") + exe, save_dir, main_program, filename="model_1") # set var to zero for var in main_program.list_vars(): @@ -1456,7 +1490,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): #fluid.load(test_program, "./test_1", None ) program_state = fluid.load_program_state( - os.path.join("test_program_2", "model_1"), + os.path.join(save_dir, "model_1"), var_list=fluid.io.get_program_persistable_vars(main_program)) fluid.set_program_state(main_program, program_state) @@ -1468,21 +1502,20 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): self.assertTrue(np.array_equal(new_t, base_t)) with self.assertRaises(ValueError): - fluid.load_program_state( - os.path.join("test_program_2", "model_1")) + fluid.load_program_state(os.path.join(save_dir, "model_1")) with self.assertRaises(TypeError): fluid.load_program_state( - os.path.join("test_program_2", "model_1"), - var_list=["str"]) + os.path.join(save_dir, "model_1"), var_list=["str"]) with self.assertRaises(RuntimeError): fluid.load_program_state( - os.path.join("test_program_2", "model_1"), + os.path.join(save_dir, "model_1"), var_list=[ main_program.global_block().create_var( name="fake_var_name", persistable=True) ]) + temp_dir.cleanup() class TestStaticSaveLoadPickle(unittest.TestCase): @@ -1511,7 +1544,8 @@ class TestStaticSaveLoadPickle(unittest.TestCase): self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - path = os.path.join("test_static_save_load_pickle", + temp_dir = tempfile.TemporaryDirectory() + path = os.path.join(temp_dir.name, "test_static_save_load_pickle", "pickle_protocol") with self.assertRaises(ValueError): diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py b/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py index bc8c3cc5b2..670b397bae 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py @@ -23,11 +23,19 @@ from paddle.fluid.optimizer import SGDOptimizer from paddle.fluid.tests.unittests.test_imperative_base import new_program_scope from paddle.fluid.tests.unittests.test_static_save_load import PtbModel import numpy as np +import tempfile +import os @unittest.skipIf(not core.supports_bfloat16(), "place does not support BF16 evaluation") class TestSaveLoadBF16(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def set_place(self): return fluid.CPUPlace() @@ -108,8 +116,8 @@ class TestSaveLoadBF16(unittest.TestCase): # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - - fluid.save(main_program, "./test_1") + save_dir = os.path.join(self.temp_dir.name, "test_1") + fluid.save(main_program, save_dir) # set var to zero for var in main_program.list_vars(): @@ -122,7 +130,8 @@ class TestSaveLoadBF16(unittest.TestCase): # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, "./test_1.pdparams", exe) + fluid.load(main_program, + os.path.join(self.temp_dir.name, "test_1.pdparams"), exe) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load_large.py b/python/paddle/fluid/tests/unittests/test_static_save_load_large.py index 389fc259b5..84877865f4 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load_large.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load_large.py @@ -23,6 +23,7 @@ from test_imperative_base import new_program_scope import numpy as np import pickle import os +import tempfile LARGE_PARAM = 2**26 @@ -51,9 +52,10 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - - path = os.path.join("test_static_save_load_large_param", - "static_save") + temp_dir = tempfile.TemporaryDirectory() + path = os.path.join(temp_dir.name, + "test_static_save_load_large_param") + path = os.path.join(path, "static_save") protocol = 4 paddle.fluid.save(prog, path, pickle_protocol=protocol) # set var to zero @@ -93,6 +95,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): .get_tensor()) base_t = base_map[var.name] self.assertTrue(np.array_equal(new_t, base_t)) + temp_dir.cleanup() if __name__ == '__main__': -- GitLab