未验证 提交 2eb60ddb 编写于 作者: Y YuanRisheng 提交者: GitHub

cherry pick 43581 (#43596)

上级 3689a126
...@@ -27,6 +27,8 @@ from paddle.fluid.layer_helper import LayerHelper ...@@ -27,6 +27,8 @@ from paddle.fluid.layer_helper import LayerHelper
from paddle import _C_ops from paddle import _C_ops
import sys import sys
import tempfile
sys.path.append("./tokenizer") sys.path.append("./tokenizer")
from tokenizer.bert_tokenizer import BertTokenizer from tokenizer.bert_tokenizer import BertTokenizer
...@@ -149,13 +151,17 @@ class Predictor(object): ...@@ -149,13 +151,17 @@ class Predictor(object):
class TestBertTokenizerOp(unittest.TestCase): class TestBertTokenizerOp(unittest.TestCase):
def setUp(self): def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.bert_tokenizer = BertTokenizer.from_pretrained("bert-base-chinese") self.bert_tokenizer = BertTokenizer.from_pretrained("bert-base-chinese")
self.faster_tokenizer = FasterTokenizer(self.bert_tokenizer.vocab) self.faster_tokenizer = FasterTokenizer(self.bert_tokenizer.vocab)
self.init_data() self.init_data()
self.save_path = os.path.join(DATA_HOME, "fast_tokenizer") self.save_path = os.path.join(self.temp_dir.name, "fast_tokenizer")
self.param_path = os.path.join(self.save_path, "model.pdparams") self.param_path = os.path.join(self.save_path, "model.pdparams")
self.inference_path = os.path.join(self.save_path, "inference") self.inference_path = os.path.join(self.save_path, "inference")
def tearDown(self):
self.temp_dir.cleanup()
def init_data(self): def init_data(self):
self.text = [ self.text = [
'选择珠江花园的原因就是方便,有电动扶梯直接到达海边,周围餐馆、食廊、商场、超市、摊位一应俱全。' '选择珠江花园的原因就是方便,有电动扶梯直接到达海边,周围餐馆、食廊、商场、超市、摊位一应俱全。'
......
...@@ -17,11 +17,14 @@ import paddle.fluid as fluid ...@@ -17,11 +17,14 @@ import paddle.fluid as fluid
import paddle.fluid.framework as framework import paddle.fluid.framework as framework
from paddle.fluid.dygraph.nn import * from paddle.fluid.dygraph.nn import *
import numpy as np import numpy as np
import os
import tempfile
class TestDygraphLoadStatic(unittest.TestCase): class TestDygraphLoadStatic(unittest.TestCase):
def testLoadStaticModel(self): def testLoadStaticModel(self):
# static mode # static mode
temp_dir = tempfile.TemporaryDirectory()
a = fluid.data(name="a", shape=[10, 10]) a = fluid.data(name="a", shape=[10, 10])
conv_in = fluid.data(name="conv_in", shape=[None, 10, 10, 10]) conv_in = fluid.data(name="conv_in", shape=[None, 10, 10, 10])
...@@ -121,9 +124,11 @@ class TestDygraphLoadStatic(unittest.TestCase): ...@@ -121,9 +124,11 @@ class TestDygraphLoadStatic(unittest.TestCase):
) if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0)) ) if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0))
out = exe.run(framework.default_startup_program()) out = exe.run(framework.default_startup_program())
fluid.save(framework.default_main_program(), "./test_1") fluid.save(framework.default_main_program(),
os.path.join(temp_dir.name, "test_1"))
para_dict = fluid.load_program_state("./test_1") para_dict = fluid.load_program_state(
os.path.join(temp_dir.name, "test_1"))
new_dict = {} new_dict = {}
for k, v in para_dict.items(): for k, v in para_dict.items():
...@@ -192,6 +197,7 @@ class TestDygraphLoadStatic(unittest.TestCase): ...@@ -192,6 +197,7 @@ class TestDygraphLoadStatic(unittest.TestCase):
my_test.set_dict(new_dict, use_structured_name=False) my_test.set_dict(new_dict, use_structured_name=False)
for k, v in my_test.state_dict().items(): for k, v in my_test.state_dict().items():
self.assertTrue(np.array_equal(v.numpy(), new_dict[v.name])) self.assertTrue(np.array_equal(v.numpy(), new_dict[v.name]))
temp_dir.cleanup()
if __name__ == '__main__': if __name__ == '__main__':
......
...@@ -29,6 +29,8 @@ import six ...@@ -29,6 +29,8 @@ import six
import paddle import paddle
from paddle.fluid.framework import _test_eager_guard from paddle.fluid.framework import _test_eager_guard
import tempfile
class SimpleLSTMRNN(fluid.Layer): class SimpleLSTMRNN(fluid.Layer):
def __init__(self, def __init__(self,
...@@ -209,6 +211,12 @@ class PtbModel(fluid.Layer): ...@@ -209,6 +211,12 @@ class PtbModel(fluid.Layer):
class TestDygraphPtbRnn(unittest.TestCase): class TestDygraphPtbRnn(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def func_setUp(self): def func_setUp(self):
seed = 90 seed = 90
hidden_size = 10 hidden_size = 10
...@@ -286,7 +294,8 @@ class TestDygraphPtbRnn(unittest.TestCase): ...@@ -286,7 +294,8 @@ class TestDygraphPtbRnn(unittest.TestCase):
else: else:
self.base_opti[k] = v self.base_opti[k] = v
paddle.save(self.opti_dict, "./test_dy_v2.pdopt") paddle.save(self.opti_dict,
os.path.join(self.temp_dir.name, "test_dy_v2.pdopt"))
self.state_dict = ptb_model.state_dict() self.state_dict = ptb_model.state_dict()
...@@ -295,10 +304,10 @@ class TestDygraphPtbRnn(unittest.TestCase): ...@@ -295,10 +304,10 @@ class TestDygraphPtbRnn(unittest.TestCase):
np_t = v.numpy() np_t = v.numpy()
self.model_base[k] = np_t self.model_base[k] = np_t
paddle.save(self.state_dict, "./test_dy_v2.pdparams") paddle.save(self.state_dict,
os.path.join(self.temp_dir.name, "test_dy_v2.pdparams"))
def func_testLoadAndSetVarBase(self): def func_testLoadAndSetVarBase(self):
self.setUp()
seed = 90 seed = 90
hidden_size = 10 hidden_size = 10
vocab_size = 1000 vocab_size = 1000
...@@ -375,8 +384,10 @@ class TestDygraphPtbRnn(unittest.TestCase): ...@@ -375,8 +384,10 @@ class TestDygraphPtbRnn(unittest.TestCase):
self.assertTrue(np.sum(np.abs(v.numpy())) == 0) self.assertTrue(np.sum(np.abs(v.numpy())) == 0)
para_state_dict = paddle.load("./test_dy_v2.pdparams") para_state_dict = paddle.load(
opti_state_dict = paddle.load("./test_dy_v2.pdopt") os.path.join(self.temp_dir.name, "test_dy_v2.pdparams"))
opti_state_dict = paddle.load(
os.path.join(self.temp_dir.name, "test_dy_v2.pdopt"))
adam.set_state_dict(opti_state_dict) adam.set_state_dict(opti_state_dict)
opti_dict = adam.state_dict() opti_dict = adam.state_dict()
...@@ -750,7 +761,8 @@ class TestDygraphPtbRnn(unittest.TestCase): ...@@ -750,7 +761,8 @@ class TestDygraphPtbRnn(unittest.TestCase):
last_hidden = None last_hidden = None
last_cell = None last_cell = None
state_dict, opti_dict = fluid.load_dygraph("./test_dy_v2") state_dict, opti_dict = fluid.load_dygraph(
os.path.join(self.temp_dir.name, "test_dy_v2"))
adam.set_state_dict(opti_dict) adam.set_state_dict(opti_dict)
ptb_model.set_dict(state_dict) ptb_model.set_dict(state_dict)
...@@ -907,19 +919,23 @@ class TestDygraphPtbRnn(unittest.TestCase): ...@@ -907,19 +919,23 @@ class TestDygraphPtbRnn(unittest.TestCase):
with fluid.dygraph.guard(): with fluid.dygraph.guard():
emb = fluid.dygraph.Embedding([10, 10]) emb = fluid.dygraph.Embedding([10, 10])
state_dict = emb.state_dict() state_dict = emb.state_dict()
paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams')) paddle.save(
state_dict,
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict = paddle.load( para_state_dict = paddle.load(
os.path.join('saved_dy', 'emb_dy.pdparams')) os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
def func_test_no_state_in_input_dict(self): def func_test_no_state_in_input_dict(self):
with fluid.dygraph.guard(): with fluid.dygraph.guard():
emb = fluid.dygraph.Embedding([10, 10]) emb = fluid.dygraph.Embedding([10, 10])
state_dict = emb.state_dict() state_dict = emb.state_dict()
paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams')) paddle.save(
state_dict,
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict = paddle.load( para_state_dict = paddle.load(
os.path.join('saved_dy', 'emb_dy.pdparams')) os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict.pop('weight') para_state_dict.pop('weight')
emb.set_state_dict(para_state_dict) emb.set_state_dict(para_state_dict)
...@@ -928,10 +944,13 @@ class TestDygraphPtbRnn(unittest.TestCase): ...@@ -928,10 +944,13 @@ class TestDygraphPtbRnn(unittest.TestCase):
with fluid.dygraph.guard(): with fluid.dygraph.guard():
emb = fluid.dygraph.Embedding([10, 10]) emb = fluid.dygraph.Embedding([10, 10])
state_dict = emb.state_dict() state_dict = emb.state_dict()
paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams')) paddle.save(
state_dict,
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict = paddle.load( para_state_dict = paddle.load(
os.path.join('saved_dy', 'emb_dy.pdparams'), return_numpy=True) os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'),
return_numpy=True)
para_state_dict['weight'] = np.expand_dims( para_state_dict['weight'] = np.expand_dims(
para_state_dict['weight'], axis=-1) para_state_dict['weight'], axis=-1)
......
...@@ -19,9 +19,18 @@ import paddle ...@@ -19,9 +19,18 @@ import paddle
import paddle.fluid as fluid import paddle.fluid as fluid
from paddle.fluid import core from paddle.fluid import core
from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph
import tempfile
import os
class TestSaveLoadAPIError(unittest.TestCase): class TestSaveLoadAPIError(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.save_dir = os.path.join(self.temp_dir.name, "fake_dir")
def tearDown(self):
self.temp_dir.cleanup()
def func_test_get_valid_program_error(self): def func_test_get_valid_program_error(self):
# case 1: CompiledProgram no program # case 1: CompiledProgram no program
graph = core.Graph(core.ProgramDesc()) graph = core.Graph(core.ProgramDesc())
...@@ -44,13 +53,13 @@ class TestSaveLoadAPIError(unittest.TestCase): ...@@ -44,13 +53,13 @@ class TestSaveLoadAPIError(unittest.TestCase):
# case 1: main_program type error when vars None # case 1: main_program type error when vars None
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
fluid.io.load_vars( fluid.io.load_vars(
executor=exe, dirname="./fake_dir", main_program="program") executor=exe, dirname=self.save_dir, main_program="program")
# case 2: main_program type error when vars not None # case 2: main_program type error when vars not None
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
fluid.io.load_vars( fluid.io.load_vars(
executor=exe, executor=exe,
dirname="./fake_dir", dirname=self.save_dir,
main_program="program", main_program="program",
vars="vars") vars="vars")
...@@ -61,6 +70,12 @@ class TestSaveLoadAPIError(unittest.TestCase): ...@@ -61,6 +70,12 @@ class TestSaveLoadAPIError(unittest.TestCase):
class TestSaveInferenceModelAPIError(unittest.TestCase): class TestSaveInferenceModelAPIError(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def func_test_useless_feeded_var_names(self): def func_test_useless_feeded_var_names(self):
start_prog = fluid.Program() start_prog = fluid.Program()
main_prog = fluid.Program() main_prog = fluid.Program()
...@@ -74,7 +89,7 @@ class TestSaveInferenceModelAPIError(unittest.TestCase): ...@@ -74,7 +89,7 @@ class TestSaveInferenceModelAPIError(unittest.TestCase):
with self.assertRaisesRegexp( with self.assertRaisesRegexp(
ValueError, "not involved in the target_vars calculation"): ValueError, "not involved in the target_vars calculation"):
fluid.io.save_inference_model( fluid.io.save_inference_model(
dirname='./model', dirname=os.path.join(self.temp_dir.name, 'model'),
feeded_var_names=['x', 'y'], feeded_var_names=['x', 'y'],
target_vars=[z], target_vars=[z],
executor=exe, executor=exe,
...@@ -87,13 +102,20 @@ class TestSaveInferenceModelAPIError(unittest.TestCase): ...@@ -87,13 +102,20 @@ class TestSaveInferenceModelAPIError(unittest.TestCase):
class TestWhenTrainWithNoGrad(unittest.TestCase): class TestWhenTrainWithNoGrad(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def func_test_when_train_with_no_grad(self): def func_test_when_train_with_no_grad(self):
paddle.disable_static() paddle.disable_static()
net = paddle.nn.Linear(1024, 1) net = paddle.nn.Linear(1024, 1)
net = paddle.jit.to_static(net) net = paddle.jit.to_static(net)
x = paddle.rand([1024], 'float32') x = paddle.rand([1024], 'float32')
net(x) net(x)
save_path = './train_with_no_grad' save_path = os.path.join(self.temp_dir.name, 'train_with_no_grad')
paddle.jit.save(net, save_path) paddle.jit.save(net, save_path)
net = paddle.jit.load(save_path) net = paddle.jit.load(save_path)
net.train() net.train()
......
...@@ -19,6 +19,8 @@ import numpy as np ...@@ -19,6 +19,8 @@ import numpy as np
from op_test import OpTest, randomize_probability from op_test import OpTest, randomize_probability
import paddle.fluid as fluid import paddle.fluid as fluid
import paddle.fluid.layers as layers import paddle.fluid.layers as layers
import os
import tempfile
class TestLoadOp(unittest.TestCase): class TestLoadOp(unittest.TestCase):
...@@ -26,6 +28,7 @@ class TestLoadOp(unittest.TestCase): ...@@ -26,6 +28,7 @@ class TestLoadOp(unittest.TestCase):
""" """
def setUp(self): def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.ones = np.ones((4, 4)).astype('float32') self.ones = np.ones((4, 4)).astype('float32')
main_prog = fluid.Program() main_prog = fluid.Program()
start_prog = fluid.Program() start_prog = fluid.Program()
...@@ -41,14 +44,20 @@ class TestLoadOp(unittest.TestCase): ...@@ -41,14 +44,20 @@ class TestLoadOp(unittest.TestCase):
exe = fluid.Executor(fluid.CPUPlace()) exe = fluid.Executor(fluid.CPUPlace())
exe.run(start_prog) exe.run(start_prog)
fluid.io.save_persistables( fluid.io.save_persistables(
exe, dirname="./model", main_program=main_prog) exe,
dirname=os.path.join(self.temp_dir.name, "./model"),
main_program=main_prog)
def tearDown(self):
self.temp_dir.cleanup()
def test_load(self): def test_load(self):
main_prog = fluid.Program() main_prog = fluid.Program()
start_prog = fluid.Program() start_prog = fluid.Program()
with fluid.program_guard(main_prog, start_prog): with fluid.program_guard(main_prog, start_prog):
var = layers.create_tensor(dtype='float32') var = layers.create_tensor(dtype='float32')
layers.load(var, file_path='./model/w') layers.load(
var, file_path=os.path.join(self.temp_dir.name, './model/w'))
exe = fluid.Executor(fluid.CPUPlace()) exe = fluid.Executor(fluid.CPUPlace())
exe.run(start_prog) exe.run(start_prog)
......
...@@ -23,6 +23,7 @@ import paddle ...@@ -23,6 +23,7 @@ import paddle
import paddle.fluid as fluid import paddle.fluid as fluid
from paddle.fluid import core from paddle.fluid import core
from test_imperative_base import new_program_scope from test_imperative_base import new_program_scope
import tempfile
def convolutional_neural_network(img): def convolutional_neural_network(img):
...@@ -59,6 +60,7 @@ def static_train_net(img, label): ...@@ -59,6 +60,7 @@ def static_train_net(img, label):
class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
def setUp(self): def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.seed = 90 self.seed = 90
self.epoch_num = 1 self.epoch_num = 1
self.batch_size = 128 self.batch_size = 128
...@@ -66,6 +68,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): ...@@ -66,6 +68,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
# enable static mode # enable static mode
paddle.enable_static() paddle.enable_static()
def tearDown(self):
self.temp_dir.cleanup()
def train_and_save_model(self, only_params=False): def train_and_save_model(self, only_params=False):
with new_program_scope(): with new_program_scope():
startup_program = fluid.default_startup_program() startup_program = fluid.default_startup_program()
...@@ -121,7 +126,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): ...@@ -121,7 +126,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.assertTrue(np.array_equal(value, load_dict[var_name])) self.assertTrue(np.array_equal(value, load_dict[var_name]))
def test_load_default(self): def test_load_default(self):
self.save_dirname = "static_mnist.load_state_dict.default" self.save_dirname = os.path.join(self.temp_dir.name,
"static_mnist.load_state_dict.default")
self.model_filename = None self.model_filename = None
self.params_filename = None self.params_filename = None
orig_param_dict = self.train_and_save_model() orig_param_dict = self.train_and_save_model()
...@@ -133,7 +139,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): ...@@ -133,7 +139,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict) self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_with_model_filename(self): def test_load_with_model_filename(self):
self.save_dirname = "static_mnist.load_state_dict.model_filename" self.save_dirname = os.path.join(
self.temp_dir.name, "static_mnist.load_state_dict.model_filename")
self.model_filename = "static_mnist.model" self.model_filename = "static_mnist.model"
self.params_filename = None self.params_filename = None
orig_param_dict = self.train_and_save_model() orig_param_dict = self.train_and_save_model()
...@@ -147,7 +154,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): ...@@ -147,7 +154,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict) self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_with_param_filename(self): def test_load_with_param_filename(self):
self.save_dirname = "static_mnist.load_state_dict.param_filename" self.save_dirname = os.path.join(
self.temp_dir.name, "static_mnist.load_state_dict.param_filename")
self.model_filename = None self.model_filename = None
self.params_filename = "static_mnist.params" self.params_filename = "static_mnist.params"
orig_param_dict = self.train_and_save_model() orig_param_dict = self.train_and_save_model()
...@@ -161,7 +169,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): ...@@ -161,7 +169,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict) self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_with_model_and_param_filename(self): def test_load_with_model_and_param_filename(self):
self.save_dirname = "static_mnist.load_state_dict.model_and_param_filename" self.save_dirname = os.path.join(
self.temp_dir.name,
"static_mnist.load_state_dict.model_and_param_filename")
self.model_filename = "static_mnist.model" self.model_filename = "static_mnist.model"
self.params_filename = "static_mnist.params" self.params_filename = "static_mnist.params"
orig_param_dict = self.train_and_save_model() orig_param_dict = self.train_and_save_model()
...@@ -179,7 +189,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): ...@@ -179,7 +189,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict) self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_state_dict_from_save_params(self): def test_load_state_dict_from_save_params(self):
self.save_dirname = "static_mnist.load_state_dict.save_params" self.save_dirname = os.path.join(
self.temp_dir.name, "static_mnist.load_state_dict.save_params")
self.params_filename = None self.params_filename = None
orig_param_dict = self.train_and_save_model(True) orig_param_dict = self.train_and_save_model(True)
......
...@@ -19,6 +19,7 @@ import numpy as np ...@@ -19,6 +19,7 @@ import numpy as np
import os import os
import sys import sys
from io import BytesIO from io import BytesIO
import tempfile
import paddle import paddle
import paddle.nn as nn import paddle.nn as nn
...@@ -90,7 +91,10 @@ def train(layer, loader, loss_fn, opt): ...@@ -90,7 +91,10 @@ def train(layer, loader, loss_fn, opt):
class TestSaveLoadLargeParameters(unittest.TestCase): class TestSaveLoadLargeParameters(unittest.TestCase):
def setUp(self): def setUp(self):
pass self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_large_parameters_paddle_save(self): def test_large_parameters_paddle_save(self):
# enable dygraph mode # enable dygraph mode
...@@ -100,7 +104,8 @@ class TestSaveLoadLargeParameters(unittest.TestCase): ...@@ -100,7 +104,8 @@ class TestSaveLoadLargeParameters(unittest.TestCase):
layer = LayerWithLargeParameters() layer = LayerWithLargeParameters()
save_dict = layer.state_dict() save_dict = layer.state_dict()
path = os.path.join("test_paddle_save_load_large_param_save", path = os.path.join(self.temp_dir.name,
"test_paddle_save_load_large_param_save",
"layer.pdparams") "layer.pdparams")
protocol = 4 protocol = 4
paddle.save(save_dict, path, protocol=protocol) paddle.save(save_dict, path, protocol=protocol)
...@@ -111,6 +116,12 @@ class TestSaveLoadLargeParameters(unittest.TestCase): ...@@ -111,6 +116,12 @@ class TestSaveLoadLargeParameters(unittest.TestCase):
class TestSaveLoadPickle(unittest.TestCase): class TestSaveLoadPickle(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_pickle_protocol(self): def test_pickle_protocol(self):
# enable dygraph mode # enable dygraph mode
paddle.disable_static() paddle.disable_static()
...@@ -118,7 +129,8 @@ class TestSaveLoadPickle(unittest.TestCase): ...@@ -118,7 +129,8 @@ class TestSaveLoadPickle(unittest.TestCase):
layer = LinearNet() layer = LinearNet()
save_dict = layer.state_dict() save_dict = layer.state_dict()
path = os.path.join("test_paddle_save_load_pickle_protocol", path = os.path.join(self.temp_dir.name,
"test_paddle_save_load_pickle_protocol",
"layer.pdparams") "layer.pdparams")
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
...@@ -143,6 +155,12 @@ class TestSaveLoadPickle(unittest.TestCase): ...@@ -143,6 +155,12 @@ class TestSaveLoadPickle(unittest.TestCase):
class TestSaveLoadAny(unittest.TestCase): class TestSaveLoadAny(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_zero(self, prog, place, scope=None): def set_zero(self, prog, place, scope=None):
if scope is None: if scope is None:
scope = fluid.global_scope() scope = fluid.global_scope()
...@@ -203,7 +221,8 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -203,7 +221,8 @@ class TestSaveLoadAny(unittest.TestCase):
t = np.array(fluid.global_scope().find_var(var.name) t = np.array(fluid.global_scope().find_var(var.name)
.get_tensor()) .get_tensor())
base_map[var.name] = t base_map[var.name] = t
path = os.path.join("test_replace_static_save_load", "model") path = os.path.join(self.temp_dir.name,
"test_replace_static_save_load", "model")
# paddle.save, legacy paddle.fluid.load # paddle.save, legacy paddle.fluid.load
self.replace_static_save(prog, path) self.replace_static_save(prog, path)
self.set_zero(prog, place) self.set_zero(prog, place)
...@@ -229,7 +248,9 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -229,7 +248,9 @@ class TestSaveLoadAny(unittest.TestCase):
for var in prog.list_vars(): for var in prog.list_vars():
if var.persistable: if var.persistable:
tensor = var.get_value(fluid.global_scope()) tensor = var.get_value(fluid.global_scope())
paddle.save(tensor, os.path.join(path_vars, var.name)) paddle.save(
tensor,
os.path.join(self.temp_dir.name, path_vars, var.name))
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
var.get_value('fluid.global_scope()') var.get_value('fluid.global_scope()')
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
...@@ -248,7 +269,8 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -248,7 +269,8 @@ class TestSaveLoadAny(unittest.TestCase):
for var in prog.list_vars(): for var in prog.list_vars():
if var.persistable: if var.persistable:
tensor = paddle.load( tensor = paddle.load(
os.path.join(path_vars, var.name), return_numpy=False) os.path.join(self.temp_dir.name, path_vars, var.name),
return_numpy=False)
var.set_value(tensor) var.set_value(tensor)
new_t = np.array(fluid.global_scope().find_var(var.name) new_t = np.array(fluid.global_scope().find_var(var.name)
.get_tensor()) .get_tensor())
...@@ -282,7 +304,8 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -282,7 +304,8 @@ class TestSaveLoadAny(unittest.TestCase):
y.mean().backward() y.mean().backward()
adam.step() adam.step()
state_dict = adam.state_dict() state_dict = adam.state_dict()
path = 'paddle_save_load_v2/model.pdparams' path = os.path.join(self.temp_dir.name,
'paddle_save_load_v2/model.pdparams')
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
paddle.save(state_dict, path, use_binary_format='False') paddle.save(state_dict, path, use_binary_format='False')
# legacy paddle.save, paddle.load # legacy paddle.save, paddle.load
...@@ -306,7 +329,8 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -306,7 +329,8 @@ class TestSaveLoadAny(unittest.TestCase):
# enable dygraph mode # enable dygraph mode
paddle.disable_static() paddle.disable_static()
layer = LinearNet() layer = LinearNet()
path = 'paddle_save_load_v2/var_dygraph' path = os.path.join(self.temp_dir.name,
'paddle_save_load_v2/var_dygraph')
tensor = layer._linear.weight tensor = layer._linear.weight
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.save(tensor, path, pickle_protocol='3') paddle.save(tensor, path, pickle_protocol='3')
...@@ -348,7 +372,8 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -348,7 +372,8 @@ class TestSaveLoadAny(unittest.TestCase):
break break
scope = fluid.global_scope() scope = fluid.global_scope()
origin_tensor = np.array(tensor) origin_tensor = np.array(tensor)
path = 'test_single_pickle_var_static/var' path = os.path.join(self.temp_dir.name,
'test_single_pickle_var_static/var')
paddle.save(tensor, path) paddle.save(tensor, path)
self.set_zero(prog, place, scope) self.set_zero(prog, place, scope)
# static load # static load
...@@ -370,7 +395,8 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -370,7 +395,8 @@ class TestSaveLoadAny(unittest.TestCase):
def test_dygraph_save_static_load(self): def test_dygraph_save_static_load(self):
inps = np.random.randn(1, IMAGE_SIZE).astype('float32') inps = np.random.randn(1, IMAGE_SIZE).astype('float32')
path = 'test_dygraph_save_static_load/dy-static.pdparams' path = os.path.join(self.temp_dir.name,
'test_dygraph_save_static_load/dy-static.pdparams')
paddle.disable_static() paddle.disable_static()
with paddle.utils.unique_name.guard(): with paddle.utils.unique_name.guard():
layer = LinearNet() layer = LinearNet()
...@@ -414,10 +440,14 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -414,10 +440,14 @@ class TestSaveLoadAny(unittest.TestCase):
}) })
obj4 = (np.random.randn(5, 6), (123, )) obj4 = (np.random.randn(5, 6), (123, ))
path1 = "test_save_load_any_complex_object_dygraph/obj1" path1 = os.path.join(self.temp_dir.name,
path2 = "test_save_load_any_complex_object_dygraph/obj2" "test_save_load_any_complex_object_dygraph/obj1")
path3 = "test_save_load_any_complex_object_dygraph/obj3" path2 = os.path.join(self.temp_dir.name,
path4 = "test_save_load_any_complex_object_dygraph/obj4" "test_save_load_any_complex_object_dygraph/obj2")
path3 = os.path.join(self.temp_dir.name,
"test_save_load_any_complex_object_dygraph/obj3")
path4 = os.path.join(self.temp_dir.name,
"test_save_load_any_complex_object_dygraph/obj4")
paddle.save(obj1, path1) paddle.save(obj1, path1)
paddle.save(obj2, path2) paddle.save(obj2, path2)
paddle.save(obj3, path3) paddle.save(obj3, path3)
...@@ -584,10 +614,18 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -584,10 +614,18 @@ class TestSaveLoadAny(unittest.TestCase):
}) })
obj4 = (np.ndarray([3, 4], dtype="float32"), ) obj4 = (np.ndarray([3, 4], dtype="float32"), )
path1 = "test_save_load_any_complex_object_static/obj1" path1 = os.path.join(
path2 = "test_save_load_any_complex_object_static/obj2" self.temp_dir.name,
path3 = "test_save_load_any_complex_object_static/obj3" "test_save_load_any_complex_object_static/obj1")
path4 = "test_save_load_any_complex_object_static/obj4" path2 = os.path.join(
self.temp_dir.name,
"test_save_load_any_complex_object_static/obj2")
path3 = os.path.join(
self.temp_dir.name,
"test_save_load_any_complex_object_static/obj3")
path4 = os.path.join(
self.temp_dir.name,
"test_save_load_any_complex_object_static/obj4")
paddle.save(obj1, path1) paddle.save(obj1, path1)
paddle.save(obj2, path2) paddle.save(obj2, path2)
paddle.save(obj3, path3) paddle.save(obj3, path3)
...@@ -751,7 +789,8 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -751,7 +789,8 @@ class TestSaveLoadAny(unittest.TestCase):
def test_varbase_binary_var(self): def test_varbase_binary_var(self):
paddle.disable_static() paddle.disable_static()
varbase = paddle.randn([3, 2], dtype='float32') varbase = paddle.randn([3, 2], dtype='float32')
path = 'test_paddle_save_load_varbase_binary_var/varbase' path = os.path.join(self.temp_dir.name,
'test_paddle_save_load_varbase_binary_var/varbase')
paddle.save(varbase, path, use_binary_format=True) paddle.save(varbase, path, use_binary_format=True)
load_array = paddle.load(path, return_numpy=True) load_array = paddle.load(path, return_numpy=True)
load_tensor = paddle.load(path, return_numpy=False) load_tensor = paddle.load(path, return_numpy=False)
...@@ -836,6 +875,10 @@ class TestSaveLoad(unittest.TestCase): ...@@ -836,6 +875,10 @@ class TestSaveLoad(unittest.TestCase):
# config seed # config seed
paddle.seed(SEED) paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED) paddle.framework.random._manual_program_seed(SEED)
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def build_and_train_model(self): def build_and_train_model(self):
# create network # create network
...@@ -863,8 +906,10 @@ class TestSaveLoad(unittest.TestCase): ...@@ -863,8 +906,10 @@ class TestSaveLoad(unittest.TestCase):
layer, opt = self.build_and_train_model() layer, opt = self.build_and_train_model()
# save # save
layer_save_path = "test_paddle_save_load.linear.pdparams" layer_save_path = os.path.join(self.temp_dir.name,
opt_save_path = "test_paddle_save_load.linear.pdopt" "test_paddle_save_load.linear.pdparams")
opt_save_path = os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear.pdopt")
layer_state_dict = layer.state_dict() layer_state_dict = layer.state_dict()
opt_state_dict = opt.state_dict() opt_state_dict = opt.state_dict()
...@@ -880,7 +925,9 @@ class TestSaveLoad(unittest.TestCase): ...@@ -880,7 +925,9 @@ class TestSaveLoad(unittest.TestCase):
# test save load in static mode # test save load in static mode
paddle.enable_static() paddle.enable_static()
static_save_path = "static_mode_test/test_paddle_save_load.linear.pdparams" static_save_path = os.path.join(
self.temp_dir.name,
"static_mode_test/test_paddle_save_load.linear.pdparams")
paddle.save(layer_state_dict, static_save_path) paddle.save(layer_state_dict, static_save_path)
load_static_state_dict = paddle.load(static_save_path) load_static_state_dict = paddle.load(static_save_path)
self.check_load_state_dict(layer_state_dict, load_static_state_dict) self.check_load_state_dict(layer_state_dict, load_static_state_dict)
...@@ -891,20 +938,28 @@ class TestSaveLoad(unittest.TestCase): ...@@ -891,20 +938,28 @@ class TestSaveLoad(unittest.TestCase):
# 2. test save path format error # 2. test save path format error
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.save(layer_state_dict, "test_paddle_save_load.linear.model/") paddle.save(layer_state_dict,
os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear.model/"))
# 3. test load path not exist error # 3. test load path not exist error
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.load("test_paddle_save_load.linear.params") paddle.load(
os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear.params"))
# 4. test load old save path error # 4. test load old save path error
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.load("test_paddle_save_load.linear") paddle.load(
os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear"))
class TestSaveLoadProgram(unittest.TestCase): class TestSaveLoadProgram(unittest.TestCase):
def test_save_load_program(self): def test_save_load_program(self):
paddle.enable_static() paddle.enable_static()
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope(): with new_program_scope():
layer = LinearNet() layer = LinearNet()
data = paddle.static.data( data = paddle.static.data(
...@@ -914,8 +969,12 @@ class TestSaveLoadProgram(unittest.TestCase): ...@@ -914,8 +969,12 @@ class TestSaveLoadProgram(unittest.TestCase):
startup_program = paddle.static.default_startup_program() startup_program = paddle.static.default_startup_program()
origin_main = main_program.desc.serialize_to_string() origin_main = main_program.desc.serialize_to_string()
origin_startup = startup_program.desc.serialize_to_string() origin_startup = startup_program.desc.serialize_to_string()
path1 = "test_paddle_save_load_program/main_program.pdmodel" path1 = os.path.join(
path2 = "test_paddle_save_load_program/startup_program.pdmodel" temp_dir.name,
"test_paddle_save_load_program/main_program.pdmodel")
path2 = os.path.join(
temp_dir.name,
"test_paddle_save_load_program/startup_program.pdmodel")
paddle.save(main_program, path1) paddle.save(main_program, path1)
paddle.save(startup_program, path2) paddle.save(startup_program, path2)
...@@ -924,11 +983,13 @@ class TestSaveLoadProgram(unittest.TestCase): ...@@ -924,11 +983,13 @@ class TestSaveLoadProgram(unittest.TestCase):
load_startup = paddle.load(path2).desc.serialize_to_string() load_startup = paddle.load(path2).desc.serialize_to_string()
self.assertTrue(origin_main == load_main) self.assertTrue(origin_main == load_main)
self.assertTrue(origin_startup == load_startup) self.assertTrue(origin_startup == load_startup)
temp_dir.cleanup()
class TestSaveLoadLayer(unittest.TestCase): class TestSaveLoadLayer(unittest.TestCase):
def test_save_load_layer(self): def test_save_load_layer(self):
paddle.disable_static() paddle.disable_static()
temp_dir = tempfile.TemporaryDirectory()
inps = paddle.randn([1, IMAGE_SIZE], dtype='float32') inps = paddle.randn([1, IMAGE_SIZE], dtype='float32')
layer1 = LinearNet() layer1 = LinearNet()
layer2 = LinearNet() layer2 = LinearNet()
...@@ -936,9 +997,11 @@ class TestSaveLoadLayer(unittest.TestCase): ...@@ -936,9 +997,11 @@ class TestSaveLoadLayer(unittest.TestCase):
layer2.eval() layer2.eval()
origin_layer = (layer1, layer2) origin_layer = (layer1, layer2)
origin = (layer1(inps), layer2(inps)) origin = (layer1(inps), layer2(inps))
path = "test_save_load_layer_/layer.pdmodel" path = os.path.join(temp_dir.name,
"test_save_load_layer_/layer.pdmodel")
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.save(origin_layer, path) paddle.save(origin_layer, path)
temp_dir.cleanup()
if __name__ == '__main__': if __name__ == '__main__':
......
...@@ -21,6 +21,7 @@ import os ...@@ -21,6 +21,7 @@ import os
import sys import sys
import six import six
import platform import platform
import tempfile
import paddle import paddle
import paddle.nn as nn import paddle.nn as nn
...@@ -37,6 +38,10 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -37,6 +38,10 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
def setUp(self): def setUp(self):
# enable static graph mode # enable static graph mode
paddle.enable_static() paddle.enable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_zero(self, prog, place, scope=None): def set_zero(self, prog, place, scope=None):
if scope is None: if scope is None:
...@@ -94,7 +99,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -94,7 +99,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
# test for replace_save_vars/io.load_vars # test for replace_save_vars/io.load_vars
path_vars1 = 'test_replace_save_load_vars_binary1/model' path_vars1 = os.path.join(
self.temp_dir.name, 'test_replace_save_load_vars_binary1/model')
self.replace_save_vars(prog, path_vars1) self.replace_save_vars(prog, path_vars1)
# set var to zero # set var to zero
self.set_zero(prog, place) self.set_zero(prog, place)
...@@ -111,7 +117,9 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -111,7 +117,9 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
# test for io.save_vars/replace_load_vars # test for io.save_vars/replace_load_vars
path_vars2 = 'test_replace_save_load_vars_binary2/model/' path_vars2 = os.path.join(
self.temp_dir.name,
'test_replace_save_load_vars_binary2/model/')
fluid.io.save_vars( fluid.io.save_vars(
exe, path_vars2, main_program=prog, vars=var_list) exe, path_vars2, main_program=prog, vars=var_list)
self.set_zero(prog, place) self.set_zero(prog, place)
...@@ -141,7 +149,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -141,7 +149,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
prog = paddle.static.default_main_program() prog = paddle.static.default_main_program()
exe.run(fluid.default_startup_program()) exe.run(fluid.default_startup_program())
dirname = 'test_save_load_lod_tensor1/tensor_' dirname = os.path.join(self.temp_dir.name,
'test_save_load_lod_tensor1/tensor_')
for var in prog.list_vars(): for var in prog.list_vars():
if var.persistable and list( if var.persistable and list(
var.shape) == [IMAGE_SIZE, OUTPUT_NUM]: var.shape) == [IMAGE_SIZE, OUTPUT_NUM]:
...@@ -162,12 +171,13 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -162,12 +171,13 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self.assertTrue(np.array_equal(origin, to_array)) self.assertTrue(np.array_equal(origin, to_array))
with self.assertRaises(NotImplementedError): with self.assertRaises(NotImplementedError):
path = 'test_save_load_error/temp' path = os.path.join(self.temp_dir.name, 'test_save_load_error/temp')
paddle.save({}, path, use_binary_format=True) paddle.save({}, path, use_binary_format=True)
# On the Windows platform, when parsing a string that can't be parsed as a `Program`, `desc_.ParseFromString` has a timeout risk. # On the Windows platform, when parsing a string that can't be parsed as a `Program`, `desc_.ParseFromString` has a timeout risk.
if 'Windows' != platform.system(): if 'Windows' != platform.system():
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
path = 'test_save_load_error/temp' path = os.path.join(self.temp_dir.name,
'test_save_load_error/temp')
with open(path, "w") as f: with open(path, "w") as f:
f.write('\0') f.write('\0')
paddle.load(path) paddle.load(path)
...@@ -178,11 +188,17 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -178,11 +188,17 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError):
fluid.core.save_lod_tensor( fluid.core.save_lod_tensor(
temp_lod, 'test_save_load_error_not_exist_file/not_exist_file') temp_lod,
os.path.join(
self.temp_dir.name,
'test_save_load_error_not_exist_file/not_exist_file'))
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError):
fluid.core.load_lod_tensor( fluid.core.load_lod_tensor(
temp_lod, 'test_save_load_error_not_exist_file/not_exist_file') temp_lod,
os.path.join(
self.temp_dir.name,
'test_save_load_error_not_exist_file/not_exist_file'))
# save to memory # save to memory
byio = BytesIO() byio = BytesIO()
...@@ -206,7 +222,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -206,7 +222,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
rows = [0, 4, 7] rows = [0, 4, 7]
row_numel = 12 row_numel = 12
selected_rows = fluid.core.SelectedRows(rows, height) selected_rows = fluid.core.SelectedRows(rows, height)
path = 'test_paddle_save_load_selected_rows/sr.pdsr' path = os.path.join(self.temp_dir.name,
'test_paddle_save_load_selected_rows/sr.pdsr')
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.save(selected_rows, path, use_binary_format=True) paddle.save(selected_rows, path, use_binary_format=True)
...@@ -227,11 +244,15 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -227,11 +244,15 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError):
fluid.core.save_selected_rows( fluid.core.save_selected_rows(
selected_rows, selected_rows,
'test_paddle_save_load_selected_rows_not_exist_file/temp') os.path.join(
self.temp_dir.name,
'test_paddle_save_load_selected_rows_not_exist_file/temp'))
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError):
fluid.core.load_selected_rows( fluid.core.load_selected_rows(
selected_rows, selected_rows,
'test_paddle_save_load_selected_rows_not_exist_file/temp') os.path.join(
self.temp_dir.name,
'test_paddle_save_load_selected_rows_not_exist_file/temp'))
# save to memory # save to memory
byio = BytesIO() byio = BytesIO()
......
...@@ -30,6 +30,7 @@ import six ...@@ -30,6 +30,7 @@ import six
import pickle import pickle
import os import os
import errno import errno
import tempfile
paddle.enable_static() paddle.enable_static()
...@@ -230,6 +231,7 @@ class TestSaveLoadBase(unittest.TestCase): ...@@ -230,6 +231,7 @@ class TestSaveLoadBase(unittest.TestCase):
init_scale = 0.1 init_scale = 0.1
batch_size = 4 batch_size = 4
batch_num = 200 batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope(): with new_program_scope():
fluid.default_startup_program().random_seed = seed fluid.default_startup_program().random_seed = seed
...@@ -297,7 +299,7 @@ class TestSaveLoadBase(unittest.TestCase): ...@@ -297,7 +299,7 @@ class TestSaveLoadBase(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
fluid.save(main_program, "./test_1") fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -310,7 +312,8 @@ class TestSaveLoadBase(unittest.TestCase): ...@@ -310,7 +312,8 @@ class TestSaveLoadBase(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero # make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "./test_1.pdparams", exe) fluid.load(main_program,
os.path.join(temp_dir.name, "test_1.pdparams"), exe)
for var in main_program.list_vars(): for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable: if isinstance(var, framework.Parameter) or var.persistable:
...@@ -318,6 +321,7 @@ class TestSaveLoadBase(unittest.TestCase): ...@@ -318,6 +321,7 @@ class TestSaveLoadBase(unittest.TestCase):
.get_tensor()) .get_tensor())
base_t = base_map[var.name] base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestSaveLoadPartial(unittest.TestCase): class TestSaveLoadPartial(unittest.TestCase):
...@@ -334,6 +338,7 @@ class TestSaveLoadPartial(unittest.TestCase): ...@@ -334,6 +338,7 @@ class TestSaveLoadPartial(unittest.TestCase):
init_scale = 0.1 init_scale = 0.1
batch_size = 4 batch_size = 4
batch_num = 200 batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope(): with new_program_scope():
fluid.default_startup_program().random_seed = seed fluid.default_startup_program().random_seed = seed
...@@ -409,7 +414,7 @@ class TestSaveLoadPartial(unittest.TestCase): ...@@ -409,7 +414,7 @@ class TestSaveLoadPartial(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
fluid.save(main_program, "./test_1") fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -422,7 +427,8 @@ class TestSaveLoadPartial(unittest.TestCase): ...@@ -422,7 +427,8 @@ class TestSaveLoadPartial(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero # make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(test_program, "./test_1.pdopt", None) fluid.load(test_program,
os.path.join(temp_dir.name, "test_1.pdopt"), None)
for var in test_program.list_vars(): for var in test_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable: if isinstance(var, framework.Parameter) or var.persistable:
...@@ -430,7 +436,9 @@ class TestSaveLoadPartial(unittest.TestCase): ...@@ -430,7 +436,9 @@ class TestSaveLoadPartial(unittest.TestCase):
.get_tensor()) .get_tensor())
base_t = base_map[var.name] base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
fluid.load(test_program, "./test_1.pdmodel", None) fluid.load(test_program,
os.path.join(temp_dir.name, "test_1.pdmodel"), None)
temp_dir.cleanup()
class TestSaveLoadSetStateDict(unittest.TestCase): class TestSaveLoadSetStateDict(unittest.TestCase):
...@@ -447,6 +455,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): ...@@ -447,6 +455,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
init_scale = 0.1 init_scale = 0.1
batch_size = 4 batch_size = 4
batch_num = 200 batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope(): with new_program_scope():
fluid.default_startup_program().random_seed = seed fluid.default_startup_program().random_seed = seed
...@@ -514,7 +523,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): ...@@ -514,7 +523,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
fluid.save(main_program, "./test_1") fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -527,7 +536,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): ...@@ -527,7 +536,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero # make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "./test_1", exe) fluid.load(main_program, os.path.join(temp_dir.name, "test_1"), exe)
for var in main_program.list_vars(): for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable: if isinstance(var, framework.Parameter) or var.persistable:
...@@ -535,6 +544,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): ...@@ -535,6 +544,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
.get_tensor()) .get_tensor())
base_t = base_map[var.name] base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestProgramStatePartial(unittest.TestCase): class TestProgramStatePartial(unittest.TestCase):
...@@ -551,6 +561,7 @@ class TestProgramStatePartial(unittest.TestCase): ...@@ -551,6 +561,7 @@ class TestProgramStatePartial(unittest.TestCase):
init_scale = 0.1 init_scale = 0.1
batch_size = 4 batch_size = 4
batch_num = 200 batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope(): with new_program_scope():
fluid.default_startup_program().random_seed = seed fluid.default_startup_program().random_seed = seed
...@@ -626,7 +637,7 @@ class TestProgramStatePartial(unittest.TestCase): ...@@ -626,7 +637,7 @@ class TestProgramStatePartial(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
fluid.save(main_program, os.path.join('some_dir', 'test_1')) fluid.save(main_program, os.path.join(temp_dir.name, 'test_1'))
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -641,16 +652,16 @@ class TestProgramStatePartial(unittest.TestCase): ...@@ -641,16 +652,16 @@ class TestProgramStatePartial(unittest.TestCase):
#fluid.load(test_program, "./test_1", None ) #fluid.load(test_program, "./test_1", None )
program_state = fluid.load_program_state( program_state = fluid.load_program_state(
os.path.join('some_dir', 'test_1')) os.path.join(temp_dir.name, 'test_1'))
program_state_1 = fluid.load_program_state( program_state_1 = fluid.load_program_state(
os.path.join('some_dir', 'test_1.pdparams')) os.path.join(temp_dir.name, 'test_1.pdparams'))
program_state_2 = fluid.load_program_state( program_state_2 = fluid.load_program_state(
os.path.join('some_dir', 'test_1.pdopt')) os.path.join(temp_dir.name, 'test_1.pdopt'))
program_state_3 = fluid.load_program_state( program_state_3 = fluid.load_program_state(
os.path.join('some_dir', 'test_1.pdmodel')) os.path.join(temp_dir.name, 'test_1.pdmodel'))
fluid.set_program_state(test_program, program_state) fluid.set_program_state(test_program, program_state)
...@@ -720,6 +731,7 @@ class TestProgramStatePartial(unittest.TestCase): ...@@ -720,6 +731,7 @@ class TestProgramStatePartial(unittest.TestCase):
.get_tensor()) .get_tensor())
base_t = base_map[var.name] base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestVariableInit(unittest.TestCase): class TestVariableInit(unittest.TestCase):
...@@ -737,7 +749,9 @@ class TestVariableInit(unittest.TestCase): ...@@ -737,7 +749,9 @@ class TestVariableInit(unittest.TestCase):
exe = fluid.Executor(place) exe = fluid.Executor(place)
exe.run(fluid.default_startup_program()) exe.run(fluid.default_startup_program())
fluid.save(fluid.default_main_program(), "./test_path") temp_dir = tempfile.TemporaryDirectory()
fluid.save(fluid.default_main_program(),
os.path.join(temp_dir.name, "test_path"))
def set_var(var, ndarray): def set_var(var, ndarray):
t = var.get_tensor() t = var.get_tensor()
...@@ -763,7 +777,7 @@ class TestVariableInit(unittest.TestCase): ...@@ -763,7 +777,7 @@ class TestVariableInit(unittest.TestCase):
fluid.core._create_loaded_parameter(parameter_list, new_scope, fluid.core._create_loaded_parameter(parameter_list, new_scope,
exe._default_executor) exe._default_executor)
parameter_file_name = "./test_path.pdparams" parameter_file_name = os.path.join(temp_dir.name, "test_path.pdparams")
with open(parameter_file_name, 'rb') as f: with open(parameter_file_name, 'rb') as f:
load_dict = pickle.load(f) load_dict = pickle.load(f)
...@@ -779,7 +793,7 @@ class TestVariableInit(unittest.TestCase): ...@@ -779,7 +793,7 @@ class TestVariableInit(unittest.TestCase):
fluid.core._create_loaded_parameter(opt_list, new_scope, fluid.core._create_loaded_parameter(opt_list, new_scope,
exe._default_executor) exe._default_executor)
opt_file_name = "./test_path.pdopt" opt_file_name = os.path.join(temp_dir.name, "test_path.pdopt")
with open(opt_file_name, 'rb') as f: with open(opt_file_name, 'rb') as f:
load_dict = pickle.load(f) load_dict = pickle.load(f)
...@@ -805,6 +819,7 @@ class TestVariableInit(unittest.TestCase): ...@@ -805,6 +819,7 @@ class TestVariableInit(unittest.TestCase):
base_t = base_map[var.name] base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestLoadFromOldInterface(unittest.TestCase): class TestLoadFromOldInterface(unittest.TestCase):
...@@ -815,10 +830,15 @@ class TestLoadFromOldInterface(unittest.TestCase): ...@@ -815,10 +830,15 @@ class TestLoadFromOldInterface(unittest.TestCase):
if os.path.exists("test_static_load_var_list.pdparams"): if os.path.exists("test_static_load_var_list.pdparams"):
os.remove("test_static_load_var_list.pdparams") os.remove("test_static_load_var_list.pdparams")
self.temp_dir = tempfile.TemporaryDirectory()
def set_place(self): def set_place(self):
return fluid.CPUPlace() if not core.is_compiled_with_cuda( return fluid.CPUPlace() if not core.is_compiled_with_cuda(
) else fluid.CUDAPlace(0) ) else fluid.CUDAPlace(0)
def tearDown(self):
self.temp_dir.cleanup()
def test_load_from_old_interface(self): def test_load_from_old_interface(self):
seed = 90 seed = 90
hidden_size = 10 hidden_size = 10
...@@ -898,7 +918,9 @@ class TestLoadFromOldInterface(unittest.TestCase): ...@@ -898,7 +918,9 @@ class TestLoadFromOldInterface(unittest.TestCase):
base_map[var.name] = t base_map[var.name] = t
#fluid.save(main_program, "./test_1") #fluid.save(main_program, "./test_1")
fluid.io.save_persistables(exe, "test_path", main_program) fluid.io.save_persistables(
exe,
os.path.join(self.temp_dir.name, "test_path"), main_program)
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -911,7 +933,8 @@ class TestLoadFromOldInterface(unittest.TestCase): ...@@ -911,7 +933,8 @@ class TestLoadFromOldInterface(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero # make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "test_path", exe) fluid.load(main_program,
os.path.join(self.temp_dir.name, "test_path"), exe)
for var in main_program.list_vars(): for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable: if isinstance(var, framework.Parameter) or var.persistable:
...@@ -928,11 +951,13 @@ class TestLoadFromOldInterface(unittest.TestCase): ...@@ -928,11 +951,13 @@ class TestLoadFromOldInterface(unittest.TestCase):
var.desc.set_shape(new_shape) var.desc.set_shape(new_shape)
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError):
fluid.load(main_program, "test_path", exe) fluid.load(main_program,
os.path.join(self.temp_dir.name, "test_path"), exe)
# check unused parameter # check unused parameter
fluid.load(test_clone_program, "test_path", exe) fluid.load(test_clone_program,
os.path.join(self.temp_dir.name, "test_path"), exe)
def test_load_from_old_interface_var_list(self): def test_load_from_old_interface_var_list(self):
seed = 90 seed = 90
...@@ -1013,8 +1038,10 @@ class TestLoadFromOldInterface(unittest.TestCase): ...@@ -1013,8 +1038,10 @@ class TestLoadFromOldInterface(unittest.TestCase):
base_map[var.name] = t base_map[var.name] = t
#fluid.save(main_program, "./test_1") #fluid.save(main_program, "./test_1")
fluid.io.save_persistables(exe, "test_static_load_var_list", fluid.io.save_persistables(
main_program) exe,
os.path.join(self.temp_dir.name, "test_static_load_var_list"),
main_program)
# set var to zero # set var to zero
var_list = [] var_list = []
...@@ -1030,7 +1057,10 @@ class TestLoadFromOldInterface(unittest.TestCase): ...@@ -1030,7 +1057,10 @@ class TestLoadFromOldInterface(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero # make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "test_static_load_var_list", exe, var_list) fluid.load(
main_program,
os.path.join(self.temp_dir.name, "test_static_load_var_list"),
exe, var_list)
var_list_names = [var.name for var in var_list] var_list_names = [var.name for var in var_list]
for var in main_program.list_vars(): for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable: if isinstance(var, framework.Parameter) or var.persistable:
...@@ -1059,6 +1089,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): ...@@ -1059,6 +1089,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
init_scale = 0.1 init_scale = 0.1
batch_size = 4 batch_size = 4
batch_num = 200 batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope(): with new_program_scope():
fluid.default_startup_program().random_seed = seed fluid.default_startup_program().random_seed = seed
...@@ -1125,10 +1156,10 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): ...@@ -1125,10 +1156,10 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update # make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
save_dir = os.path.join(temp_dir.name, "test_path")
#fluid.save(main_program, "./test_1") #fluid.save(main_program, "./test_1")
fluid.io.save_persistables( fluid.io.save_persistables(
exe, "test_path", main_program, filename="model_single") exe, save_dir, main_program, filename="model_single")
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -1141,7 +1172,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): ...@@ -1141,7 +1172,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero # make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
file_model_path = os.path.join("test_path", "model_single") file_model_path = os.path.join(save_dir, "model_single")
fluid.load(main_program, file_model_path, exe, fluid.load(main_program, file_model_path, exe,
fluid.io.get_program_persistable_vars(main_program)) fluid.io.get_program_persistable_vars(main_program))
...@@ -1190,11 +1221,16 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): ...@@ -1190,11 +1221,16 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
all_var_list = list(main_program.list_vars()) all_var_list = list(main_program.list_vars())
fluid.load(main_program, file_model_path, exe, fluid.load(main_program, file_model_path, exe,
all_var_list + [temp_var]) all_var_list + [temp_var])
temp_dir.cleanup()
class TestProgramStateOldSave(unittest.TestCase): class TestProgramStateOldSave(unittest.TestCase):
def setUp(self): def setUp(self):
self.test_dygraph = True self.test_dygraph = True
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_place(self): def set_place(self):
return fluid.CPUPlace() if not core.is_compiled_with_cuda( return fluid.CPUPlace() if not core.is_compiled_with_cuda(
...@@ -1283,8 +1319,8 @@ class TestProgramStateOldSave(unittest.TestCase): ...@@ -1283,8 +1319,8 @@ class TestProgramStateOldSave(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update # make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
save_dir = os.path.join(self.temp_dir.name, "test_program_1")
fluid.io.save_persistables(exe, "test_program_1", main_program) fluid.io.save_persistables(exe, save_dir, main_program)
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -1298,7 +1334,7 @@ class TestProgramStateOldSave(unittest.TestCase): ...@@ -1298,7 +1334,7 @@ class TestProgramStateOldSave(unittest.TestCase):
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
# case 1: load basic # case 1: load basic
program_state = fluid.load_program_state("test_program_1") program_state = fluid.load_program_state(save_dir)
fluid.set_program_state(main_program, program_state) fluid.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map) self.check_in_static(main_program, base_map)
...@@ -1313,24 +1349,20 @@ class TestProgramStateOldSave(unittest.TestCase): ...@@ -1313,24 +1349,20 @@ class TestProgramStateOldSave(unittest.TestCase):
else: else:
raise e raise e
orig_filepath = './test_program_1/fc_0.w_0' program_state = fluid.load_program_state(save_dir)
symlink_filepath = './test_program_1/link_fc_0.w_0'
# create a needless link file for coverage
symlink_force(orig_filepath, symlink_filepath)
program_state = fluid.load_program_state("test_program_1")
fluid.set_program_state(main_program, program_state) fluid.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map) self.check_in_static(main_program, base_map)
# case 3: load with var_list # case 3: load with var_list
program_state = fluid.load_program_state( program_state = fluid.load_program_state(
"test_program_1", main_program.all_parameters()) save_dir, main_program.all_parameters())
fluid.set_program_state(main_program, program_state) fluid.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map) self.check_in_static(main_program, base_map)
if self.test_dygraph: if self.test_dygraph:
# make sure `load_program_state` can be used in dynamic graph mode # make sure `load_program_state` can be used in dynamic graph mode
with fluid.dygraph.guard(place): with fluid.dygraph.guard(place):
load_state = fluid.load_program_state("test_program_1") load_state = fluid.load_program_state(save_dir)
for k, v in load_state.items(): for k, v in load_state.items():
self.assertTrue(np.array_equal(base_map[k], v)) self.assertTrue(np.array_equal(base_map[k], v))
...@@ -1365,6 +1397,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): ...@@ -1365,6 +1397,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
init_scale = 0.1 init_scale = 0.1
batch_size = 4 batch_size = 4
batch_num = 200 batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope(): with new_program_scope():
fluid.default_startup_program().random_seed = seed fluid.default_startup_program().random_seed = seed
...@@ -1440,8 +1473,9 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): ...@@ -1440,8 +1473,9 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
save_dir = os.path.join(temp_dir.name, "test_program_2")
fluid.io.save_persistables( fluid.io.save_persistables(
exe, "test_program_2", main_program, filename="model_1") exe, save_dir, main_program, filename="model_1")
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -1456,7 +1490,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): ...@@ -1456,7 +1490,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
#fluid.load(test_program, "./test_1", None ) #fluid.load(test_program, "./test_1", None )
program_state = fluid.load_program_state( program_state = fluid.load_program_state(
os.path.join("test_program_2", "model_1"), os.path.join(save_dir, "model_1"),
var_list=fluid.io.get_program_persistable_vars(main_program)) var_list=fluid.io.get_program_persistable_vars(main_program))
fluid.set_program_state(main_program, program_state) fluid.set_program_state(main_program, program_state)
...@@ -1468,21 +1502,20 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): ...@@ -1468,21 +1502,20 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
fluid.load_program_state( fluid.load_program_state(os.path.join(save_dir, "model_1"))
os.path.join("test_program_2", "model_1"))
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
fluid.load_program_state( fluid.load_program_state(
os.path.join("test_program_2", "model_1"), os.path.join(save_dir, "model_1"), var_list=["str"])
var_list=["str"])
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError):
fluid.load_program_state( fluid.load_program_state(
os.path.join("test_program_2", "model_1"), os.path.join(save_dir, "model_1"),
var_list=[ var_list=[
main_program.global_block().create_var( main_program.global_block().create_var(
name="fake_var_name", persistable=True) name="fake_var_name", persistable=True)
]) ])
temp_dir.cleanup()
class TestStaticSaveLoadPickle(unittest.TestCase): class TestStaticSaveLoadPickle(unittest.TestCase):
...@@ -1511,7 +1544,8 @@ class TestStaticSaveLoadPickle(unittest.TestCase): ...@@ -1511,7 +1544,8 @@ class TestStaticSaveLoadPickle(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
path = os.path.join("test_static_save_load_pickle", temp_dir = tempfile.TemporaryDirectory()
path = os.path.join(temp_dir.name, "test_static_save_load_pickle",
"pickle_protocol") "pickle_protocol")
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
......
...@@ -23,11 +23,19 @@ from paddle.fluid.optimizer import SGDOptimizer ...@@ -23,11 +23,19 @@ from paddle.fluid.optimizer import SGDOptimizer
from paddle.fluid.tests.unittests.test_imperative_base import new_program_scope from paddle.fluid.tests.unittests.test_imperative_base import new_program_scope
from paddle.fluid.tests.unittests.test_static_save_load import PtbModel from paddle.fluid.tests.unittests.test_static_save_load import PtbModel
import numpy as np import numpy as np
import tempfile
import os
@unittest.skipIf(not core.supports_bfloat16(), @unittest.skipIf(not core.supports_bfloat16(),
"place does not support BF16 evaluation") "place does not support BF16 evaluation")
class TestSaveLoadBF16(unittest.TestCase): class TestSaveLoadBF16(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_place(self): def set_place(self):
return fluid.CPUPlace() return fluid.CPUPlace()
...@@ -108,8 +116,8 @@ class TestSaveLoadBF16(unittest.TestCase): ...@@ -108,8 +116,8 @@ class TestSaveLoadBF16(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update # make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
save_dir = os.path.join(self.temp_dir.name, "test_1")
fluid.save(main_program, "./test_1") fluid.save(main_program, save_dir)
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -122,7 +130,8 @@ class TestSaveLoadBF16(unittest.TestCase): ...@@ -122,7 +130,8 @@ class TestSaveLoadBF16(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero # make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "./test_1.pdparams", exe) fluid.load(main_program,
os.path.join(self.temp_dir.name, "test_1.pdparams"), exe)
for var in main_program.list_vars(): for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable: if isinstance(var, framework.Parameter) or var.persistable:
......
...@@ -23,6 +23,7 @@ from test_imperative_base import new_program_scope ...@@ -23,6 +23,7 @@ from test_imperative_base import new_program_scope
import numpy as np import numpy as np
import pickle import pickle
import os import os
import tempfile
LARGE_PARAM = 2**26 LARGE_PARAM = 2**26
...@@ -51,9 +52,10 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): ...@@ -51,9 +52,10 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update # make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
temp_dir = tempfile.TemporaryDirectory()
path = os.path.join("test_static_save_load_large_param", path = os.path.join(temp_dir.name,
"static_save") "test_static_save_load_large_param")
path = os.path.join(path, "static_save")
protocol = 4 protocol = 4
paddle.fluid.save(prog, path, pickle_protocol=protocol) paddle.fluid.save(prog, path, pickle_protocol=protocol)
# set var to zero # set var to zero
...@@ -93,6 +95,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): ...@@ -93,6 +95,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
.get_tensor()) .get_tensor())
base_t = base_map[var.name] base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
if __name__ == '__main__': if __name__ == '__main__':
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册