Skip to content

P
Paddle

PaddlePaddle / Paddle
1 年多 前同步成功

通知 2302
Star 20931
Fork 5422
P
Paddle
未验证 提交 2eb60ddb 编写于 作者: Y YuanRisheng 提交者： GitHub
浏览文件
操作

cherry pick 43581 (#43596)

上级 3689a126
显示空白变更内容
内联 并排
Showing with 501 addition and 164 deletion
python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py
浏览文件 @ 2eb60ddb
...@@ -27,6 +27,8 @@ from paddle.fluid.layer_helper import LayerHelper ...@@ -27,6 +27,8 @@ from paddle.fluid.layer_helper import LayerHelper
from paddle import _C_ops from paddle import _C_ops
import sys import sys
import tempfile
sys.path.append("./tokenizer") sys.path.append("./tokenizer")
from tokenizer.bert_tokenizer import BertTokenizer from tokenizer.bert_tokenizer import BertTokenizer
...@@ -149,13 +151,17 @@ class Predictor(object): ...@@ -149,13 +151,17 @@ class Predictor(object):
class TestBertTokenizerOp(unittest.TestCase): class TestBertTokenizerOp(unittest.TestCase):
def setUp(self): def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.bert_tokenizer = BertTokenizer.from_pretrained("bert-base-chinese") self.bert_tokenizer = BertTokenizer.from_pretrained("bert-base-chinese")
self.faster_tokenizer = FasterTokenizer(self.bert_tokenizer.vocab) self.faster_tokenizer = FasterTokenizer(self.bert_tokenizer.vocab)
self.init_data() self.init_data()
self.save_path = os.path.join(DATA_HOME, "fast_tokenizer") self.save_path = os.path.join(self.temp_dir.name, "fast_tokenizer")
self.param_path = os.path.join(self.save_path, "model.pdparams") self.param_path = os.path.join(self.save_path, "model.pdparams")
self.inference_path = os.path.join(self.save_path, "inference") self.inference_path = os.path.join(self.save_path, "inference")
def tearDown(self):
self.temp_dir.cleanup()
def init_data(self): def init_data(self):
self.text = [ self.text = [
'选择珠江花园的原因就是方便，有电动扶梯直接到达海边，周围餐馆、食廊、商场、超市、摊位一应俱全。' '选择珠江花园的原因就是方便，有电动扶梯直接到达海边，周围餐馆、食廊、商场、超市、摊位一应俱全。'
......
python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py
浏览文件 @ 2eb60ddb
...@@ -17,11 +17,14 @@ import paddle.fluid as fluid ...@@ -17,11 +17,14 @@ import paddle.fluid as fluid
import paddle.fluid.framework as framework import paddle.fluid.framework as framework
from paddle.fluid.dygraph.nn import * from paddle.fluid.dygraph.nn import *
import numpy as np import numpy as np
import os
import tempfile
class TestDygraphLoadStatic(unittest.TestCase): class TestDygraphLoadStatic(unittest.TestCase):
def testLoadStaticModel(self): def testLoadStaticModel(self):
# static mode # static mode
temp_dir = tempfile.TemporaryDirectory()
a = fluid.data(name="a", shape=[10, 10]) a = fluid.data(name="a", shape=[10, 10])
conv_in = fluid.data(name="conv_in", shape=[None, 10, 10, 10]) conv_in = fluid.data(name="conv_in", shape=[None, 10, 10, 10])
...@@ -121,9 +124,11 @@ class TestDygraphLoadStatic(unittest.TestCase): ...@@ -121,9 +124,11 @@ class TestDygraphLoadStatic(unittest.TestCase):
) if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0)) ) if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0))
out = exe.run(framework.default_startup_program()) out = exe.run(framework.default_startup_program())
fluid.save(framework.default_main_program(), "./test_1") fluid.save(framework.default_main_program(),
os.path.join(temp_dir.name, "test_1"))
para_dict = fluid.load_program_state("./test_1") para_dict = fluid.load_program_state(
os.path.join(temp_dir.name, "test_1"))
new_dict = {} new_dict = {}
for k, v in para_dict.items(): for k, v in para_dict.items():
...@@ -192,6 +197,7 @@ class TestDygraphLoadStatic(unittest.TestCase): ...@@ -192,6 +197,7 @@ class TestDygraphLoadStatic(unittest.TestCase):
my_test.set_dict(new_dict, use_structured_name=False) my_test.set_dict(new_dict, use_structured_name=False)
for k, v in my_test.state_dict().items(): for k, v in my_test.state_dict().items():
self.assertTrue(np.array_equal(v.numpy(), new_dict[v.name])) self.assertTrue(np.array_equal(v.numpy(), new_dict[v.name]))
temp_dir.cleanup()
if __name__ == '__main__': if __name__ == '__main__':
......
python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py
浏览文件 @ 2eb60ddb
...@@ -29,6 +29,8 @@ import six ...@@ -29,6 +29,8 @@ import six
import paddle import paddle
from paddle.fluid.framework import _test_eager_guard from paddle.fluid.framework import _test_eager_guard
import tempfile
class SimpleLSTMRNN(fluid.Layer): class SimpleLSTMRNN(fluid.Layer):
def __init__(self, def __init__(self,
...@@ -209,6 +211,12 @@ class PtbModel(fluid.Layer): ...@@ -209,6 +211,12 @@ class PtbModel(fluid.Layer):
class TestDygraphPtbRnn(unittest.TestCase): class TestDygraphPtbRnn(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def func_setUp(self): def func_setUp(self):
seed = 90 seed = 90
hidden_size = 10 hidden_size = 10
...@@ -286,7 +294,8 @@ class TestDygraphPtbRnn(unittest.TestCase): ...@@ -286,7 +294,8 @@ class TestDygraphPtbRnn(unittest.TestCase):
else: else:
self.base_opti[k] = v self.base_opti[k] = v
paddle.save(self.opti_dict, "./test_dy_v2.pdopt") paddle.save(self.opti_dict,
os.path.join(self.temp_dir.name, "test_dy_v2.pdopt"))
self.state_dict = ptb_model.state_dict() self.state_dict = ptb_model.state_dict()
...@@ -295,10 +304,10 @@ class TestDygraphPtbRnn(unittest.TestCase): ...@@ -295,10 +304,10 @@ class TestDygraphPtbRnn(unittest.TestCase):
np_t = v.numpy() np_t = v.numpy()
self.model_base[k] = np_t self.model_base[k] = np_t
paddle.save(self.state_dict, "./test_dy_v2.pdparams") paddle.save(self.state_dict,
os.path.join(self.temp_dir.name, "test_dy_v2.pdparams"))
def func_testLoadAndSetVarBase(self): def func_testLoadAndSetVarBase(self):
self.setUp()
seed = 90 seed = 90
hidden_size = 10 hidden_size = 10
vocab_size = 1000 vocab_size = 1000
...@@ -375,8 +384,10 @@ class TestDygraphPtbRnn(unittest.TestCase): ...@@ -375,8 +384,10 @@ class TestDygraphPtbRnn(unittest.TestCase):
self.assertTrue(np.sum(np.abs(v.numpy())) == 0) self.assertTrue(np.sum(np.abs(v.numpy())) == 0)
para_state_dict = paddle.load("./test_dy_v2.pdparams") para_state_dict = paddle.load(
opti_state_dict = paddle.load("./test_dy_v2.pdopt") os.path.join(self.temp_dir.name, "test_dy_v2.pdparams"))
opti_state_dict = paddle.load(
os.path.join(self.temp_dir.name, "test_dy_v2.pdopt"))
adam.set_state_dict(opti_state_dict) adam.set_state_dict(opti_state_dict)
opti_dict = adam.state_dict() opti_dict = adam.state_dict()
...@@ -750,7 +761,8 @@ class TestDygraphPtbRnn(unittest.TestCase): ...@@ -750,7 +761,8 @@ class TestDygraphPtbRnn(unittest.TestCase):
last_hidden = None last_hidden = None
last_cell = None last_cell = None
state_dict, opti_dict = fluid.load_dygraph("./test_dy_v2") state_dict, opti_dict = fluid.load_dygraph(
os.path.join(self.temp_dir.name, "test_dy_v2"))
adam.set_state_dict(opti_dict) adam.set_state_dict(opti_dict)
ptb_model.set_dict(state_dict) ptb_model.set_dict(state_dict)
...@@ -907,19 +919,23 @@ class TestDygraphPtbRnn(unittest.TestCase): ...@@ -907,19 +919,23 @@ class TestDygraphPtbRnn(unittest.TestCase):
with fluid.dygraph.guard(): with fluid.dygraph.guard():
emb = fluid.dygraph.Embedding([10, 10]) emb = fluid.dygraph.Embedding([10, 10])
state_dict = emb.state_dict() state_dict = emb.state_dict()
paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams')) paddle.save(
state_dict,
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict = paddle.load( para_state_dict = paddle.load(
os.path.join('saved_dy', 'emb_dy.pdparams')) os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
def func_test_no_state_in_input_dict(self): def func_test_no_state_in_input_dict(self):
with fluid.dygraph.guard(): with fluid.dygraph.guard():
emb = fluid.dygraph.Embedding([10, 10]) emb = fluid.dygraph.Embedding([10, 10])
state_dict = emb.state_dict() state_dict = emb.state_dict()
paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams')) paddle.save(
state_dict,
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict = paddle.load( para_state_dict = paddle.load(
os.path.join('saved_dy', 'emb_dy.pdparams')) os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict.pop('weight') para_state_dict.pop('weight')
emb.set_state_dict(para_state_dict) emb.set_state_dict(para_state_dict)
...@@ -928,10 +944,13 @@ class TestDygraphPtbRnn(unittest.TestCase): ...@@ -928,10 +944,13 @@ class TestDygraphPtbRnn(unittest.TestCase):
with fluid.dygraph.guard(): with fluid.dygraph.guard():
emb = fluid.dygraph.Embedding([10, 10]) emb = fluid.dygraph.Embedding([10, 10])
state_dict = emb.state_dict() state_dict = emb.state_dict()
paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams')) paddle.save(
state_dict,
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict = paddle.load( para_state_dict = paddle.load(
os.path.join('saved_dy', 'emb_dy.pdparams'), return_numpy=True) os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'),
return_numpy=True)
para_state_dict['weight'] = np.expand_dims( para_state_dict['weight'] = np.expand_dims(
para_state_dict['weight'], axis=-1) para_state_dict['weight'], axis=-1)
......
python/paddle/fluid/tests/unittests/test_io_save_load.py
浏览文件 @ 2eb60ddb
...@@ -19,9 +19,18 @@ import paddle ...@@ -19,9 +19,18 @@ import paddle
import paddle.fluid as fluid import paddle.fluid as fluid
from paddle.fluid import core from paddle.fluid import core
from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph
import tempfile
import os
class TestSaveLoadAPIError(unittest.TestCase): class TestSaveLoadAPIError(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.save_dir = os.path.join(self.temp_dir.name, "fake_dir")
def tearDown(self):
self.temp_dir.cleanup()
def func_test_get_valid_program_error(self): def func_test_get_valid_program_error(self):
# case 1: CompiledProgram no program # case 1: CompiledProgram no program
graph = core.Graph(core.ProgramDesc()) graph = core.Graph(core.ProgramDesc())
...@@ -44,13 +53,13 @@ class TestSaveLoadAPIError(unittest.TestCase): ...@@ -44,13 +53,13 @@ class TestSaveLoadAPIError(unittest.TestCase):
# case 1: main_program type error when vars None # case 1: main_program type error when vars None
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
fluid.io.load_vars( fluid.io.load_vars(
executor=exe, dirname="./fake_dir", main_program="program") executor=exe, dirname=self.save_dir, main_program="program")
# case 2: main_program type error when vars not None # case 2: main_program type error when vars not None
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
fluid.io.load_vars( fluid.io.load_vars(
executor=exe, executor=exe,
dirname="./fake_dir", dirname=self.save_dir,
main_program="program", main_program="program",
vars="vars") vars="vars")
...@@ -61,6 +70,12 @@ class TestSaveLoadAPIError(unittest.TestCase): ...@@ -61,6 +70,12 @@ class TestSaveLoadAPIError(unittest.TestCase):
class TestSaveInferenceModelAPIError(unittest.TestCase): class TestSaveInferenceModelAPIError(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def func_test_useless_feeded_var_names(self): def func_test_useless_feeded_var_names(self):
start_prog = fluid.Program() start_prog = fluid.Program()
main_prog = fluid.Program() main_prog = fluid.Program()
...@@ -74,7 +89,7 @@ class TestSaveInferenceModelAPIError(unittest.TestCase): ...@@ -74,7 +89,7 @@ class TestSaveInferenceModelAPIError(unittest.TestCase):
with self.assertRaisesRegexp( with self.assertRaisesRegexp(
ValueError, "not involved in the target_vars calculation"): ValueError, "not involved in the target_vars calculation"):
fluid.io.save_inference_model( fluid.io.save_inference_model(
dirname='./model', dirname=os.path.join(self.temp_dir.name, 'model'),
feeded_var_names=['x', 'y'], feeded_var_names=['x', 'y'],
target_vars=[z], target_vars=[z],
executor=exe, executor=exe,
...@@ -87,13 +102,20 @@ class TestSaveInferenceModelAPIError(unittest.TestCase): ...@@ -87,13 +102,20 @@ class TestSaveInferenceModelAPIError(unittest.TestCase):
class TestWhenTrainWithNoGrad(unittest.TestCase): class TestWhenTrainWithNoGrad(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def func_test_when_train_with_no_grad(self): def func_test_when_train_with_no_grad(self):
paddle.disable_static() paddle.disable_static()
net = paddle.nn.Linear(1024, 1) net = paddle.nn.Linear(1024, 1)
net = paddle.jit.to_static(net) net = paddle.jit.to_static(net)
x = paddle.rand([1024], 'float32') x = paddle.rand([1024], 'float32')
net(x) net(x)
save_path = './train_with_no_grad' save_path = os.path.join(self.temp_dir.name, 'train_with_no_grad')
paddle.jit.save(net, save_path) paddle.jit.save(net, save_path)
net = paddle.jit.load(save_path) net = paddle.jit.load(save_path)
net.train() net.train()
......
python/paddle/fluid/tests/unittests/test_jit_save_load.py
浏览文件 @ 2eb60ddb
...@@ -18,6 +18,7 @@ import os ...@@ -18,6 +18,7 @@ import os
import pickle import pickle
import shutil import shutil
import unittest import unittest
import tempfile
import numpy as np import numpy as np
import paddle import paddle
from paddle.static import InputSpec from paddle.static import InputSpec
...@@ -333,13 +334,18 @@ def train_with_label(layer, input_size=784, label_size=1): ...@@ -333,13 +334,18 @@ def train_with_label(layer, input_size=784, label_size=1):
class TestJitSaveLoad(unittest.TestCase): class TestJitSaveLoad(unittest.TestCase):
def setUp(self): def setUp(self):
self.model_path = "test_jit_save_load/model" self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"test_jit_save_load/model")
# enable dygraph mode # enable dygraph mode
fluid.enable_dygraph() fluid.enable_dygraph()
# config seed # config seed
paddle.seed(SEED) paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED) paddle.framework.random._manual_program_seed(SEED)
def tearDown(self):
self.temp_dir.cleanup()
def train_and_save_model(self, model_path=None): def train_and_save_model(self, model_path=None):
layer = LinearNet(784, 1) layer = LinearNet(784, 1)
example_inputs, layer, _ = train(layer) example_inputs, layer, _ = train(layer)
...@@ -395,12 +401,14 @@ class TestJitSaveLoad(unittest.TestCase): ...@@ -395,12 +401,14 @@ class TestJitSaveLoad(unittest.TestCase):
np.array_equal(train_layer(x).numpy(), new_layer(x).numpy())) np.array_equal(train_layer(x).numpy(), new_layer(x).numpy()))
def test_load_dygraph_no_path(self): def test_load_dygraph_no_path(self):
model_path = "test_jit_save_load.no_path/model_path" model_path = os.path.join(self.temp_dir.name,
"test_jit_save_load.no_path/model_path")
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
model_dict, _ = fluid.dygraph.load_dygraph(model_path) model_dict, _ = fluid.dygraph.load_dygraph(model_path)
def test_jit_load_no_path(self): def test_jit_load_no_path(self):
path = "test_jit_save_load.no_path/model_path" path = os.path.join(self.temp_dir.name,
"test_jit_save_load.no_path/model_path")
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
loaded_layer = paddle.jit.load(path) loaded_layer = paddle.jit.load(path)
...@@ -409,6 +417,10 @@ class TestSaveLoadWithNestOut(unittest.TestCase): ...@@ -409,6 +417,10 @@ class TestSaveLoadWithNestOut(unittest.TestCase):
def setUp(self): def setUp(self):
# enable dygraph mode # enable dygraph mode
fluid.enable_dygraph() fluid.enable_dygraph()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_nest_output(self): def test_nest_output(self):
x = fluid.dygraph.to_variable( x = fluid.dygraph.to_variable(
...@@ -418,7 +430,7 @@ class TestSaveLoadWithNestOut(unittest.TestCase): ...@@ -418,7 +430,7 @@ class TestSaveLoadWithNestOut(unittest.TestCase):
dy_outs = flatten(net(x)) dy_outs = flatten(net(x))
net = declarative(net, input_spec=[InputSpec([None, 8], name='x')]) net = declarative(net, input_spec=[InputSpec([None, 8], name='x')])
model_path = "net_with_nest_out/model" model_path = os.path.join(self.temp_dir.name, "net_with_nest_out/model")
paddle.jit.save(net, model_path) paddle.jit.save(net, model_path)
load_net = paddle.jit.load(model_path) load_net = paddle.jit.load(model_path)
...@@ -440,8 +452,9 @@ class TestSaveLoadWithDictInput(unittest.TestCase): ...@@ -440,8 +452,9 @@ class TestSaveLoadWithDictInput(unittest.TestCase):
# {'img': var img : fluid.VarType.LOD_TENSOR.shape(-1, 8).astype(VarType.FP32)}, # {'img': var img : fluid.VarType.LOD_TENSOR.shape(-1, 8).astype(VarType.FP32)},
# {'label': var label : fluid.VarType.LOD_TENSOR.shape(-1, 1).astype(VarType.INT64)}) # {'label': var label : fluid.VarType.LOD_TENSOR.shape(-1, 1).astype(VarType.INT64)})
self.assertEqual(len(net.forward.concrete_program.inputs), 3) self.assertEqual(len(net.forward.concrete_program.inputs), 3)
temp_dir = tempfile.TemporaryDirectory()
path = "test_jit_save_load_with_dict_input/model" path = os.path.join(temp_dir.name,
"test_jit_save_load_with_dict_input/model")
# prune inputs # prune inputs
paddle.jit.save( paddle.jit.save(
layer=net, layer=net,
...@@ -458,13 +471,15 @@ class TestSaveLoadWithDictInput(unittest.TestCase): ...@@ -458,13 +471,15 @@ class TestSaveLoadWithDictInput(unittest.TestCase):
# loaded_net._input_spec(): # loaded_net._input_spec():
# [InputSpec(shape=(-1, 8), dtype=VarType.FP32, name=img)] # [InputSpec(shape=(-1, 8), dtype=VarType.FP32, name=img)]
self.assertEqual(len(loaded_net._input_spec()), 1) self.assertEqual(len(loaded_net._input_spec()), 1)
temp_dir.cleanup()
class TestSaveLoadWithDictInputNoPrune(unittest.TestCase): class TestSaveLoadWithDictInputNoPrune(unittest.TestCase):
def test_dict_input(self): def test_dict_input(self):
net = LinearNetWithDictInputNoPrune(8, 8) net = LinearNetWithDictInputNoPrune(8, 8)
temp_dir = tempfile.TemporaryDirectory()
path = "test_jit_save_load_with_dict_input_no_prune/model" path = os.path.join(temp_dir.name,
"test_jit_save_load_with_dict_input_no_prune/model")
# prune inputs # prune inputs
paddle.jit.save( paddle.jit.save(
layer=net, layer=net,
...@@ -482,12 +497,17 @@ class TestSaveLoadWithDictInputNoPrune(unittest.TestCase): ...@@ -482,12 +497,17 @@ class TestSaveLoadWithDictInputNoPrune(unittest.TestCase):
loaded_out = loaded_net(img, img2) loaded_out = loaded_net(img, img2)
self.assertEqual(len(loaded_net._input_spec()), 2) self.assertEqual(len(loaded_net._input_spec()), 2)
temp_dir.cleanup()
class TestSaveLoadWithInputSpec(unittest.TestCase): class TestSaveLoadWithInputSpec(unittest.TestCase):
def setUp(self): def setUp(self):
# enable dygraph mode # enable dygraph mode
fluid.enable_dygraph() fluid.enable_dygraph()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_with_input_spec(self): def test_with_input_spec(self):
net = LinearNetReturnLoss(8, 8) net = LinearNetReturnLoss(8, 8)
...@@ -496,7 +516,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): ...@@ -496,7 +516,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
net.forward, input_spec=[InputSpec( net.forward, input_spec=[InputSpec(
[None, 8], name='x')]) [None, 8], name='x')])
model_path = "input_spec.output_spec/model" model_path = os.path.join(self.temp_dir.name,
"input_spec.output_spec/model")
# check inputs and outputs # check inputs and outputs
self.assertTrue(len(net.forward.inputs) == 1) self.assertTrue(len(net.forward.inputs) == 1)
input_x = net.forward.inputs[0] input_x = net.forward.inputs[0]
...@@ -516,7 +537,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): ...@@ -516,7 +537,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
def test_multi_in_out(self): def test_multi_in_out(self):
net = LinearNetMultiInput(8, 8) net = LinearNetMultiInput(8, 8)
model_path = "multi_inout.output_spec1/model" model_path = os.path.join(self.temp_dir.name,
"multi_inout.output_spec1/model")
# 1. check inputs and outputs # 1. check inputs and outputs
self.assertTrue(len(net.forward.inputs) == 2) self.assertTrue(len(net.forward.inputs) == 2)
input_x = net.forward.inputs[0] input_x = net.forward.inputs[0]
...@@ -538,7 +560,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): ...@@ -538,7 +560,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
pred_x, pred_y = infer_layer(x, y) pred_x, pred_y = infer_layer(x, y)
# 1. prune y and loss # 1. prune y and loss
model_path = "multi_inout.output_spec2/model" model_path = os.path.join(self.temp_dir.name,
"multi_inout.output_spec2/model")
output_spec = net.forward.outputs[:1] output_spec = net.forward.outputs[:1]
paddle.jit.save(net, model_path, [input_x], output_spec=output_spec) paddle.jit.save(net, model_path, [input_x], output_spec=output_spec)
# 2. load again # 2. load again
...@@ -552,7 +575,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): ...@@ -552,7 +575,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
def test_multi_in_out1(self): def test_multi_in_out1(self):
net = LinearNetMultiInput1(8, 8) net = LinearNetMultiInput1(8, 8)
model_path = "multi_inout1.output_spec1/model" model_path = os.path.join(self.temp_dir.name,
"multi_inout1.output_spec1/model")
# 1. check inputs and outputs # 1. check inputs and outputs
self.assertTrue(len(net.forward.inputs) == 2) self.assertTrue(len(net.forward.inputs) == 2)
input_x = net.forward.inputs[0] input_x = net.forward.inputs[0]
...@@ -574,7 +598,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): ...@@ -574,7 +598,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
pred_x, pred_y = infer_layer(x, y) pred_x, pred_y = infer_layer(x, y)
# 1. prune y and loss # 1. prune y and loss
model_path = "multi_inout1.output_spec2/model" model_path = os.path.join(self.temp_dir.name,
"multi_inout1.output_spec2/model")
output_spec = net.forward.outputs[:1] output_spec = net.forward.outputs[:1]
paddle.jit.save(net, model_path, (input_x, ), output_spec=output_spec) paddle.jit.save(net, model_path, (input_x, ), output_spec=output_spec)
# 2. load again # 2. load again
...@@ -593,6 +618,10 @@ class TestJitSaveLoadConfig(unittest.TestCase): ...@@ -593,6 +618,10 @@ class TestJitSaveLoadConfig(unittest.TestCase):
# config seed # config seed
paddle.seed(SEED) paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED) paddle.framework.random._manual_program_seed(SEED)
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_output_spec(self): def test_output_spec(self):
train_layer = LinearNetReturnLoss(8, 8) train_layer = LinearNetReturnLoss(8, 8)
...@@ -606,7 +635,8 @@ class TestJitSaveLoadConfig(unittest.TestCase): ...@@ -606,7 +635,8 @@ class TestJitSaveLoadConfig(unittest.TestCase):
adam.minimize(loss) adam.minimize(loss)
train_layer.clear_gradients() train_layer.clear_gradients()
model_path = "save_load_config.output_spec" model_path = os.path.join(self.temp_dir.name,
"save_load_config.output_spec")
output_spec = [out] output_spec = [out]
paddle.jit.save( paddle.jit.save(
layer=train_layer, layer=train_layer,
...@@ -623,22 +653,22 @@ class TestJitSaveLoadConfig(unittest.TestCase): ...@@ -623,22 +653,22 @@ class TestJitSaveLoadConfig(unittest.TestCase):
def test_save_no_support_config_error(self): def test_save_no_support_config_error(self):
layer = LinearNet(784, 1) layer = LinearNet(784, 1)
path = "no_support_config_test" path = os.path.join(self.temp_dir.name, "no_support_config_test")
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.jit.save(layer=layer, path=path, model_filename="") paddle.jit.save(layer=layer, path=path, model_filename="")
def test_load_empty_model_filename_error(self): def test_load_empty_model_filename_error(self):
path = "error_model_filename_test" path = os.path.join(self.temp_dir.name, "error_model_filename_test")
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.jit.load(path, model_filename="") paddle.jit.load(path, model_filename="")
def test_load_empty_params_filename_error(self): def test_load_empty_params_filename_error(self):
path = "error_params_filename_test" path = os.path.join(self.temp_dir.name, "error_params_filename_test")
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.jit.load(path, params_filename="") paddle.jit.load(path, params_filename="")
def test_load_with_no_support_config(self): def test_load_with_no_support_config(self):
path = "no_support_config_test" path = os.path.join(self.temp_dir.name, "no_support_config_test")
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.jit.load(path, separate_params=True) paddle.jit.load(path, separate_params=True)
...@@ -646,7 +676,9 @@ class TestJitSaveLoadConfig(unittest.TestCase): ...@@ -646,7 +676,9 @@ class TestJitSaveLoadConfig(unittest.TestCase):
class TestJitMultipleLoading(unittest.TestCase): class TestJitMultipleLoading(unittest.TestCase):
def setUp(self): def setUp(self):
self.linear_size = 4 self.linear_size = 4
self.model_path = "jit_multi_load/model" self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"jit_multi_load/model")
# enable dygraph mode # enable dygraph mode
fluid.enable_dygraph() fluid.enable_dygraph()
# config seed # config seed
...@@ -655,6 +687,9 @@ class TestJitMultipleLoading(unittest.TestCase): ...@@ -655,6 +687,9 @@ class TestJitMultipleLoading(unittest.TestCase):
# train and save base model # train and save base model
self.train_and_save_orig_model() self.train_and_save_orig_model()
def tearDown(self):
self.temp_dir.cleanup()
def train_and_save_orig_model(self): def train_and_save_orig_model(self):
layer = LinearNet(self.linear_size, self.linear_size) layer = LinearNet(self.linear_size, self.linear_size)
example_inputs, layer, _ = train(layer, self.linear_size, 1) example_inputs, layer, _ = train(layer, self.linear_size, 1)
...@@ -674,13 +709,18 @@ class TestJitMultipleLoading(unittest.TestCase): ...@@ -674,13 +709,18 @@ class TestJitMultipleLoading(unittest.TestCase):
class TestJitPruneModelAndLoad(unittest.TestCase): class TestJitPruneModelAndLoad(unittest.TestCase):
def setUp(self): def setUp(self):
self.linear_size = 4 self.linear_size = 4
self.model_path = "jit_prune_model_and_load/model" self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"jit_prune_model_and_load/model")
# enable dygraph mode # enable dygraph mode
fluid.enable_dygraph() fluid.enable_dygraph()
# config seed # config seed
paddle.seed(SEED) paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED) paddle.framework.random._manual_program_seed(SEED)
def tearDown(self):
self.temp_dir.cleanup()
def train_and_save(self): def train_and_save(self):
train_layer = LinearNetReturnHidden(8, 8) train_layer = LinearNetReturnHidden(8, 8)
adam = fluid.optimizer.AdamOptimizer( adam = fluid.optimizer.AdamOptimizer(
...@@ -735,6 +775,10 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -735,6 +775,10 @@ class TestJitSaveMultiCases(unittest.TestCase):
# config seed # config seed
paddle.seed(SEED) paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED) paddle.framework.random._manual_program_seed(SEED)
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def verify_inference_correctness(self, def verify_inference_correctness(self,
layer, layer,
...@@ -767,7 +811,8 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -767,7 +811,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train(layer) train(layer)
model_path = "test_no_prune_to_static_after_train/model" model_path = os.path.join(self.temp_dir.name,
"test_no_prune_to_static_after_train/model")
paddle.jit.save(layer, model_path) paddle.jit.save(layer, model_path)
self.verify_inference_correctness(layer, model_path) self.verify_inference_correctness(layer, model_path)
...@@ -775,7 +820,8 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -775,7 +820,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_no_prune_to_static_no_train(self): def test_no_prune_to_static_no_train(self):
layer = LinearNetWithInputSpec(784, 1) layer = LinearNetWithInputSpec(784, 1)
model_path = "test_no_prune_to_static_no_train/model" model_path = os.path.join(self.temp_dir.name,
"test_no_prune_to_static_no_train/model")
paddle.jit.save(layer, model_path) paddle.jit.save(layer, model_path)
self.verify_inference_correctness(layer, model_path) self.verify_inference_correctness(layer, model_path)
...@@ -785,7 +831,8 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -785,7 +831,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train(layer) train(layer)
model_path = "test_no_prune_no_to_static_after_train/model" model_path = os.path.join(
self.temp_dir.name, "test_no_prune_no_to_static_after_train/model")
paddle.jit.save( paddle.jit.save(
layer, layer,
model_path, model_path,
...@@ -799,7 +846,9 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -799,7 +846,9 @@ class TestJitSaveMultiCases(unittest.TestCase):
example_inputs, _, _ = train(layer) example_inputs, _, _ = train(layer)
model_path = "test_no_prune_no_to_static_after_train_with_examples/model" model_path = os.path.join(
self.temp_dir.name,
"test_no_prune_no_to_static_after_train_with_examples/model")
paddle.jit.save(layer=layer, path=model_path, input_spec=example_inputs) paddle.jit.save(layer=layer, path=model_path, input_spec=example_inputs)
self.verify_inference_correctness(layer, model_path) self.verify_inference_correctness(layer, model_path)
...@@ -807,7 +856,8 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -807,7 +856,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_no_prune_no_to_static_no_train(self): def test_no_prune_no_to_static_no_train(self):
layer = LinearNetNotDeclarative(784, 1) layer = LinearNetNotDeclarative(784, 1)
model_path = "test_no_prune_no_to_static_no_train/model" model_path = os.path.join(self.temp_dir.name,
"test_no_prune_no_to_static_no_train/model")
paddle.jit.save( paddle.jit.save(
layer, layer,
model_path, model_path,
...@@ -821,7 +871,8 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -821,7 +871,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
out = train_with_label(layer) out = train_with_label(layer)
model_path = "test_prune_to_static_after_train/model" model_path = os.path.join(self.temp_dir.name,
"test_prune_to_static_after_train/model")
paddle.jit.save( paddle.jit.save(
layer, layer,
model_path, model_path,
...@@ -837,7 +888,8 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -837,7 +888,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_prune_to_static_no_train(self): def test_prune_to_static_no_train(self):
layer = LinerNetWithLabel(784, 1) layer = LinerNetWithLabel(784, 1)
model_path = "test_prune_to_static_no_train/model" model_path = os.path.join(self.temp_dir.name,
"test_prune_to_static_no_train/model")
# TODO: no train, cannot get output_spec var here # TODO: no train, cannot get output_spec var here
# now only can use index # now only can use index
output_spec = layer.forward.outputs[:1] output_spec = layer.forward.outputs[:1]
...@@ -856,7 +908,8 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -856,7 +908,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_prune_input_to_static_no_train(self): def test_prune_input_to_static_no_train(self):
layer = LinerNetWithPruneInput(784, 1) layer = LinerNetWithPruneInput(784, 1)
model_path = "test_prune_input_to_static_no_train/model" model_path = os.path.join(self.temp_dir.name,
"test_prune_input_to_static_no_train/model")
paddle.jit.save( paddle.jit.save(
layer, layer,
model_path, model_path,
...@@ -870,7 +923,9 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -870,7 +923,9 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_prune_useless_input_to_static_no_train(self): def test_prune_useless_input_to_static_no_train(self):
layer = LinerNetWithUselessInput(784, 1) layer = LinerNetWithUselessInput(784, 1)
model_path = "test_prune_useless_input_to_static_no_train/model" model_path = os.path.join(
self.temp_dir.name,
"test_prune_useless_input_to_static_no_train/model")
paddle.jit.save( paddle.jit.save(
layer, layer,
model_path, model_path,
...@@ -886,7 +941,8 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -886,7 +941,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train(layer) train(layer)
model_path = "test_no_prune_input_spec_name_warning/model" model_path = os.path.join(self.temp_dir.name,
"test_no_prune_input_spec_name_warning/model")
paddle.jit.save( paddle.jit.save(
layer, layer,
model_path, model_path,
...@@ -907,7 +963,8 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -907,7 +963,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train(layer) train(layer)
model_path = "test_not_prune_output_spec_name_warning/model" model_path = os.path.join(
self.temp_dir.name, "test_not_prune_output_spec_name_warning/model")
out = paddle.to_tensor(np.random.random((1, 1)).astype('float')) out = paddle.to_tensor(np.random.random((1, 1)).astype('float'))
paddle.jit.save(layer, model_path, output_spec=[out]) paddle.jit.save(layer, model_path, output_spec=[out])
...@@ -916,7 +973,8 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -916,7 +973,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_prune_input_spec_name_error(self): def test_prune_input_spec_name_error(self):
layer = LinerNetWithLabel(784, 1) layer = LinerNetWithLabel(784, 1)
model_path = "test_prune_input_spec_name_error/model" model_path = os.path.join(self.temp_dir.name,
"test_prune_input_spec_name_error/model")
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.jit.save( paddle.jit.save(
layer, layer,
...@@ -937,7 +995,8 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -937,7 +995,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train_with_label(layer) train_with_label(layer)
model_path = "test_prune_to_static_after_train/model" model_path = os.path.join(self.temp_dir.name,
"test_prune_to_static_after_train/model")
out = paddle.to_tensor(np.random.random((1, 1)).astype('float')) out = paddle.to_tensor(np.random.random((1, 1)).astype('float'))
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.jit.save( paddle.jit.save(
...@@ -952,10 +1011,15 @@ class TestJitSaveMultiCases(unittest.TestCase): ...@@ -952,10 +1011,15 @@ class TestJitSaveMultiCases(unittest.TestCase):
class TestJitSaveLoadEmptyLayer(unittest.TestCase): class TestJitSaveLoadEmptyLayer(unittest.TestCase):
def setUp(self): def setUp(self):
self.model_path = "jit_save_load_empty_layer/model" self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"jit_save_load_empty_layer/model")
# enable dygraph mode # enable dygraph mode
paddle.disable_static() paddle.disable_static()
def tearDown(self):
self.temp_dir.cleanup()
def test_save_load_empty_layer(self): def test_save_load_empty_layer(self):
layer = EmptyLayer() layer = EmptyLayer()
x = paddle.to_tensor(np.random.random((10)).astype('float32')) x = paddle.to_tensor(np.random.random((10)).astype('float32'))
...@@ -968,10 +1032,15 @@ class TestJitSaveLoadEmptyLayer(unittest.TestCase): ...@@ -968,10 +1032,15 @@ class TestJitSaveLoadEmptyLayer(unittest.TestCase):
class TestJitSaveLoadNoParamLayer(unittest.TestCase): class TestJitSaveLoadNoParamLayer(unittest.TestCase):
def setUp(self): def setUp(self):
self.model_path = "jit_save_load_no_param_layer/model" self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"jit_save_load_no_param_layer/model")
# enable dygraph mode # enable dygraph mode
paddle.disable_static() paddle.disable_static()
def tearDown(self):
self.temp_dir.cleanup()
def test_save_load_no_param_layer(self): def test_save_load_no_param_layer(self):
layer = NoParamLayer() layer = NoParamLayer()
x = paddle.to_tensor(np.random.random((5)).astype('float32')) x = paddle.to_tensor(np.random.random((5)).astype('float32'))
...@@ -987,9 +1056,14 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase): ...@@ -987,9 +1056,14 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase):
def setUp(self): def setUp(self):
# enable dygraph mode # enable dygraph mode
paddle.disable_static() paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_inference(self): def test_jit_save_load_inference(self):
model_path_inference = "jit_save_load_multi_methods/model" model_path_inference = os.path.join(self.temp_dir.name,
"jit_save_load_multi_methods/model")
IMAGE_SIZE = 224 IMAGE_SIZE = 224
layer = LinearNetWithMultiStaticFunc(IMAGE_SIZE, 10) layer = LinearNetWithMultiStaticFunc(IMAGE_SIZE, 10)
inps = paddle.randn([1, IMAGE_SIZE]) inps = paddle.randn([1, IMAGE_SIZE])
...@@ -1005,14 +1079,16 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase): ...@@ -1005,14 +1079,16 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase):
)) < 1e-5) )) < 1e-5)
def test_jit_save_load_multi_methods_inputspec(self): def test_jit_save_load_multi_methods_inputspec(self):
model_path = 'jit_save_load_multi_methods/model' model_path = os.path.join(self.temp_dir.name,
'jit_save_load_multi_methods/model')
layer = LinearNetWithMultiStaticFunc(784, 1) layer = LinearNetWithMultiStaticFunc(784, 1)
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.jit.save( paddle.jit.save(
layer, model_path, input_spec=[InputSpec(shape=[None, 784])]) layer, model_path, input_spec=[InputSpec(shape=[None, 784])])
def test_parse_name(self): def test_parse_name(self):
model_path_inference = "jit_save_load_parse_name/model" model_path_inference = os.path.join(self.temp_dir.name,
"jit_save_load_parse_name/model")
IMAGE_SIZE = 224 IMAGE_SIZE = 224
layer = LinearNet(IMAGE_SIZE, 1) layer = LinearNet(IMAGE_SIZE, 1)
inps = paddle.randn([1, IMAGE_SIZE]) inps = paddle.randn([1, IMAGE_SIZE])
...@@ -1082,9 +1158,14 @@ class TestJitSaveLoadSaveWithoutRunning(unittest.TestCase): ...@@ -1082,9 +1158,14 @@ class TestJitSaveLoadSaveWithoutRunning(unittest.TestCase):
def setUp(self): def setUp(self):
# enable dygraph mode # enable dygraph mode
paddle.disable_static() paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_save_load_finetune_load(self): def test_save_load_finetune_load(self):
model_path = "test_jit_save_load_save_without_running/model" model_path = os.path.join(
self.temp_dir.name, "test_jit_save_load_save_without_running/model")
IMAGE_SIZE = 224 IMAGE_SIZE = 224
inps0 = paddle.randn([1, IMAGE_SIZE]) inps0 = paddle.randn([1, IMAGE_SIZE])
inps1 = paddle.randn([2, IMAGE_SIZE]) inps1 = paddle.randn([2, IMAGE_SIZE])
...@@ -1124,9 +1205,14 @@ class TestJitSaveLoadFinetuneLoad(unittest.TestCase): ...@@ -1124,9 +1205,14 @@ class TestJitSaveLoadFinetuneLoad(unittest.TestCase):
def setUp(self): def setUp(self):
# enable dygraph mode # enable dygraph mode
paddle.disable_static() paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_save_load_finetune_load(self): def test_save_load_finetune_load(self):
model_path = "test_jit_save_load_finetune_load/model" model_path = os.path.join(self.temp_dir.name,
"test_jit_save_load_finetune_load/model")
IMAGE_SIZE = 224 IMAGE_SIZE = 224
inps0 = paddle.randn([1, IMAGE_SIZE]) inps0 = paddle.randn([1, IMAGE_SIZE])
inps1 = paddle.randn([2, IMAGE_SIZE]) inps1 = paddle.randn([2, IMAGE_SIZE])
...@@ -1162,13 +1248,18 @@ class TestJitSaveLoadFinetuneLoad(unittest.TestCase): ...@@ -1162,13 +1248,18 @@ class TestJitSaveLoadFinetuneLoad(unittest.TestCase):
class TestJitSaveLoadFunctionCase1(unittest.TestCase): class TestJitSaveLoadFunctionCase1(unittest.TestCase):
def setUp(self): def setUp(self):
paddle.disable_static() paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_static_function(self): def test_jit_save_load_static_function(self):
@paddle.jit.to_static @paddle.jit.to_static
def fun(inputs): def fun(inputs):
return paddle.tanh(inputs) return paddle.tanh(inputs)
path = 'test_jit_save_load_function_1/func' path = os.path.join(self.temp_dir.name,
'test_jit_save_load_function_1/func')
inps = paddle.rand([3, 6]) inps = paddle.rand([3, 6])
origin = fun(inps) origin = fun(inps)
...@@ -1182,6 +1273,10 @@ class TestJitSaveLoadFunctionCase1(unittest.TestCase): ...@@ -1182,6 +1273,10 @@ class TestJitSaveLoadFunctionCase1(unittest.TestCase):
class TestJitSaveLoadFunctionCase2(unittest.TestCase): class TestJitSaveLoadFunctionCase2(unittest.TestCase):
def setUp(self): def setUp(self):
paddle.disable_static() paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function_input_spec(self): def test_jit_save_load_function_input_spec(self):
@paddle.jit.to_static(input_spec=[ @paddle.jit.to_static(input_spec=[
...@@ -1191,7 +1286,8 @@ class TestJitSaveLoadFunctionCase2(unittest.TestCase): ...@@ -1191,7 +1286,8 @@ class TestJitSaveLoadFunctionCase2(unittest.TestCase):
def fun(inputs): def fun(inputs):
return paddle.nn.functional.relu(inputs) return paddle.nn.functional.relu(inputs)
path = 'test_jit_save_load_function_2/func' path = os.path.join(self.temp_dir.name,
'test_jit_save_load_function_2/func')
inps = paddle.rand([3, 6]) inps = paddle.rand([3, 6])
origin = fun(inps) origin = fun(inps)
...@@ -1204,12 +1300,17 @@ class TestJitSaveLoadFunctionCase2(unittest.TestCase): ...@@ -1204,12 +1300,17 @@ class TestJitSaveLoadFunctionCase2(unittest.TestCase):
class TestJitSaveLoadFunctionCase3(unittest.TestCase): class TestJitSaveLoadFunctionCase3(unittest.TestCase):
def setUp(self): def setUp(self):
paddle.disable_static() paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function_function(self): def test_jit_save_load_function_function(self):
def fun(inputs): def fun(inputs):
return paddle.tanh(inputs) return paddle.tanh(inputs)
path = 'test_jit_save_load_function_3/func' path = os.path.join(self.temp_dir.name,
'test_jit_save_load_function_3/func')
inps = paddle.rand([3, 6]) inps = paddle.rand([3, 6])
origin = fun(inps) origin = fun(inps)
...@@ -1229,6 +1330,10 @@ class TestJitSaveLoadFunctionCase3(unittest.TestCase): ...@@ -1229,6 +1330,10 @@ class TestJitSaveLoadFunctionCase3(unittest.TestCase):
class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase): class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase):
def setUp(self): def setUp(self):
paddle.disable_static() paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function(self): def test_jit_save_load_function(self):
class LinearNet(paddle.nn.Layer): class LinearNet(paddle.nn.Layer):
...@@ -1249,7 +1354,9 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase): ...@@ -1249,7 +1354,9 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase):
func = paddle.jit.to_static( func = paddle.jit.to_static(
layer.anothor_forward, [paddle.static.InputSpec(shape=[-1, 5])]) layer.anothor_forward, [paddle.static.InputSpec(shape=[-1, 5])])
path = 'test_jit_save_load_function_with_params_case1/func' path = os.path.join(
self.temp_dir.name,
'test_jit_save_load_function_with_params_case1/func')
paddle.jit.save(func, path) paddle.jit.save(func, path)
load_func = paddle.jit.load(path) load_func = paddle.jit.load(path)
...@@ -1260,6 +1367,10 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase): ...@@ -1260,6 +1367,10 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase):
class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase): class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase):
def setUp(self): def setUp(self):
paddle.disable_static() paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function(self): def test_jit_save_load_function(self):
class LinearNet(paddle.nn.Layer): class LinearNet(paddle.nn.Layer):
...@@ -1278,7 +1389,9 @@ class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase): ...@@ -1278,7 +1389,9 @@ class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase):
inps = paddle.rand([3, 5]) inps = paddle.rand([3, 5])
path = 'test_jit_save_load_function_with_params_case2/func' path = os.path.join(
self.temp_dir.name,
'test_jit_save_load_function_with_params_case2/func')
paddle.jit.save(layer.anothor_forward, path) paddle.jit.save(layer.anothor_forward, path)
origin_result = layer.anothor_forward(inps) origin_result = layer.anothor_forward(inps)
load_func = paddle.jit.load(path) load_func = paddle.jit.load(path)
...@@ -1292,6 +1405,10 @@ class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase): ...@@ -1292,6 +1405,10 @@ class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase):
class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase): class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase):
def setUp(self): def setUp(self):
paddle.disable_static() paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function(self): def test_jit_save_load_function(self):
class LinearNet(paddle.nn.Layer): class LinearNet(paddle.nn.Layer):
...@@ -1311,7 +1428,9 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase): ...@@ -1311,7 +1428,9 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase):
inps = paddle.rand([3, 5]) inps = paddle.rand([3, 5])
origin = layer.anothor_forward(inps) origin = layer.anothor_forward(inps)
path = 'test_jit_save_load_function_with_params_case3/func' path = os.path.join(
self.temp_dir.name,
'test_jit_save_load_function_with_params_case3/func')
paddle.jit.save(layer.anothor_forward, path) paddle.jit.save(layer.anothor_forward, path)
load_func = paddle.jit.load(path) load_func = paddle.jit.load(path)
...@@ -1320,6 +1439,12 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase): ...@@ -1320,6 +1439,12 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase):
class TestJitSaveLoadDataParallel(unittest.TestCase): class TestJitSaveLoadDataParallel(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def verify_inference_correctness(self, layer, path): def verify_inference_correctness(self, layer, path):
layer.eval() layer.eval()
loaded_layer = paddle.jit.load(path) loaded_layer = paddle.jit.load(path)
...@@ -1336,8 +1461,8 @@ class TestJitSaveLoadDataParallel(unittest.TestCase): ...@@ -1336,8 +1461,8 @@ class TestJitSaveLoadDataParallel(unittest.TestCase):
def test_jit_save_data_parallel_with_inputspec(self): def test_jit_save_data_parallel_with_inputspec(self):
layer = LinearNetNotDeclarative(784, 1) layer = LinearNetNotDeclarative(784, 1)
layer = paddle.DataParallel(layer) layer = paddle.DataParallel(layer)
path = os.path.join(self.temp_dir.name,
path = "jit_save_data_parallel_with_inputspec/model" "jit_save_data_parallel_with_inputspec/model")
paddle.jit.save( paddle.jit.save(
layer=layer, path=path, input_spec=[InputSpec(shape=[None, 784])]) layer=layer, path=path, input_spec=[InputSpec(shape=[None, 784])])
...@@ -1347,7 +1472,8 @@ class TestJitSaveLoadDataParallel(unittest.TestCase): ...@@ -1347,7 +1472,8 @@ class TestJitSaveLoadDataParallel(unittest.TestCase):
layer = LinearNetWithInputSpec(784, 1) layer = LinearNetWithInputSpec(784, 1)
layer = paddle.DataParallel(layer) layer = paddle.DataParallel(layer)
path = "jit_save_data_parallel_with_to_static/model" path = os.path.join(self.temp_dir.name,
"jit_save_data_parallel_with_to_static/model")
paddle.jit.save(layer, path) paddle.jit.save(layer, path)
self.verify_inference_correctness(layer, path) self.verify_inference_correctness(layer, path)
...@@ -1368,6 +1494,12 @@ class InputSepcLayer(paddle.nn.Layer): ...@@ -1368,6 +1494,12 @@ class InputSepcLayer(paddle.nn.Layer):
class TestInputSpecCompatibility(unittest.TestCase): class TestInputSpecCompatibility(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def _assert_input_spec_layer_return(self, expect_layer, test_layer): def _assert_input_spec_layer_return(self, expect_layer, test_layer):
input_x = paddle.uniform([8, 8], dtype='float32') input_x = paddle.uniform([8, 8], dtype='float32')
input_y = paddle.uniform([8, 1], dtype='float64') input_y = paddle.uniform([8, 1], dtype='float64')
...@@ -1380,7 +1512,8 @@ class TestInputSpecCompatibility(unittest.TestCase): ...@@ -1380,7 +1512,8 @@ class TestInputSpecCompatibility(unittest.TestCase):
def test_jit_save_compatible_input_sepc(self): def test_jit_save_compatible_input_sepc(self):
layer = InputSepcLayer() layer = InputSepcLayer()
save_dir = "jit_save_compatible_input_spec" save_dir = os.path.join(self.temp_dir.name,
"jit_save_compatible_input_spec")
path = save_dir + "/model" path = save_dir + "/model"
paddle.jit.save(layer=layer, path=path) paddle.jit.save(layer=layer, path=path)
...@@ -1414,7 +1547,8 @@ class TestInputSpecCompatibility(unittest.TestCase): ...@@ -1414,7 +1547,8 @@ class TestInputSpecCompatibility(unittest.TestCase):
def test_jit_save_incompatible_input_sepc(self): def test_jit_save_incompatible_input_sepc(self):
layer = InputSepcLayer() layer = InputSepcLayer()
save_dir = "jit_save_compatible_input_spec" save_dir = os.path.join(self.temp_dir.name,
"jit_save_compatible_input_spec")
path = save_dir + "/model" path = save_dir + "/model"
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
......
python/paddle/fluid/tests/unittests/test_load_op.py
浏览文件 @ 2eb60ddb
...@@ -19,6 +19,8 @@ import numpy as np ...@@ -19,6 +19,8 @@ import numpy as np
from op_test import OpTest, randomize_probability from op_test import OpTest, randomize_probability
import paddle.fluid as fluid import paddle.fluid as fluid
import paddle.fluid.layers as layers import paddle.fluid.layers as layers
import os
import tempfile
class TestLoadOp(unittest.TestCase): class TestLoadOp(unittest.TestCase):
...@@ -26,6 +28,7 @@ class TestLoadOp(unittest.TestCase): ...@@ -26,6 +28,7 @@ class TestLoadOp(unittest.TestCase):
""" """
def setUp(self): def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.ones = np.ones((4, 4)).astype('float32') self.ones = np.ones((4, 4)).astype('float32')
main_prog = fluid.Program() main_prog = fluid.Program()
start_prog = fluid.Program() start_prog = fluid.Program()
...@@ -41,14 +44,20 @@ class TestLoadOp(unittest.TestCase): ...@@ -41,14 +44,20 @@ class TestLoadOp(unittest.TestCase):
exe = fluid.Executor(fluid.CPUPlace()) exe = fluid.Executor(fluid.CPUPlace())
exe.run(start_prog) exe.run(start_prog)
fluid.io.save_persistables( fluid.io.save_persistables(
exe, dirname="./model", main_program=main_prog) exe,
dirname=os.path.join(self.temp_dir.name, "./model"),
main_program=main_prog)
def tearDown(self):
self.temp_dir.cleanup()
def test_load(self): def test_load(self):
main_prog = fluid.Program() main_prog = fluid.Program()
start_prog = fluid.Program() start_prog = fluid.Program()
with fluid.program_guard(main_prog, start_prog): with fluid.program_guard(main_prog, start_prog):
var = layers.create_tensor(dtype='float32') var = layers.create_tensor(dtype='float32')
layers.load(var, file_path='./model/w') layers.load(
var, file_path=os.path.join(self.temp_dir.name, './model/w'))
exe = fluid.Executor(fluid.CPUPlace()) exe = fluid.Executor(fluid.CPUPlace())
exe.run(start_prog) exe.run(start_prog)
......
python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py
浏览文件 @ 2eb60ddb
...@@ -23,6 +23,7 @@ import paddle ...@@ -23,6 +23,7 @@ import paddle
import paddle.fluid as fluid import paddle.fluid as fluid
from paddle.fluid import core from paddle.fluid import core
from test_imperative_base import new_program_scope from test_imperative_base import new_program_scope
import tempfile
def convolutional_neural_network(img): def convolutional_neural_network(img):
...@@ -59,6 +60,7 @@ def static_train_net(img, label): ...@@ -59,6 +60,7 @@ def static_train_net(img, label):
class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
def setUp(self): def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.seed = 90 self.seed = 90
self.epoch_num = 1 self.epoch_num = 1
self.batch_size = 128 self.batch_size = 128
...@@ -66,6 +68,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): ...@@ -66,6 +68,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
# enable static mode # enable static mode
paddle.enable_static() paddle.enable_static()
def tearDown(self):
self.temp_dir.cleanup()
def train_and_save_model(self, only_params=False): def train_and_save_model(self, only_params=False):
with new_program_scope(): with new_program_scope():
startup_program = fluid.default_startup_program() startup_program = fluid.default_startup_program()
...@@ -121,7 +126,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): ...@@ -121,7 +126,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.assertTrue(np.array_equal(value, load_dict[var_name])) self.assertTrue(np.array_equal(value, load_dict[var_name]))
def test_load_default(self): def test_load_default(self):
self.save_dirname = "static_mnist.load_state_dict.default" self.save_dirname = os.path.join(self.temp_dir.name,
"static_mnist.load_state_dict.default")
self.model_filename = None self.model_filename = None
self.params_filename = None self.params_filename = None
orig_param_dict = self.train_and_save_model() orig_param_dict = self.train_and_save_model()
...@@ -133,7 +139,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): ...@@ -133,7 +139,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict) self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_with_model_filename(self): def test_load_with_model_filename(self):
self.save_dirname = "static_mnist.load_state_dict.model_filename" self.save_dirname = os.path.join(
self.temp_dir.name, "static_mnist.load_state_dict.model_filename")
self.model_filename = "static_mnist.model" self.model_filename = "static_mnist.model"
self.params_filename = None self.params_filename = None
orig_param_dict = self.train_and_save_model() orig_param_dict = self.train_and_save_model()
...@@ -147,7 +154,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): ...@@ -147,7 +154,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict) self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_with_param_filename(self): def test_load_with_param_filename(self):
self.save_dirname = "static_mnist.load_state_dict.param_filename" self.save_dirname = os.path.join(
self.temp_dir.name, "static_mnist.load_state_dict.param_filename")
self.model_filename = None self.model_filename = None
self.params_filename = "static_mnist.params" self.params_filename = "static_mnist.params"
orig_param_dict = self.train_and_save_model() orig_param_dict = self.train_and_save_model()
...@@ -161,7 +169,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): ...@@ -161,7 +169,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict) self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_with_model_and_param_filename(self): def test_load_with_model_and_param_filename(self):
self.save_dirname = "static_mnist.load_state_dict.model_and_param_filename" self.save_dirname = os.path.join(
self.temp_dir.name,
"static_mnist.load_state_dict.model_and_param_filename")
self.model_filename = "static_mnist.model" self.model_filename = "static_mnist.model"
self.params_filename = "static_mnist.params" self.params_filename = "static_mnist.params"
orig_param_dict = self.train_and_save_model() orig_param_dict = self.train_and_save_model()
...@@ -179,7 +189,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): ...@@ -179,7 +189,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict) self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_state_dict_from_save_params(self): def test_load_state_dict_from_save_params(self):
self.save_dirname = "static_mnist.load_state_dict.save_params" self.save_dirname = os.path.join(
self.temp_dir.name, "static_mnist.load_state_dict.save_params")
self.params_filename = None self.params_filename = None
orig_param_dict = self.train_and_save_model(True) orig_param_dict = self.train_and_save_model(True)
......
python/paddle/fluid/tests/unittests/test_paddle_save_load.py
浏览文件 @ 2eb60ddb
...@@ -19,6 +19,7 @@ import numpy as np ...@@ -19,6 +19,7 @@ import numpy as np
import os import os
import sys import sys
from io import BytesIO from io import BytesIO
import tempfile
import paddle import paddle
import paddle.nn as nn import paddle.nn as nn
...@@ -90,7 +91,10 @@ def train(layer, loader, loss_fn, opt): ...@@ -90,7 +91,10 @@ def train(layer, loader, loss_fn, opt):
class TestSaveLoadLargeParameters(unittest.TestCase): class TestSaveLoadLargeParameters(unittest.TestCase):
def setUp(self): def setUp(self):
pass self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_large_parameters_paddle_save(self): def test_large_parameters_paddle_save(self):
# enable dygraph mode # enable dygraph mode
...@@ -100,7 +104,8 @@ class TestSaveLoadLargeParameters(unittest.TestCase): ...@@ -100,7 +104,8 @@ class TestSaveLoadLargeParameters(unittest.TestCase):
layer = LayerWithLargeParameters() layer = LayerWithLargeParameters()
save_dict = layer.state_dict() save_dict = layer.state_dict()
path = os.path.join("test_paddle_save_load_large_param_save", path = os.path.join(self.temp_dir.name,
"test_paddle_save_load_large_param_save",
"layer.pdparams") "layer.pdparams")
protocol = 4 protocol = 4
paddle.save(save_dict, path, protocol=protocol) paddle.save(save_dict, path, protocol=protocol)
...@@ -111,6 +116,12 @@ class TestSaveLoadLargeParameters(unittest.TestCase): ...@@ -111,6 +116,12 @@ class TestSaveLoadLargeParameters(unittest.TestCase):
class TestSaveLoadPickle(unittest.TestCase): class TestSaveLoadPickle(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_pickle_protocol(self): def test_pickle_protocol(self):
# enable dygraph mode # enable dygraph mode
paddle.disable_static() paddle.disable_static()
...@@ -118,7 +129,8 @@ class TestSaveLoadPickle(unittest.TestCase): ...@@ -118,7 +129,8 @@ class TestSaveLoadPickle(unittest.TestCase):
layer = LinearNet() layer = LinearNet()
save_dict = layer.state_dict() save_dict = layer.state_dict()
path = os.path.join("test_paddle_save_load_pickle_protocol", path = os.path.join(self.temp_dir.name,
"test_paddle_save_load_pickle_protocol",
"layer.pdparams") "layer.pdparams")
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
...@@ -143,6 +155,12 @@ class TestSaveLoadPickle(unittest.TestCase): ...@@ -143,6 +155,12 @@ class TestSaveLoadPickle(unittest.TestCase):
class TestSaveLoadAny(unittest.TestCase): class TestSaveLoadAny(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_zero(self, prog, place, scope=None): def set_zero(self, prog, place, scope=None):
if scope is None: if scope is None:
scope = fluid.global_scope() scope = fluid.global_scope()
...@@ -203,7 +221,8 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -203,7 +221,8 @@ class TestSaveLoadAny(unittest.TestCase):
t = np.array(fluid.global_scope().find_var(var.name) t = np.array(fluid.global_scope().find_var(var.name)
.get_tensor()) .get_tensor())
base_map[var.name] = t base_map[var.name] = t
path = os.path.join("test_replace_static_save_load", "model") path = os.path.join(self.temp_dir.name,
"test_replace_static_save_load", "model")
# paddle.save, legacy paddle.fluid.load # paddle.save, legacy paddle.fluid.load
self.replace_static_save(prog, path) self.replace_static_save(prog, path)
self.set_zero(prog, place) self.set_zero(prog, place)
...@@ -229,7 +248,9 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -229,7 +248,9 @@ class TestSaveLoadAny(unittest.TestCase):
for var in prog.list_vars(): for var in prog.list_vars():
if var.persistable: if var.persistable:
tensor = var.get_value(fluid.global_scope()) tensor = var.get_value(fluid.global_scope())
paddle.save(tensor, os.path.join(path_vars, var.name)) paddle.save(
tensor,
os.path.join(self.temp_dir.name, path_vars, var.name))
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
var.get_value('fluid.global_scope()') var.get_value('fluid.global_scope()')
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
...@@ -248,7 +269,8 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -248,7 +269,8 @@ class TestSaveLoadAny(unittest.TestCase):
for var in prog.list_vars(): for var in prog.list_vars():
if var.persistable: if var.persistable:
tensor = paddle.load( tensor = paddle.load(
os.path.join(path_vars, var.name), return_numpy=False) os.path.join(self.temp_dir.name, path_vars, var.name),
return_numpy=False)
var.set_value(tensor) var.set_value(tensor)
new_t = np.array(fluid.global_scope().find_var(var.name) new_t = np.array(fluid.global_scope().find_var(var.name)
.get_tensor()) .get_tensor())
...@@ -282,7 +304,8 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -282,7 +304,8 @@ class TestSaveLoadAny(unittest.TestCase):
y.mean().backward() y.mean().backward()
adam.step() adam.step()
state_dict = adam.state_dict() state_dict = adam.state_dict()
path = 'paddle_save_load_v2/model.pdparams' path = os.path.join(self.temp_dir.name,
'paddle_save_load_v2/model.pdparams')
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
paddle.save(state_dict, path, use_binary_format='False') paddle.save(state_dict, path, use_binary_format='False')
# legacy paddle.save, paddle.load # legacy paddle.save, paddle.load
...@@ -306,7 +329,8 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -306,7 +329,8 @@ class TestSaveLoadAny(unittest.TestCase):
# enable dygraph mode # enable dygraph mode
paddle.disable_static() paddle.disable_static()
layer = LinearNet() layer = LinearNet()
path = 'paddle_save_load_v2/var_dygraph' path = os.path.join(self.temp_dir.name,
'paddle_save_load_v2/var_dygraph')
tensor = layer._linear.weight tensor = layer._linear.weight
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.save(tensor, path, pickle_protocol='3') paddle.save(tensor, path, pickle_protocol='3')
...@@ -348,7 +372,8 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -348,7 +372,8 @@ class TestSaveLoadAny(unittest.TestCase):
break break
scope = fluid.global_scope() scope = fluid.global_scope()
origin_tensor = np.array(tensor) origin_tensor = np.array(tensor)
path = 'test_single_pickle_var_static/var' path = os.path.join(self.temp_dir.name,
'test_single_pickle_var_static/var')
paddle.save(tensor, path) paddle.save(tensor, path)
self.set_zero(prog, place, scope) self.set_zero(prog, place, scope)
# static load # static load
...@@ -370,7 +395,8 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -370,7 +395,8 @@ class TestSaveLoadAny(unittest.TestCase):
def test_dygraph_save_static_load(self): def test_dygraph_save_static_load(self):
inps = np.random.randn(1, IMAGE_SIZE).astype('float32') inps = np.random.randn(1, IMAGE_SIZE).astype('float32')
path = 'test_dygraph_save_static_load/dy-static.pdparams' path = os.path.join(self.temp_dir.name,
'test_dygraph_save_static_load/dy-static.pdparams')
paddle.disable_static() paddle.disable_static()
with paddle.utils.unique_name.guard(): with paddle.utils.unique_name.guard():
layer = LinearNet() layer = LinearNet()
...@@ -414,10 +440,14 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -414,10 +440,14 @@ class TestSaveLoadAny(unittest.TestCase):
}) })
obj4 = (np.random.randn(5, 6), (123, )) obj4 = (np.random.randn(5, 6), (123, ))
path1 = "test_save_load_any_complex_object_dygraph/obj1" path1 = os.path.join(self.temp_dir.name,
path2 = "test_save_load_any_complex_object_dygraph/obj2" "test_save_load_any_complex_object_dygraph/obj1")
path3 = "test_save_load_any_complex_object_dygraph/obj3" path2 = os.path.join(self.temp_dir.name,
path4 = "test_save_load_any_complex_object_dygraph/obj4" "test_save_load_any_complex_object_dygraph/obj2")
path3 = os.path.join(self.temp_dir.name,
"test_save_load_any_complex_object_dygraph/obj3")
path4 = os.path.join(self.temp_dir.name,
"test_save_load_any_complex_object_dygraph/obj4")
paddle.save(obj1, path1) paddle.save(obj1, path1)
paddle.save(obj2, path2) paddle.save(obj2, path2)
paddle.save(obj3, path3) paddle.save(obj3, path3)
...@@ -584,10 +614,18 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -584,10 +614,18 @@ class TestSaveLoadAny(unittest.TestCase):
}) })
obj4 = (np.ndarray([3, 4], dtype="float32"), ) obj4 = (np.ndarray([3, 4], dtype="float32"), )
path1 = "test_save_load_any_complex_object_static/obj1" path1 = os.path.join(
path2 = "test_save_load_any_complex_object_static/obj2" self.temp_dir.name,
path3 = "test_save_load_any_complex_object_static/obj3" "test_save_load_any_complex_object_static/obj1")
path4 = "test_save_load_any_complex_object_static/obj4" path2 = os.path.join(
self.temp_dir.name,
"test_save_load_any_complex_object_static/obj2")
path3 = os.path.join(
self.temp_dir.name,
"test_save_load_any_complex_object_static/obj3")
path4 = os.path.join(
self.temp_dir.name,
"test_save_load_any_complex_object_static/obj4")
paddle.save(obj1, path1) paddle.save(obj1, path1)
paddle.save(obj2, path2) paddle.save(obj2, path2)
paddle.save(obj3, path3) paddle.save(obj3, path3)
...@@ -751,7 +789,8 @@ class TestSaveLoadAny(unittest.TestCase): ...@@ -751,7 +789,8 @@ class TestSaveLoadAny(unittest.TestCase):
def test_varbase_binary_var(self): def test_varbase_binary_var(self):
paddle.disable_static() paddle.disable_static()
varbase = paddle.randn([3, 2], dtype='float32') varbase = paddle.randn([3, 2], dtype='float32')
path = 'test_paddle_save_load_varbase_binary_var/varbase' path = os.path.join(self.temp_dir.name,
'test_paddle_save_load_varbase_binary_var/varbase')
paddle.save(varbase, path, use_binary_format=True) paddle.save(varbase, path, use_binary_format=True)
load_array = paddle.load(path, return_numpy=True) load_array = paddle.load(path, return_numpy=True)
load_tensor = paddle.load(path, return_numpy=False) load_tensor = paddle.load(path, return_numpy=False)
...@@ -836,6 +875,10 @@ class TestSaveLoad(unittest.TestCase): ...@@ -836,6 +875,10 @@ class TestSaveLoad(unittest.TestCase):
# config seed # config seed
paddle.seed(SEED) paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED) paddle.framework.random._manual_program_seed(SEED)
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def build_and_train_model(self): def build_and_train_model(self):
# create network # create network
...@@ -863,8 +906,10 @@ class TestSaveLoad(unittest.TestCase): ...@@ -863,8 +906,10 @@ class TestSaveLoad(unittest.TestCase):
layer, opt = self.build_and_train_model() layer, opt = self.build_and_train_model()
# save # save
layer_save_path = "test_paddle_save_load.linear.pdparams" layer_save_path = os.path.join(self.temp_dir.name,
opt_save_path = "test_paddle_save_load.linear.pdopt" "test_paddle_save_load.linear.pdparams")
opt_save_path = os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear.pdopt")
layer_state_dict = layer.state_dict() layer_state_dict = layer.state_dict()
opt_state_dict = opt.state_dict() opt_state_dict = opt.state_dict()
...@@ -880,7 +925,9 @@ class TestSaveLoad(unittest.TestCase): ...@@ -880,7 +925,9 @@ class TestSaveLoad(unittest.TestCase):
# test save load in static mode # test save load in static mode
paddle.enable_static() paddle.enable_static()
static_save_path = "static_mode_test/test_paddle_save_load.linear.pdparams" static_save_path = os.path.join(
self.temp_dir.name,
"static_mode_test/test_paddle_save_load.linear.pdparams")
paddle.save(layer_state_dict, static_save_path) paddle.save(layer_state_dict, static_save_path)
load_static_state_dict = paddle.load(static_save_path) load_static_state_dict = paddle.load(static_save_path)
self.check_load_state_dict(layer_state_dict, load_static_state_dict) self.check_load_state_dict(layer_state_dict, load_static_state_dict)
...@@ -891,20 +938,28 @@ class TestSaveLoad(unittest.TestCase): ...@@ -891,20 +938,28 @@ class TestSaveLoad(unittest.TestCase):
# 2. test save path format error # 2. test save path format error
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.save(layer_state_dict, "test_paddle_save_load.linear.model/") paddle.save(layer_state_dict,
os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear.model/"))
# 3. test load path not exist error # 3. test load path not exist error
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.load("test_paddle_save_load.linear.params") paddle.load(
os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear.params"))
# 4. test load old save path error # 4. test load old save path error
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.load("test_paddle_save_load.linear") paddle.load(
os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear"))
class TestSaveLoadProgram(unittest.TestCase): class TestSaveLoadProgram(unittest.TestCase):
def test_save_load_program(self): def test_save_load_program(self):
paddle.enable_static() paddle.enable_static()
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope(): with new_program_scope():
layer = LinearNet() layer = LinearNet()
data = paddle.static.data( data = paddle.static.data(
...@@ -914,8 +969,12 @@ class TestSaveLoadProgram(unittest.TestCase): ...@@ -914,8 +969,12 @@ class TestSaveLoadProgram(unittest.TestCase):
startup_program = paddle.static.default_startup_program() startup_program = paddle.static.default_startup_program()
origin_main = main_program.desc.serialize_to_string() origin_main = main_program.desc.serialize_to_string()
origin_startup = startup_program.desc.serialize_to_string() origin_startup = startup_program.desc.serialize_to_string()
path1 = "test_paddle_save_load_program/main_program.pdmodel" path1 = os.path.join(
path2 = "test_paddle_save_load_program/startup_program.pdmodel" temp_dir.name,
"test_paddle_save_load_program/main_program.pdmodel")
path2 = os.path.join(
temp_dir.name,
"test_paddle_save_load_program/startup_program.pdmodel")
paddle.save(main_program, path1) paddle.save(main_program, path1)
paddle.save(startup_program, path2) paddle.save(startup_program, path2)
...@@ -924,11 +983,13 @@ class TestSaveLoadProgram(unittest.TestCase): ...@@ -924,11 +983,13 @@ class TestSaveLoadProgram(unittest.TestCase):
load_startup = paddle.load(path2).desc.serialize_to_string() load_startup = paddle.load(path2).desc.serialize_to_string()
self.assertTrue(origin_main == load_main) self.assertTrue(origin_main == load_main)
self.assertTrue(origin_startup == load_startup) self.assertTrue(origin_startup == load_startup)
temp_dir.cleanup()
class TestSaveLoadLayer(unittest.TestCase): class TestSaveLoadLayer(unittest.TestCase):
def test_save_load_layer(self): def test_save_load_layer(self):
paddle.disable_static() paddle.disable_static()
temp_dir = tempfile.TemporaryDirectory()
inps = paddle.randn([1, IMAGE_SIZE], dtype='float32') inps = paddle.randn([1, IMAGE_SIZE], dtype='float32')
layer1 = LinearNet() layer1 = LinearNet()
layer2 = LinearNet() layer2 = LinearNet()
...@@ -936,9 +997,11 @@ class TestSaveLoadLayer(unittest.TestCase): ...@@ -936,9 +997,11 @@ class TestSaveLoadLayer(unittest.TestCase):
layer2.eval() layer2.eval()
origin_layer = (layer1, layer2) origin_layer = (layer1, layer2)
origin = (layer1(inps), layer2(inps)) origin = (layer1(inps), layer2(inps))
path = "test_save_load_layer_/layer.pdmodel" path = os.path.join(temp_dir.name,
"test_save_load_layer_/layer.pdmodel")
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.save(origin_layer, path) paddle.save(origin_layer, path)
temp_dir.cleanup()
if __name__ == '__main__': if __name__ == '__main__':
......
python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py
浏览文件 @ 2eb60ddb
...@@ -21,6 +21,7 @@ import os ...@@ -21,6 +21,7 @@ import os
import sys import sys
import six import six
import platform import platform
import tempfile
import paddle import paddle
import paddle.nn as nn import paddle.nn as nn
...@@ -37,6 +38,10 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -37,6 +38,10 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
def setUp(self): def setUp(self):
# enable static graph mode # enable static graph mode
paddle.enable_static() paddle.enable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_zero(self, prog, place, scope=None): def set_zero(self, prog, place, scope=None):
if scope is None: if scope is None:
...@@ -94,7 +99,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -94,7 +99,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
# test for replace_save_vars/io.load_vars # test for replace_save_vars/io.load_vars
path_vars1 = 'test_replace_save_load_vars_binary1/model' path_vars1 = os.path.join(
self.temp_dir.name, 'test_replace_save_load_vars_binary1/model')
self.replace_save_vars(prog, path_vars1) self.replace_save_vars(prog, path_vars1)
# set var to zero # set var to zero
self.set_zero(prog, place) self.set_zero(prog, place)
...@@ -111,7 +117,9 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -111,7 +117,9 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
# test for io.save_vars/replace_load_vars # test for io.save_vars/replace_load_vars
path_vars2 = 'test_replace_save_load_vars_binary2/model/' path_vars2 = os.path.join(
self.temp_dir.name,
'test_replace_save_load_vars_binary2/model/')
fluid.io.save_vars( fluid.io.save_vars(
exe, path_vars2, main_program=prog, vars=var_list) exe, path_vars2, main_program=prog, vars=var_list)
self.set_zero(prog, place) self.set_zero(prog, place)
...@@ -141,7 +149,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -141,7 +149,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
prog = paddle.static.default_main_program() prog = paddle.static.default_main_program()
exe.run(fluid.default_startup_program()) exe.run(fluid.default_startup_program())
dirname = 'test_save_load_lod_tensor1/tensor_' dirname = os.path.join(self.temp_dir.name,
'test_save_load_lod_tensor1/tensor_')
for var in prog.list_vars(): for var in prog.list_vars():
if var.persistable and list( if var.persistable and list(
var.shape) == [IMAGE_SIZE, OUTPUT_NUM]: var.shape) == [IMAGE_SIZE, OUTPUT_NUM]:
...@@ -162,12 +171,13 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -162,12 +171,13 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self.assertTrue(np.array_equal(origin, to_array)) self.assertTrue(np.array_equal(origin, to_array))
with self.assertRaises(NotImplementedError): with self.assertRaises(NotImplementedError):
path = 'test_save_load_error/temp' path = os.path.join(self.temp_dir.name, 'test_save_load_error/temp')
paddle.save({}, path, use_binary_format=True) paddle.save({}, path, use_binary_format=True)
# On the Windows platform, when parsing a string that can't be parsed as a `Program`, `desc_.ParseFromString` has a timeout risk. # On the Windows platform, when parsing a string that can't be parsed as a `Program`, `desc_.ParseFromString` has a timeout risk.
if 'Windows' != platform.system(): if 'Windows' != platform.system():
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
path = 'test_save_load_error/temp' path = os.path.join(self.temp_dir.name,
'test_save_load_error/temp')
with open(path, "w") as f: with open(path, "w") as f:
f.write('\0') f.write('\0')
paddle.load(path) paddle.load(path)
...@@ -178,11 +188,17 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -178,11 +188,17 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError):
fluid.core.save_lod_tensor( fluid.core.save_lod_tensor(
temp_lod, 'test_save_load_error_not_exist_file/not_exist_file') temp_lod,
os.path.join(
self.temp_dir.name,
'test_save_load_error_not_exist_file/not_exist_file'))
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError):
fluid.core.load_lod_tensor( fluid.core.load_lod_tensor(
temp_lod, 'test_save_load_error_not_exist_file/not_exist_file') temp_lod,
os.path.join(
self.temp_dir.name,
'test_save_load_error_not_exist_file/not_exist_file'))
# save to memory # save to memory
byio = BytesIO() byio = BytesIO()
...@@ -206,7 +222,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -206,7 +222,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
rows = [0, 4, 7] rows = [0, 4, 7]
row_numel = 12 row_numel = 12
selected_rows = fluid.core.SelectedRows(rows, height) selected_rows = fluid.core.SelectedRows(rows, height)
path = 'test_paddle_save_load_selected_rows/sr.pdsr' path = os.path.join(self.temp_dir.name,
'test_paddle_save_load_selected_rows/sr.pdsr')
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
paddle.save(selected_rows, path, use_binary_format=True) paddle.save(selected_rows, path, use_binary_format=True)
...@@ -227,11 +244,15 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): ...@@ -227,11 +244,15 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError):
fluid.core.save_selected_rows( fluid.core.save_selected_rows(
selected_rows, selected_rows,
'test_paddle_save_load_selected_rows_not_exist_file/temp') os.path.join(
self.temp_dir.name,
'test_paddle_save_load_selected_rows_not_exist_file/temp'))
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError):
fluid.core.load_selected_rows( fluid.core.load_selected_rows(
selected_rows, selected_rows,
'test_paddle_save_load_selected_rows_not_exist_file/temp') os.path.join(
self.temp_dir.name,
'test_paddle_save_load_selected_rows_not_exist_file/temp'))
# save to memory # save to memory
byio = BytesIO() byio = BytesIO()
......
python/paddle/fluid/tests/unittests/test_static_save_load.py
浏览文件 @ 2eb60ddb
...@@ -30,6 +30,7 @@ import six ...@@ -30,6 +30,7 @@ import six
import pickle import pickle
import os import os
import errno import errno
import tempfile
paddle.enable_static() paddle.enable_static()
...@@ -230,6 +231,7 @@ class TestSaveLoadBase(unittest.TestCase): ...@@ -230,6 +231,7 @@ class TestSaveLoadBase(unittest.TestCase):
init_scale = 0.1 init_scale = 0.1
batch_size = 4 batch_size = 4
batch_num = 200 batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope(): with new_program_scope():
fluid.default_startup_program().random_seed = seed fluid.default_startup_program().random_seed = seed
...@@ -297,7 +299,7 @@ class TestSaveLoadBase(unittest.TestCase): ...@@ -297,7 +299,7 @@ class TestSaveLoadBase(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
fluid.save(main_program, "./test_1") fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -310,7 +312,8 @@ class TestSaveLoadBase(unittest.TestCase): ...@@ -310,7 +312,8 @@ class TestSaveLoadBase(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero # make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "./test_1.pdparams", exe) fluid.load(main_program,
os.path.join(temp_dir.name, "test_1.pdparams"), exe)
for var in main_program.list_vars(): for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable: if isinstance(var, framework.Parameter) or var.persistable:
...@@ -318,6 +321,7 @@ class TestSaveLoadBase(unittest.TestCase): ...@@ -318,6 +321,7 @@ class TestSaveLoadBase(unittest.TestCase):
.get_tensor()) .get_tensor())
base_t = base_map[var.name] base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestSaveLoadPartial(unittest.TestCase): class TestSaveLoadPartial(unittest.TestCase):
...@@ -334,6 +338,7 @@ class TestSaveLoadPartial(unittest.TestCase): ...@@ -334,6 +338,7 @@ class TestSaveLoadPartial(unittest.TestCase):
init_scale = 0.1 init_scale = 0.1
batch_size = 4 batch_size = 4
batch_num = 200 batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope(): with new_program_scope():
fluid.default_startup_program().random_seed = seed fluid.default_startup_program().random_seed = seed
...@@ -409,7 +414,7 @@ class TestSaveLoadPartial(unittest.TestCase): ...@@ -409,7 +414,7 @@ class TestSaveLoadPartial(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
fluid.save(main_program, "./test_1") fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -422,7 +427,8 @@ class TestSaveLoadPartial(unittest.TestCase): ...@@ -422,7 +427,8 @@ class TestSaveLoadPartial(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero # make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(test_program, "./test_1.pdopt", None) fluid.load(test_program,
os.path.join(temp_dir.name, "test_1.pdopt"), None)
for var in test_program.list_vars(): for var in test_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable: if isinstance(var, framework.Parameter) or var.persistable:
...@@ -430,7 +436,9 @@ class TestSaveLoadPartial(unittest.TestCase): ...@@ -430,7 +436,9 @@ class TestSaveLoadPartial(unittest.TestCase):
.get_tensor()) .get_tensor())
base_t = base_map[var.name] base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
fluid.load(test_program, "./test_1.pdmodel", None) fluid.load(test_program,
os.path.join(temp_dir.name, "test_1.pdmodel"), None)
temp_dir.cleanup()
class TestSaveLoadSetStateDict(unittest.TestCase): class TestSaveLoadSetStateDict(unittest.TestCase):
...@@ -447,6 +455,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): ...@@ -447,6 +455,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
init_scale = 0.1 init_scale = 0.1
batch_size = 4 batch_size = 4
batch_num = 200 batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope(): with new_program_scope():
fluid.default_startup_program().random_seed = seed fluid.default_startup_program().random_seed = seed
...@@ -514,7 +523,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): ...@@ -514,7 +523,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
fluid.save(main_program, "./test_1") fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -527,7 +536,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): ...@@ -527,7 +536,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero # make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "./test_1", exe) fluid.load(main_program, os.path.join(temp_dir.name, "test_1"), exe)
for var in main_program.list_vars(): for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable: if isinstance(var, framework.Parameter) or var.persistable:
...@@ -535,6 +544,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase): ...@@ -535,6 +544,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
.get_tensor()) .get_tensor())
base_t = base_map[var.name] base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestProgramStatePartial(unittest.TestCase): class TestProgramStatePartial(unittest.TestCase):
...@@ -551,6 +561,7 @@ class TestProgramStatePartial(unittest.TestCase): ...@@ -551,6 +561,7 @@ class TestProgramStatePartial(unittest.TestCase):
init_scale = 0.1 init_scale = 0.1
batch_size = 4 batch_size = 4
batch_num = 200 batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope(): with new_program_scope():
fluid.default_startup_program().random_seed = seed fluid.default_startup_program().random_seed = seed
...@@ -626,7 +637,7 @@ class TestProgramStatePartial(unittest.TestCase): ...@@ -626,7 +637,7 @@ class TestProgramStatePartial(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
fluid.save(main_program, os.path.join('some_dir', 'test_1')) fluid.save(main_program, os.path.join(temp_dir.name, 'test_1'))
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -641,16 +652,16 @@ class TestProgramStatePartial(unittest.TestCase): ...@@ -641,16 +652,16 @@ class TestProgramStatePartial(unittest.TestCase):
#fluid.load(test_program, "./test_1", None ) #fluid.load(test_program, "./test_1", None )
program_state = fluid.load_program_state( program_state = fluid.load_program_state(
os.path.join('some_dir', 'test_1')) os.path.join(temp_dir.name, 'test_1'))
program_state_1 = fluid.load_program_state( program_state_1 = fluid.load_program_state(
os.path.join('some_dir', 'test_1.pdparams')) os.path.join(temp_dir.name, 'test_1.pdparams'))
program_state_2 = fluid.load_program_state( program_state_2 = fluid.load_program_state(
os.path.join('some_dir', 'test_1.pdopt')) os.path.join(temp_dir.name, 'test_1.pdopt'))
program_state_3 = fluid.load_program_state( program_state_3 = fluid.load_program_state(
os.path.join('some_dir', 'test_1.pdmodel')) os.path.join(temp_dir.name, 'test_1.pdmodel'))
fluid.set_program_state(test_program, program_state) fluid.set_program_state(test_program, program_state)
...@@ -720,6 +731,7 @@ class TestProgramStatePartial(unittest.TestCase): ...@@ -720,6 +731,7 @@ class TestProgramStatePartial(unittest.TestCase):
.get_tensor()) .get_tensor())
base_t = base_map[var.name] base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestVariableInit(unittest.TestCase): class TestVariableInit(unittest.TestCase):
...@@ -737,7 +749,9 @@ class TestVariableInit(unittest.TestCase): ...@@ -737,7 +749,9 @@ class TestVariableInit(unittest.TestCase):
exe = fluid.Executor(place) exe = fluid.Executor(place)
exe.run(fluid.default_startup_program()) exe.run(fluid.default_startup_program())
fluid.save(fluid.default_main_program(), "./test_path") temp_dir = tempfile.TemporaryDirectory()
fluid.save(fluid.default_main_program(),
os.path.join(temp_dir.name, "test_path"))
def set_var(var, ndarray): def set_var(var, ndarray):
t = var.get_tensor() t = var.get_tensor()
...@@ -763,7 +777,7 @@ class TestVariableInit(unittest.TestCase): ...@@ -763,7 +777,7 @@ class TestVariableInit(unittest.TestCase):
fluid.core._create_loaded_parameter(parameter_list, new_scope, fluid.core._create_loaded_parameter(parameter_list, new_scope,
exe._default_executor) exe._default_executor)
parameter_file_name = "./test_path.pdparams" parameter_file_name = os.path.join(temp_dir.name, "test_path.pdparams")
with open(parameter_file_name, 'rb') as f: with open(parameter_file_name, 'rb') as f:
load_dict = pickle.load(f) load_dict = pickle.load(f)
...@@ -779,7 +793,7 @@ class TestVariableInit(unittest.TestCase): ...@@ -779,7 +793,7 @@ class TestVariableInit(unittest.TestCase):
fluid.core._create_loaded_parameter(opt_list, new_scope, fluid.core._create_loaded_parameter(opt_list, new_scope,
exe._default_executor) exe._default_executor)
opt_file_name = "./test_path.pdopt" opt_file_name = os.path.join(temp_dir.name, "test_path.pdopt")
with open(opt_file_name, 'rb') as f: with open(opt_file_name, 'rb') as f:
load_dict = pickle.load(f) load_dict = pickle.load(f)
...@@ -805,6 +819,7 @@ class TestVariableInit(unittest.TestCase): ...@@ -805,6 +819,7 @@ class TestVariableInit(unittest.TestCase):
base_t = base_map[var.name] base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestLoadFromOldInterface(unittest.TestCase): class TestLoadFromOldInterface(unittest.TestCase):
...@@ -815,10 +830,15 @@ class TestLoadFromOldInterface(unittest.TestCase): ...@@ -815,10 +830,15 @@ class TestLoadFromOldInterface(unittest.TestCase):
if os.path.exists("test_static_load_var_list.pdparams"): if os.path.exists("test_static_load_var_list.pdparams"):
os.remove("test_static_load_var_list.pdparams") os.remove("test_static_load_var_list.pdparams")
self.temp_dir = tempfile.TemporaryDirectory()
def set_place(self): def set_place(self):
return fluid.CPUPlace() if not core.is_compiled_with_cuda( return fluid.CPUPlace() if not core.is_compiled_with_cuda(
) else fluid.CUDAPlace(0) ) else fluid.CUDAPlace(0)
def tearDown(self):
self.temp_dir.cleanup()
def test_load_from_old_interface(self): def test_load_from_old_interface(self):
seed = 90 seed = 90
hidden_size = 10 hidden_size = 10
...@@ -898,7 +918,9 @@ class TestLoadFromOldInterface(unittest.TestCase): ...@@ -898,7 +918,9 @@ class TestLoadFromOldInterface(unittest.TestCase):
base_map[var.name] = t base_map[var.name] = t
#fluid.save(main_program, "./test_1") #fluid.save(main_program, "./test_1")
fluid.io.save_persistables(exe, "test_path", main_program) fluid.io.save_persistables(
exe,
os.path.join(self.temp_dir.name, "test_path"), main_program)
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -911,7 +933,8 @@ class TestLoadFromOldInterface(unittest.TestCase): ...@@ -911,7 +933,8 @@ class TestLoadFromOldInterface(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero # make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "test_path", exe) fluid.load(main_program,
os.path.join(self.temp_dir.name, "test_path"), exe)
for var in main_program.list_vars(): for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable: if isinstance(var, framework.Parameter) or var.persistable:
...@@ -928,11 +951,13 @@ class TestLoadFromOldInterface(unittest.TestCase): ...@@ -928,11 +951,13 @@ class TestLoadFromOldInterface(unittest.TestCase):
var.desc.set_shape(new_shape) var.desc.set_shape(new_shape)
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError):
fluid.load(main_program, "test_path", exe) fluid.load(main_program,
os.path.join(self.temp_dir.name, "test_path"), exe)
# check unused parameter # check unused parameter
fluid.load(test_clone_program, "test_path", exe) fluid.load(test_clone_program,
os.path.join(self.temp_dir.name, "test_path"), exe)
def test_load_from_old_interface_var_list(self): def test_load_from_old_interface_var_list(self):
seed = 90 seed = 90
...@@ -1013,7 +1038,9 @@ class TestLoadFromOldInterface(unittest.TestCase): ...@@ -1013,7 +1038,9 @@ class TestLoadFromOldInterface(unittest.TestCase):
base_map[var.name] = t base_map[var.name] = t
#fluid.save(main_program, "./test_1") #fluid.save(main_program, "./test_1")
fluid.io.save_persistables(exe, "test_static_load_var_list", fluid.io.save_persistables(
exe,
os.path.join(self.temp_dir.name, "test_static_load_var_list"),
main_program) main_program)
# set var to zero # set var to zero
...@@ -1030,7 +1057,10 @@ class TestLoadFromOldInterface(unittest.TestCase): ...@@ -1030,7 +1057,10 @@ class TestLoadFromOldInterface(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero # make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "test_static_load_var_list", exe, var_list) fluid.load(
main_program,
os.path.join(self.temp_dir.name, "test_static_load_var_list"),
exe, var_list)
var_list_names = [var.name for var in var_list] var_list_names = [var.name for var in var_list]
for var in main_program.list_vars(): for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable: if isinstance(var, framework.Parameter) or var.persistable:
...@@ -1059,6 +1089,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): ...@@ -1059,6 +1089,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
init_scale = 0.1 init_scale = 0.1
batch_size = 4 batch_size = 4
batch_num = 200 batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope(): with new_program_scope():
fluid.default_startup_program().random_seed = seed fluid.default_startup_program().random_seed = seed
...@@ -1125,10 +1156,10 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): ...@@ -1125,10 +1156,10 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update # make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
save_dir = os.path.join(temp_dir.name, "test_path")
#fluid.save(main_program, "./test_1") #fluid.save(main_program, "./test_1")
fluid.io.save_persistables( fluid.io.save_persistables(
exe, "test_path", main_program, filename="model_single") exe, save_dir, main_program, filename="model_single")
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -1141,7 +1172,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): ...@@ -1141,7 +1172,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero # make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
file_model_path = os.path.join("test_path", "model_single") file_model_path = os.path.join(save_dir, "model_single")
fluid.load(main_program, file_model_path, exe, fluid.load(main_program, file_model_path, exe,
fluid.io.get_program_persistable_vars(main_program)) fluid.io.get_program_persistable_vars(main_program))
...@@ -1190,11 +1221,16 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): ...@@ -1190,11 +1221,16 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
all_var_list = list(main_program.list_vars()) all_var_list = list(main_program.list_vars())
fluid.load(main_program, file_model_path, exe, fluid.load(main_program, file_model_path, exe,
all_var_list + [temp_var]) all_var_list + [temp_var])
temp_dir.cleanup()
class TestProgramStateOldSave(unittest.TestCase): class TestProgramStateOldSave(unittest.TestCase):
def setUp(self): def setUp(self):
self.test_dygraph = True self.test_dygraph = True
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_place(self): def set_place(self):
return fluid.CPUPlace() if not core.is_compiled_with_cuda( return fluid.CPUPlace() if not core.is_compiled_with_cuda(
...@@ -1283,8 +1319,8 @@ class TestProgramStateOldSave(unittest.TestCase): ...@@ -1283,8 +1319,8 @@ class TestProgramStateOldSave(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update # make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
save_dir = os.path.join(self.temp_dir.name, "test_program_1")
fluid.io.save_persistables(exe, "test_program_1", main_program) fluid.io.save_persistables(exe, save_dir, main_program)
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -1298,7 +1334,7 @@ class TestProgramStateOldSave(unittest.TestCase): ...@@ -1298,7 +1334,7 @@ class TestProgramStateOldSave(unittest.TestCase):
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
# case 1: load basic # case 1: load basic
program_state = fluid.load_program_state("test_program_1") program_state = fluid.load_program_state(save_dir)
fluid.set_program_state(main_program, program_state) fluid.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map) self.check_in_static(main_program, base_map)
...@@ -1313,24 +1349,20 @@ class TestProgramStateOldSave(unittest.TestCase): ...@@ -1313,24 +1349,20 @@ class TestProgramStateOldSave(unittest.TestCase):
else: else:
raise e raise e
orig_filepath = './test_program_1/fc_0.w_0' program_state = fluid.load_program_state(save_dir)
symlink_filepath = './test_program_1/link_fc_0.w_0'
# create a needless link file for coverage
symlink_force(orig_filepath, symlink_filepath)
program_state = fluid.load_program_state("test_program_1")
fluid.set_program_state(main_program, program_state) fluid.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map) self.check_in_static(main_program, base_map)
# case 3: load with var_list # case 3: load with var_list
program_state = fluid.load_program_state( program_state = fluid.load_program_state(
"test_program_1", main_program.all_parameters()) save_dir, main_program.all_parameters())
fluid.set_program_state(main_program, program_state) fluid.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map) self.check_in_static(main_program, base_map)
if self.test_dygraph: if self.test_dygraph:
# make sure `load_program_state` can be used in dynamic graph mode # make sure `load_program_state` can be used in dynamic graph mode
with fluid.dygraph.guard(place): with fluid.dygraph.guard(place):
load_state = fluid.load_program_state("test_program_1") load_state = fluid.load_program_state(save_dir)
for k, v in load_state.items(): for k, v in load_state.items():
self.assertTrue(np.array_equal(base_map[k], v)) self.assertTrue(np.array_equal(base_map[k], v))
...@@ -1365,6 +1397,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): ...@@ -1365,6 +1397,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
init_scale = 0.1 init_scale = 0.1
batch_size = 4 batch_size = 4
batch_num = 200 batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope(): with new_program_scope():
fluid.default_startup_program().random_seed = seed fluid.default_startup_program().random_seed = seed
...@@ -1440,8 +1473,9 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): ...@@ -1440,8 +1473,9 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
save_dir = os.path.join(temp_dir.name, "test_program_2")
fluid.io.save_persistables( fluid.io.save_persistables(
exe, "test_program_2", main_program, filename="model_1") exe, save_dir, main_program, filename="model_1")
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -1456,7 +1490,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): ...@@ -1456,7 +1490,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
#fluid.load(test_program, "./test_1", None ) #fluid.load(test_program, "./test_1", None )
program_state = fluid.load_program_state( program_state = fluid.load_program_state(
os.path.join("test_program_2", "model_1"), os.path.join(save_dir, "model_1"),
var_list=fluid.io.get_program_persistable_vars(main_program)) var_list=fluid.io.get_program_persistable_vars(main_program))
fluid.set_program_state(main_program, program_state) fluid.set_program_state(main_program, program_state)
...@@ -1468,21 +1502,20 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): ...@@ -1468,21 +1502,20 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
fluid.load_program_state( fluid.load_program_state(os.path.join(save_dir, "model_1"))
os.path.join("test_program_2", "model_1"))
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
fluid.load_program_state( fluid.load_program_state(
os.path.join("test_program_2", "model_1"), os.path.join(save_dir, "model_1"), var_list=["str"])
var_list=["str"])
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError):
fluid.load_program_state( fluid.load_program_state(
os.path.join("test_program_2", "model_1"), os.path.join(save_dir, "model_1"),
var_list=[ var_list=[
main_program.global_block().create_var( main_program.global_block().create_var(
name="fake_var_name", persistable=True) name="fake_var_name", persistable=True)
]) ])
temp_dir.cleanup()
class TestStaticSaveLoadPickle(unittest.TestCase): class TestStaticSaveLoadPickle(unittest.TestCase):
...@@ -1511,7 +1544,8 @@ class TestStaticSaveLoadPickle(unittest.TestCase): ...@@ -1511,7 +1544,8 @@ class TestStaticSaveLoadPickle(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
path = os.path.join("test_static_save_load_pickle", temp_dir = tempfile.TemporaryDirectory()
path = os.path.join(temp_dir.name, "test_static_save_load_pickle",
"pickle_protocol") "pickle_protocol")
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
......
python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py
浏览文件 @ 2eb60ddb
...@@ -23,11 +23,19 @@ from paddle.fluid.optimizer import SGDOptimizer ...@@ -23,11 +23,19 @@ from paddle.fluid.optimizer import SGDOptimizer
from paddle.fluid.tests.unittests.test_imperative_base import new_program_scope from paddle.fluid.tests.unittests.test_imperative_base import new_program_scope
from paddle.fluid.tests.unittests.test_static_save_load import PtbModel from paddle.fluid.tests.unittests.test_static_save_load import PtbModel
import numpy as np import numpy as np
import tempfile
import os
@unittest.skipIf(not core.supports_bfloat16(), @unittest.skipIf(not core.supports_bfloat16(),
"place does not support BF16 evaluation") "place does not support BF16 evaluation")
class TestSaveLoadBF16(unittest.TestCase): class TestSaveLoadBF16(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_place(self): def set_place(self):
return fluid.CPUPlace() return fluid.CPUPlace()
...@@ -108,8 +116,8 @@ class TestSaveLoadBF16(unittest.TestCase): ...@@ -108,8 +116,8 @@ class TestSaveLoadBF16(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update # make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
save_dir = os.path.join(self.temp_dir.name, "test_1")
fluid.save(main_program, "./test_1") fluid.save(main_program, save_dir)
# set var to zero # set var to zero
for var in main_program.list_vars(): for var in main_program.list_vars():
...@@ -122,7 +130,8 @@ class TestSaveLoadBF16(unittest.TestCase): ...@@ -122,7 +130,8 @@ class TestSaveLoadBF16(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero # make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0) self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "./test_1.pdparams", exe) fluid.load(main_program,
os.path.join(self.temp_dir.name, "test_1.pdparams"), exe)
for var in main_program.list_vars(): for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable: if isinstance(var, framework.Parameter) or var.persistable:
......
python/paddle/fluid/tests/unittests/test_static_save_load_large.py
浏览文件 @ 2eb60ddb
...@@ -23,6 +23,7 @@ from test_imperative_base import new_program_scope ...@@ -23,6 +23,7 @@ from test_imperative_base import new_program_scope
import numpy as np import numpy as np
import pickle import pickle
import os import os
import tempfile
LARGE_PARAM = 2**26 LARGE_PARAM = 2**26
...@@ -51,9 +52,10 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): ...@@ -51,9 +52,10 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update # make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0) self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t base_map[var.name] = t
temp_dir = tempfile.TemporaryDirectory()
path = os.path.join("test_static_save_load_large_param", path = os.path.join(temp_dir.name,
"static_save") "test_static_save_load_large_param")
path = os.path.join(path, "static_save")
protocol = 4 protocol = 4
paddle.fluid.save(prog, path, pickle_protocol=protocol) paddle.fluid.save(prog, path, pickle_protocol=protocol)
# set var to zero # set var to zero
...@@ -93,6 +95,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): ...@@ -93,6 +95,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
.get_tensor()) .get_tensor())
base_t = base_map[var.name] base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t)) self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
if __name__ == '__main__': if __name__ == '__main__':
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或