Skip to content

P
Paddle

PaddlePaddle / Paddle
1 年多 前同步成功

通知 2302
Star 20931
Fork 5422
P
Paddle
未验证 提交 2eb60ddb 编写于 作者: Y YuanRisheng 提交者： GitHub
浏览文件
操作

cherry pick 43581 (#43596)

上级 3689a126
显示空白变更内容
内联 并排
Showing with 501 addition and 164 deletion
python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py
浏览文件 @ 2eb60ddb
......@@ -27,6 +27,8 @@ from paddle.fluid.layer_helper import LayerHelper
from paddle import _C_ops
import sys
import tempfile
sys.path.append("./tokenizer")
from tokenizer.bert_tokenizer import BertTokenizer
......@@ -149,13 +151,17 @@ class Predictor(object):
class TestBertTokenizerOp(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.bert_tokenizer = BertTokenizer.from_pretrained("bert-base-chinese")
self.faster_tokenizer = FasterTokenizer(self.bert_tokenizer.vocab)
self.init_data()
self.save_path = os.path.join(DATA_HOME, "fast_tokenizer")
self.save_path = os.path.join(self.temp_dir.name, "fast_tokenizer")
self.param_path = os.path.join(self.save_path, "model.pdparams")
self.inference_path = os.path.join(self.save_path, "inference")
def tearDown(self):
self.temp_dir.cleanup()
def init_data(self):
self.text = [
'选择珠江花园的原因就是方便，有电动扶梯直接到达海边，周围餐馆、食廊、商场、超市、摊位一应俱全。'
......
python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py
浏览文件 @ 2eb60ddb
......@@ -17,11 +17,14 @@ import paddle.fluid as fluid
import paddle.fluid.framework as framework
from paddle.fluid.dygraph.nn import *
import numpy as np
import os
import tempfile
class TestDygraphLoadStatic(unittest.TestCase):
def testLoadStaticModel(self):
# static mode
temp_dir = tempfile.TemporaryDirectory()
a = fluid.data(name="a", shape=[10, 10])
conv_in = fluid.data(name="conv_in", shape=[None, 10, 10, 10])
......@@ -121,9 +124,11 @@ class TestDygraphLoadStatic(unittest.TestCase):
) if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0))
out = exe.run(framework.default_startup_program())
fluid.save(framework.default_main_program(), "./test_1")
fluid.save(framework.default_main_program(),
os.path.join(temp_dir.name, "test_1"))
para_dict = fluid.load_program_state("./test_1")
para_dict = fluid.load_program_state(
os.path.join(temp_dir.name, "test_1"))
new_dict = {}
for k, v in para_dict.items():
......@@ -192,6 +197,7 @@ class TestDygraphLoadStatic(unittest.TestCase):
my_test.set_dict(new_dict, use_structured_name=False)
for k, v in my_test.state_dict().items():
self.assertTrue(np.array_equal(v.numpy(), new_dict[v.name]))
temp_dir.cleanup()
if __name__ == '__main__':
......
python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py
浏览文件 @ 2eb60ddb
......@@ -29,6 +29,8 @@ import six
import paddle
from paddle.fluid.framework import _test_eager_guard
import tempfile
class SimpleLSTMRNN(fluid.Layer):
def __init__(self,
......@@ -209,6 +211,12 @@ class PtbModel(fluid.Layer):
class TestDygraphPtbRnn(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def func_setUp(self):
seed = 90
hidden_size = 10
......@@ -286,7 +294,8 @@ class TestDygraphPtbRnn(unittest.TestCase):
else:
self.base_opti[k] = v
paddle.save(self.opti_dict, "./test_dy_v2.pdopt")
paddle.save(self.opti_dict,
os.path.join(self.temp_dir.name, "test_dy_v2.pdopt"))
self.state_dict = ptb_model.state_dict()
......@@ -295,10 +304,10 @@ class TestDygraphPtbRnn(unittest.TestCase):
np_t = v.numpy()
self.model_base[k] = np_t
paddle.save(self.state_dict, "./test_dy_v2.pdparams")
paddle.save(self.state_dict,
os.path.join(self.temp_dir.name, "test_dy_v2.pdparams"))
def func_testLoadAndSetVarBase(self):
self.setUp()
seed = 90
hidden_size = 10
vocab_size = 1000
......@@ -375,8 +384,10 @@ class TestDygraphPtbRnn(unittest.TestCase):
self.assertTrue(np.sum(np.abs(v.numpy())) == 0)
para_state_dict = paddle.load("./test_dy_v2.pdparams")
opti_state_dict = paddle.load("./test_dy_v2.pdopt")
para_state_dict = paddle.load(
os.path.join(self.temp_dir.name, "test_dy_v2.pdparams"))
opti_state_dict = paddle.load(
os.path.join(self.temp_dir.name, "test_dy_v2.pdopt"))
adam.set_state_dict(opti_state_dict)
opti_dict = adam.state_dict()
......@@ -750,7 +761,8 @@ class TestDygraphPtbRnn(unittest.TestCase):
last_hidden = None
last_cell = None
state_dict, opti_dict = fluid.load_dygraph("./test_dy_v2")
state_dict, opti_dict = fluid.load_dygraph(
os.path.join(self.temp_dir.name, "test_dy_v2"))
adam.set_state_dict(opti_dict)
ptb_model.set_dict(state_dict)
......@@ -907,19 +919,23 @@ class TestDygraphPtbRnn(unittest.TestCase):
with fluid.dygraph.guard():
emb = fluid.dygraph.Embedding([10, 10])
state_dict = emb.state_dict()
paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams'))
paddle.save(
state_dict,
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict = paddle.load(
os.path.join('saved_dy', 'emb_dy.pdparams'))
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
def func_test_no_state_in_input_dict(self):
with fluid.dygraph.guard():
emb = fluid.dygraph.Embedding([10, 10])
state_dict = emb.state_dict()
paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams'))
paddle.save(
state_dict,
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict = paddle.load(
os.path.join('saved_dy', 'emb_dy.pdparams'))
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict.pop('weight')
emb.set_state_dict(para_state_dict)
......@@ -928,10 +944,13 @@ class TestDygraphPtbRnn(unittest.TestCase):
with fluid.dygraph.guard():
emb = fluid.dygraph.Embedding([10, 10])
state_dict = emb.state_dict()
paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams'))
paddle.save(
state_dict,
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict = paddle.load(
os.path.join('saved_dy', 'emb_dy.pdparams'), return_numpy=True)
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'),
return_numpy=True)
para_state_dict['weight'] = np.expand_dims(
para_state_dict['weight'], axis=-1)
......
python/paddle/fluid/tests/unittests/test_io_save_load.py
浏览文件 @ 2eb60ddb
......@@ -19,9 +19,18 @@ import paddle
import paddle.fluid as fluid
from paddle.fluid import core
from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph
import tempfile
import os
class TestSaveLoadAPIError(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.save_dir = os.path.join(self.temp_dir.name, "fake_dir")
def tearDown(self):
self.temp_dir.cleanup()
def func_test_get_valid_program_error(self):
# case 1: CompiledProgram no program
graph = core.Graph(core.ProgramDesc())
......@@ -44,13 +53,13 @@ class TestSaveLoadAPIError(unittest.TestCase):
# case 1: main_program type error when vars None
with self.assertRaises(TypeError):
fluid.io.load_vars(
executor=exe, dirname="./fake_dir", main_program="program")
executor=exe, dirname=self.save_dir, main_program="program")
# case 2: main_program type error when vars not None
with self.assertRaises(TypeError):
fluid.io.load_vars(
executor=exe,
dirname="./fake_dir",
dirname=self.save_dir,
main_program="program",
vars="vars")
......@@ -61,6 +70,12 @@ class TestSaveLoadAPIError(unittest.TestCase):
class TestSaveInferenceModelAPIError(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def func_test_useless_feeded_var_names(self):
start_prog = fluid.Program()
main_prog = fluid.Program()
......@@ -74,7 +89,7 @@ class TestSaveInferenceModelAPIError(unittest.TestCase):
with self.assertRaisesRegexp(
ValueError, "not involved in the target_vars calculation"):
fluid.io.save_inference_model(
dirname='./model',
dirname=os.path.join(self.temp_dir.name, 'model'),
feeded_var_names=['x', 'y'],
target_vars=[z],
executor=exe,
......@@ -87,13 +102,20 @@ class TestSaveInferenceModelAPIError(unittest.TestCase):
class TestWhenTrainWithNoGrad(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def func_test_when_train_with_no_grad(self):
paddle.disable_static()
net = paddle.nn.Linear(1024, 1)
net = paddle.jit.to_static(net)
x = paddle.rand([1024], 'float32')
net(x)
save_path = './train_with_no_grad'
save_path = os.path.join(self.temp_dir.name, 'train_with_no_grad')
paddle.jit.save(net, save_path)
net = paddle.jit.load(save_path)
net.train()
......
python/paddle/fluid/tests/unittests/test_jit_save_load.py
浏览文件 @ 2eb60ddb
......@@ -18,6 +18,7 @@ import os
import pickle
import shutil
import unittest
import tempfile
import numpy as np
import paddle
from paddle.static import InputSpec
......@@ -333,13 +334,18 @@ def train_with_label(layer, input_size=784, label_size=1):
class TestJitSaveLoad(unittest.TestCase):
def setUp(self):
self.model_path = "test_jit_save_load/model"
self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"test_jit_save_load/model")
# enable dygraph mode
fluid.enable_dygraph()
# config seed
paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED)
def tearDown(self):
self.temp_dir.cleanup()
def train_and_save_model(self, model_path=None):
layer = LinearNet(784, 1)
example_inputs, layer, _ = train(layer)
......@@ -395,12 +401,14 @@ class TestJitSaveLoad(unittest.TestCase):
np.array_equal(train_layer(x).numpy(), new_layer(x).numpy()))
def test_load_dygraph_no_path(self):
model_path = "test_jit_save_load.no_path/model_path"
model_path = os.path.join(self.temp_dir.name,
"test_jit_save_load.no_path/model_path")
with self.assertRaises(ValueError):
model_dict, _ = fluid.dygraph.load_dygraph(model_path)
def test_jit_load_no_path(self):
path = "test_jit_save_load.no_path/model_path"
path = os.path.join(self.temp_dir.name,
"test_jit_save_load.no_path/model_path")
with self.assertRaises(ValueError):
loaded_layer = paddle.jit.load(path)
......@@ -409,6 +417,10 @@ class TestSaveLoadWithNestOut(unittest.TestCase):
def setUp(self):
# enable dygraph mode
fluid.enable_dygraph()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_nest_output(self):
x = fluid.dygraph.to_variable(
......@@ -418,7 +430,7 @@ class TestSaveLoadWithNestOut(unittest.TestCase):
dy_outs = flatten(net(x))
net = declarative(net, input_spec=[InputSpec([None, 8], name='x')])
model_path = "net_with_nest_out/model"
model_path = os.path.join(self.temp_dir.name, "net_with_nest_out/model")
paddle.jit.save(net, model_path)
load_net = paddle.jit.load(model_path)
......@@ -440,8 +452,9 @@ class TestSaveLoadWithDictInput(unittest.TestCase):
# {'img': var img : fluid.VarType.LOD_TENSOR.shape(-1, 8).astype(VarType.FP32)},
# {'label': var label : fluid.VarType.LOD_TENSOR.shape(-1, 1).astype(VarType.INT64)})
self.assertEqual(len(net.forward.concrete_program.inputs), 3)
path = "test_jit_save_load_with_dict_input/model"
temp_dir = tempfile.TemporaryDirectory()
path = os.path.join(temp_dir.name,
"test_jit_save_load_with_dict_input/model")
# prune inputs
paddle.jit.save(
layer=net,
......@@ -458,13 +471,15 @@ class TestSaveLoadWithDictInput(unittest.TestCase):
# loaded_net._input_spec():
# [InputSpec(shape=(-1, 8), dtype=VarType.FP32, name=img)]
self.assertEqual(len(loaded_net._input_spec()), 1)
temp_dir.cleanup()
class TestSaveLoadWithDictInputNoPrune(unittest.TestCase):
def test_dict_input(self):
net = LinearNetWithDictInputNoPrune(8, 8)
path = "test_jit_save_load_with_dict_input_no_prune/model"
temp_dir = tempfile.TemporaryDirectory()
path = os.path.join(temp_dir.name,
"test_jit_save_load_with_dict_input_no_prune/model")
# prune inputs
paddle.jit.save(
layer=net,
......@@ -482,12 +497,17 @@ class TestSaveLoadWithDictInputNoPrune(unittest.TestCase):
loaded_out = loaded_net(img, img2)
self.assertEqual(len(loaded_net._input_spec()), 2)
temp_dir.cleanup()
class TestSaveLoadWithInputSpec(unittest.TestCase):
def setUp(self):
# enable dygraph mode
fluid.enable_dygraph()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_with_input_spec(self):
net = LinearNetReturnLoss(8, 8)
......@@ -496,7 +516,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
net.forward, input_spec=[InputSpec(
[None, 8], name='x')])
model_path = "input_spec.output_spec/model"
model_path = os.path.join(self.temp_dir.name,
"input_spec.output_spec/model")
# check inputs and outputs
self.assertTrue(len(net.forward.inputs) == 1)
input_x = net.forward.inputs[0]
......@@ -516,7 +537,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
def test_multi_in_out(self):
net = LinearNetMultiInput(8, 8)
model_path = "multi_inout.output_spec1/model"
model_path = os.path.join(self.temp_dir.name,
"multi_inout.output_spec1/model")
# 1. check inputs and outputs
self.assertTrue(len(net.forward.inputs) == 2)
input_x = net.forward.inputs[0]
......@@ -538,7 +560,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
pred_x, pred_y = infer_layer(x, y)
# 1. prune y and loss
model_path = "multi_inout.output_spec2/model"
model_path = os.path.join(self.temp_dir.name,
"multi_inout.output_spec2/model")
output_spec = net.forward.outputs[:1]
paddle.jit.save(net, model_path, [input_x], output_spec=output_spec)
# 2. load again
......@@ -552,7 +575,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
def test_multi_in_out1(self):
net = LinearNetMultiInput1(8, 8)
model_path = "multi_inout1.output_spec1/model"
model_path = os.path.join(self.temp_dir.name,
"multi_inout1.output_spec1/model")
# 1. check inputs and outputs
self.assertTrue(len(net.forward.inputs) == 2)
input_x = net.forward.inputs[0]
......@@ -574,7 +598,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
pred_x, pred_y = infer_layer(x, y)
# 1. prune y and loss
model_path = "multi_inout1.output_spec2/model"
model_path = os.path.join(self.temp_dir.name,
"multi_inout1.output_spec2/model")
output_spec = net.forward.outputs[:1]
paddle.jit.save(net, model_path, (input_x, ), output_spec=output_spec)
# 2. load again
......@@ -593,6 +618,10 @@ class TestJitSaveLoadConfig(unittest.TestCase):
# config seed
paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED)
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_output_spec(self):
train_layer = LinearNetReturnLoss(8, 8)
......@@ -606,7 +635,8 @@ class TestJitSaveLoadConfig(unittest.TestCase):
adam.minimize(loss)
train_layer.clear_gradients()
model_path = "save_load_config.output_spec"
model_path = os.path.join(self.temp_dir.name,
"save_load_config.output_spec")
output_spec = [out]
paddle.jit.save(
layer=train_layer,
......@@ -623,22 +653,22 @@ class TestJitSaveLoadConfig(unittest.TestCase):
def test_save_no_support_config_error(self):
layer = LinearNet(784, 1)
path = "no_support_config_test"
path = os.path.join(self.temp_dir.name, "no_support_config_test")
with self.assertRaises(ValueError):
paddle.jit.save(layer=layer, path=path, model_filename="")
def test_load_empty_model_filename_error(self):
path = "error_model_filename_test"
path = os.path.join(self.temp_dir.name, "error_model_filename_test")
with self.assertRaises(ValueError):
paddle.jit.load(path, model_filename="")
def test_load_empty_params_filename_error(self):
path = "error_params_filename_test"
path = os.path.join(self.temp_dir.name, "error_params_filename_test")
with self.assertRaises(ValueError):
paddle.jit.load(path, params_filename="")
def test_load_with_no_support_config(self):
path = "no_support_config_test"
path = os.path.join(self.temp_dir.name, "no_support_config_test")
with self.assertRaises(ValueError):
paddle.jit.load(path, separate_params=True)
......@@ -646,7 +676,9 @@ class TestJitSaveLoadConfig(unittest.TestCase):
class TestJitMultipleLoading(unittest.TestCase):
def setUp(self):
self.linear_size = 4
self.model_path = "jit_multi_load/model"
self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"jit_multi_load/model")
# enable dygraph mode
fluid.enable_dygraph()
# config seed
......@@ -655,6 +687,9 @@ class TestJitMultipleLoading(unittest.TestCase):
# train and save base model
self.train_and_save_orig_model()
def tearDown(self):
self.temp_dir.cleanup()
def train_and_save_orig_model(self):
layer = LinearNet(self.linear_size, self.linear_size)
example_inputs, layer, _ = train(layer, self.linear_size, 1)
......@@ -674,13 +709,18 @@ class TestJitMultipleLoading(unittest.TestCase):
class TestJitPruneModelAndLoad(unittest.TestCase):
def setUp(self):
self.linear_size = 4
self.model_path = "jit_prune_model_and_load/model"
self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"jit_prune_model_and_load/model")
# enable dygraph mode
fluid.enable_dygraph()
# config seed
paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED)
def tearDown(self):
self.temp_dir.cleanup()
def train_and_save(self):
train_layer = LinearNetReturnHidden(8, 8)
adam = fluid.optimizer.AdamOptimizer(
......@@ -735,6 +775,10 @@ class TestJitSaveMultiCases(unittest.TestCase):
# config seed
paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED)
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def verify_inference_correctness(self,
layer,
......@@ -767,7 +811,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train(layer)
model_path = "test_no_prune_to_static_after_train/model"
model_path = os.path.join(self.temp_dir.name,
"test_no_prune_to_static_after_train/model")
paddle.jit.save(layer, model_path)
self.verify_inference_correctness(layer, model_path)
......@@ -775,7 +820,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_no_prune_to_static_no_train(self):
layer = LinearNetWithInputSpec(784, 1)
model_path = "test_no_prune_to_static_no_train/model"
model_path = os.path.join(self.temp_dir.name,
"test_no_prune_to_static_no_train/model")
paddle.jit.save(layer, model_path)
self.verify_inference_correctness(layer, model_path)
......@@ -785,7 +831,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train(layer)
model_path = "test_no_prune_no_to_static_after_train/model"
model_path = os.path.join(
self.temp_dir.name, "test_no_prune_no_to_static_after_train/model")
paddle.jit.save(
layer,
model_path,
......@@ -799,7 +846,9 @@ class TestJitSaveMultiCases(unittest.TestCase):
example_inputs, _, _ = train(layer)
model_path = "test_no_prune_no_to_static_after_train_with_examples/model"
model_path = os.path.join(
self.temp_dir.name,
"test_no_prune_no_to_static_after_train_with_examples/model")
paddle.jit.save(layer=layer, path=model_path, input_spec=example_inputs)
self.verify_inference_correctness(layer, model_path)
......@@ -807,7 +856,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_no_prune_no_to_static_no_train(self):
layer = LinearNetNotDeclarative(784, 1)
model_path = "test_no_prune_no_to_static_no_train/model"
model_path = os.path.join(self.temp_dir.name,
"test_no_prune_no_to_static_no_train/model")
paddle.jit.save(
layer,
model_path,
......@@ -821,7 +871,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
out = train_with_label(layer)
model_path = "test_prune_to_static_after_train/model"
model_path = os.path.join(self.temp_dir.name,
"test_prune_to_static_after_train/model")
paddle.jit.save(
layer,
model_path,
......@@ -837,7 +888,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_prune_to_static_no_train(self):
layer = LinerNetWithLabel(784, 1)
model_path = "test_prune_to_static_no_train/model"
model_path = os.path.join(self.temp_dir.name,
"test_prune_to_static_no_train/model")
# TODO: no train, cannot get output_spec var here
# now only can use index
output_spec = layer.forward.outputs[:1]
......@@ -856,7 +908,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_prune_input_to_static_no_train(self):
layer = LinerNetWithPruneInput(784, 1)
model_path = "test_prune_input_to_static_no_train/model"
model_path = os.path.join(self.temp_dir.name,
"test_prune_input_to_static_no_train/model")
paddle.jit.save(
layer,
model_path,
......@@ -870,7 +923,9 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_prune_useless_input_to_static_no_train(self):
layer = LinerNetWithUselessInput(784, 1)
model_path = "test_prune_useless_input_to_static_no_train/model"
model_path = os.path.join(
self.temp_dir.name,
"test_prune_useless_input_to_static_no_train/model")
paddle.jit.save(
layer,
model_path,
......@@ -886,7 +941,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train(layer)
model_path = "test_no_prune_input_spec_name_warning/model"
model_path = os.path.join(self.temp_dir.name,
"test_no_prune_input_spec_name_warning/model")
paddle.jit.save(
layer,
model_path,
......@@ -907,7 +963,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train(layer)
model_path = "test_not_prune_output_spec_name_warning/model"
model_path = os.path.join(
self.temp_dir.name, "test_not_prune_output_spec_name_warning/model")
out = paddle.to_tensor(np.random.random((1, 1)).astype('float'))
paddle.jit.save(layer, model_path, output_spec=[out])
......@@ -916,7 +973,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_prune_input_spec_name_error(self):
layer = LinerNetWithLabel(784, 1)
model_path = "test_prune_input_spec_name_error/model"
model_path = os.path.join(self.temp_dir.name,
"test_prune_input_spec_name_error/model")
with self.assertRaises(ValueError):
paddle.jit.save(
layer,
......@@ -937,7 +995,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train_with_label(layer)
model_path = "test_prune_to_static_after_train/model"
model_path = os.path.join(self.temp_dir.name,
"test_prune_to_static_after_train/model")
out = paddle.to_tensor(np.random.random((1, 1)).astype('float'))
with self.assertRaises(ValueError):
paddle.jit.save(
......@@ -952,10 +1011,15 @@ class TestJitSaveMultiCases(unittest.TestCase):
class TestJitSaveLoadEmptyLayer(unittest.TestCase):
def setUp(self):
self.model_path = "jit_save_load_empty_layer/model"
self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"jit_save_load_empty_layer/model")
# enable dygraph mode
paddle.disable_static()
def tearDown(self):
self.temp_dir.cleanup()
def test_save_load_empty_layer(self):
layer = EmptyLayer()
x = paddle.to_tensor(np.random.random((10)).astype('float32'))
......@@ -968,10 +1032,15 @@ class TestJitSaveLoadEmptyLayer(unittest.TestCase):
class TestJitSaveLoadNoParamLayer(unittest.TestCase):
def setUp(self):
self.model_path = "jit_save_load_no_param_layer/model"
self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"jit_save_load_no_param_layer/model")
# enable dygraph mode
paddle.disable_static()
def tearDown(self):
self.temp_dir.cleanup()
def test_save_load_no_param_layer(self):
layer = NoParamLayer()
x = paddle.to_tensor(np.random.random((5)).astype('float32'))
......@@ -987,9 +1056,14 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase):
def setUp(self):
# enable dygraph mode
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_inference(self):
model_path_inference = "jit_save_load_multi_methods/model"
model_path_inference = os.path.join(self.temp_dir.name,
"jit_save_load_multi_methods/model")
IMAGE_SIZE = 224
layer = LinearNetWithMultiStaticFunc(IMAGE_SIZE, 10)
inps = paddle.randn([1, IMAGE_SIZE])
......@@ -1005,14 +1079,16 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase):
)) < 1e-5)
def test_jit_save_load_multi_methods_inputspec(self):
model_path = 'jit_save_load_multi_methods/model'
model_path = os.path.join(self.temp_dir.name,
'jit_save_load_multi_methods/model')
layer = LinearNetWithMultiStaticFunc(784, 1)
with self.assertRaises(ValueError):
paddle.jit.save(
layer, model_path, input_spec=[InputSpec(shape=[None, 784])])
def test_parse_name(self):
model_path_inference = "jit_save_load_parse_name/model"
model_path_inference = os.path.join(self.temp_dir.name,
"jit_save_load_parse_name/model")
IMAGE_SIZE = 224
layer = LinearNet(IMAGE_SIZE, 1)
inps = paddle.randn([1, IMAGE_SIZE])
......@@ -1082,9 +1158,14 @@ class TestJitSaveLoadSaveWithoutRunning(unittest.TestCase):
def setUp(self):
# enable dygraph mode
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_save_load_finetune_load(self):
model_path = "test_jit_save_load_save_without_running/model"
model_path = os.path.join(
self.temp_dir.name, "test_jit_save_load_save_without_running/model")
IMAGE_SIZE = 224
inps0 = paddle.randn([1, IMAGE_SIZE])
inps1 = paddle.randn([2, IMAGE_SIZE])
......@@ -1124,9 +1205,14 @@ class TestJitSaveLoadFinetuneLoad(unittest.TestCase):
def setUp(self):
# enable dygraph mode
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_save_load_finetune_load(self):
model_path = "test_jit_save_load_finetune_load/model"
model_path = os.path.join(self.temp_dir.name,
"test_jit_save_load_finetune_load/model")
IMAGE_SIZE = 224
inps0 = paddle.randn([1, IMAGE_SIZE])
inps1 = paddle.randn([2, IMAGE_SIZE])
......@@ -1162,13 +1248,18 @@ class TestJitSaveLoadFinetuneLoad(unittest.TestCase):
class TestJitSaveLoadFunctionCase1(unittest.TestCase):
def setUp(self):
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_static_function(self):
@paddle.jit.to_static
def fun(inputs):
return paddle.tanh(inputs)
path = 'test_jit_save_load_function_1/func'
path = os.path.join(self.temp_dir.name,
'test_jit_save_load_function_1/func')
inps = paddle.rand([3, 6])
origin = fun(inps)
......@@ -1182,6 +1273,10 @@ class TestJitSaveLoadFunctionCase1(unittest.TestCase):
class TestJitSaveLoadFunctionCase2(unittest.TestCase):
def setUp(self):
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function_input_spec(self):
@paddle.jit.to_static(input_spec=[
......@@ -1191,7 +1286,8 @@ class TestJitSaveLoadFunctionCase2(unittest.TestCase):
def fun(inputs):
return paddle.nn.functional.relu(inputs)
path = 'test_jit_save_load_function_2/func'
path = os.path.join(self.temp_dir.name,
'test_jit_save_load_function_2/func')
inps = paddle.rand([3, 6])
origin = fun(inps)
......@@ -1204,12 +1300,17 @@ class TestJitSaveLoadFunctionCase2(unittest.TestCase):
class TestJitSaveLoadFunctionCase3(unittest.TestCase):
def setUp(self):
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function_function(self):
def fun(inputs):
return paddle.tanh(inputs)
path = 'test_jit_save_load_function_3/func'
path = os.path.join(self.temp_dir.name,
'test_jit_save_load_function_3/func')
inps = paddle.rand([3, 6])
origin = fun(inps)
......@@ -1229,6 +1330,10 @@ class TestJitSaveLoadFunctionCase3(unittest.TestCase):
class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase):
def setUp(self):
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function(self):
class LinearNet(paddle.nn.Layer):
......@@ -1249,7 +1354,9 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase):
func = paddle.jit.to_static(
layer.anothor_forward, [paddle.static.InputSpec(shape=[-1, 5])])
path = 'test_jit_save_load_function_with_params_case1/func'
path = os.path.join(
self.temp_dir.name,
'test_jit_save_load_function_with_params_case1/func')
paddle.jit.save(func, path)
load_func = paddle.jit.load(path)
......@@ -1260,6 +1367,10 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase):
class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase):
def setUp(self):
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function(self):
class LinearNet(paddle.nn.Layer):
......@@ -1278,7 +1389,9 @@ class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase):
inps = paddle.rand([3, 5])
path = 'test_jit_save_load_function_with_params_case2/func'
path = os.path.join(
self.temp_dir.name,
'test_jit_save_load_function_with_params_case2/func')
paddle.jit.save(layer.anothor_forward, path)
origin_result = layer.anothor_forward(inps)
load_func = paddle.jit.load(path)
......@@ -1292,6 +1405,10 @@ class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase):
class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase):
def setUp(self):
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function(self):
class LinearNet(paddle.nn.Layer):
......@@ -1311,7 +1428,9 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase):
inps = paddle.rand([3, 5])
origin = layer.anothor_forward(inps)
path = 'test_jit_save_load_function_with_params_case3/func'
path = os.path.join(
self.temp_dir.name,
'test_jit_save_load_function_with_params_case3/func')
paddle.jit.save(layer.anothor_forward, path)
load_func = paddle.jit.load(path)
......@@ -1320,6 +1439,12 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase):
class TestJitSaveLoadDataParallel(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def verify_inference_correctness(self, layer, path):
layer.eval()
loaded_layer = paddle.jit.load(path)
......@@ -1336,8 +1461,8 @@ class TestJitSaveLoadDataParallel(unittest.TestCase):
def test_jit_save_data_parallel_with_inputspec(self):
layer = LinearNetNotDeclarative(784, 1)
layer = paddle.DataParallel(layer)
path = "jit_save_data_parallel_with_inputspec/model"
path = os.path.join(self.temp_dir.name,
"jit_save_data_parallel_with_inputspec/model")
paddle.jit.save(
layer=layer, path=path, input_spec=[InputSpec(shape=[None, 784])])
......@@ -1347,7 +1472,8 @@ class TestJitSaveLoadDataParallel(unittest.TestCase):
layer = LinearNetWithInputSpec(784, 1)
layer = paddle.DataParallel(layer)
path = "jit_save_data_parallel_with_to_static/model"
path = os.path.join(self.temp_dir.name,
"jit_save_data_parallel_with_to_static/model")
paddle.jit.save(layer, path)
self.verify_inference_correctness(layer, path)
......@@ -1368,6 +1494,12 @@ class InputSepcLayer(paddle.nn.Layer):
class TestInputSpecCompatibility(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def _assert_input_spec_layer_return(self, expect_layer, test_layer):
input_x = paddle.uniform([8, 8], dtype='float32')
input_y = paddle.uniform([8, 1], dtype='float64')
......@@ -1380,7 +1512,8 @@ class TestInputSpecCompatibility(unittest.TestCase):
def test_jit_save_compatible_input_sepc(self):
layer = InputSepcLayer()
save_dir = "jit_save_compatible_input_spec"
save_dir = os.path.join(self.temp_dir.name,
"jit_save_compatible_input_spec")
path = save_dir + "/model"
paddle.jit.save(layer=layer, path=path)
......@@ -1414,7 +1547,8 @@ class TestInputSpecCompatibility(unittest.TestCase):
def test_jit_save_incompatible_input_sepc(self):
layer = InputSepcLayer()
save_dir = "jit_save_compatible_input_spec"
save_dir = os.path.join(self.temp_dir.name,
"jit_save_compatible_input_spec")
path = save_dir + "/model"
with self.assertRaises(ValueError):
......
python/paddle/fluid/tests/unittests/test_load_op.py
浏览文件 @ 2eb60ddb
......@@ -19,6 +19,8 @@ import numpy as np
from op_test import OpTest, randomize_probability
import paddle.fluid as fluid
import paddle.fluid.layers as layers
import os
import tempfile
class TestLoadOp(unittest.TestCase):
......@@ -26,6 +28,7 @@ class TestLoadOp(unittest.TestCase):
"""
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.ones = np.ones((4, 4)).astype('float32')
main_prog = fluid.Program()
start_prog = fluid.Program()
......@@ -41,14 +44,20 @@ class TestLoadOp(unittest.TestCase):
exe = fluid.Executor(fluid.CPUPlace())
exe.run(start_prog)
fluid.io.save_persistables(
exe, dirname="./model", main_program=main_prog)
exe,
dirname=os.path.join(self.temp_dir.name, "./model"),
main_program=main_prog)
def tearDown(self):
self.temp_dir.cleanup()
def test_load(self):
main_prog = fluid.Program()
start_prog = fluid.Program()
with fluid.program_guard(main_prog, start_prog):
var = layers.create_tensor(dtype='float32')
layers.load(var, file_path='./model/w')
layers.load(
var, file_path=os.path.join(self.temp_dir.name, './model/w'))
exe = fluid.Executor(fluid.CPUPlace())
exe.run(start_prog)
......
python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py
浏览文件 @ 2eb60ddb
......@@ -23,6 +23,7 @@ import paddle
import paddle.fluid as fluid
from paddle.fluid import core
from test_imperative_base import new_program_scope
import tempfile
def convolutional_neural_network(img):
......@@ -59,6 +60,7 @@ def static_train_net(img, label):
class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.seed = 90
self.epoch_num = 1
self.batch_size = 128
......@@ -66,6 +68,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
# enable static mode
paddle.enable_static()
def tearDown(self):
self.temp_dir.cleanup()
def train_and_save_model(self, only_params=False):
with new_program_scope():
startup_program = fluid.default_startup_program()
......@@ -121,7 +126,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.assertTrue(np.array_equal(value, load_dict[var_name]))
def test_load_default(self):
self.save_dirname = "static_mnist.load_state_dict.default"
self.save_dirname = os.path.join(self.temp_dir.name,
"static_mnist.load_state_dict.default")
self.model_filename = None
self.params_filename = None
orig_param_dict = self.train_and_save_model()
......@@ -133,7 +139,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_with_model_filename(self):
self.save_dirname = "static_mnist.load_state_dict.model_filename"
self.save_dirname = os.path.join(
self.temp_dir.name, "static_mnist.load_state_dict.model_filename")
self.model_filename = "static_mnist.model"
self.params_filename = None
orig_param_dict = self.train_and_save_model()
......@@ -147,7 +154,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_with_param_filename(self):
self.save_dirname = "static_mnist.load_state_dict.param_filename"
self.save_dirname = os.path.join(
self.temp_dir.name, "static_mnist.load_state_dict.param_filename")
self.model_filename = None
self.params_filename = "static_mnist.params"
orig_param_dict = self.train_and_save_model()
......@@ -161,7 +169,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_with_model_and_param_filename(self):
self.save_dirname = "static_mnist.load_state_dict.model_and_param_filename"
self.save_dirname = os.path.join(
self.temp_dir.name,
"static_mnist.load_state_dict.model_and_param_filename")
self.model_filename = "static_mnist.model"
self.params_filename = "static_mnist.params"
orig_param_dict = self.train_and_save_model()
......@@ -179,7 +189,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_state_dict_from_save_params(self):
self.save_dirname = "static_mnist.load_state_dict.save_params"
self.save_dirname = os.path.join(
self.temp_dir.name, "static_mnist.load_state_dict.save_params")
self.params_filename = None
orig_param_dict = self.train_and_save_model(True)
......
python/paddle/fluid/tests/unittests/test_paddle_save_load.py
浏览文件 @ 2eb60ddb
......@@ -19,6 +19,7 @@ import numpy as np
import os
import sys
from io import BytesIO
import tempfile
import paddle
import paddle.nn as nn
......@@ -90,7 +91,10 @@ def train(layer, loader, loss_fn, opt):
class TestSaveLoadLargeParameters(unittest.TestCase):
def setUp(self):
pass
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_large_parameters_paddle_save(self):
# enable dygraph mode
......@@ -100,7 +104,8 @@ class TestSaveLoadLargeParameters(unittest.TestCase):
layer = LayerWithLargeParameters()
save_dict = layer.state_dict()
path = os.path.join("test_paddle_save_load_large_param_save",
path = os.path.join(self.temp_dir.name,
"test_paddle_save_load_large_param_save",
"layer.pdparams")
protocol = 4
paddle.save(save_dict, path, protocol=protocol)
......@@ -111,6 +116,12 @@ class TestSaveLoadLargeParameters(unittest.TestCase):
class TestSaveLoadPickle(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_pickle_protocol(self):
# enable dygraph mode
paddle.disable_static()
......@@ -118,7 +129,8 @@ class TestSaveLoadPickle(unittest.TestCase):
layer = LinearNet()
save_dict = layer.state_dict()
path = os.path.join("test_paddle_save_load_pickle_protocol",
path = os.path.join(self.temp_dir.name,
"test_paddle_save_load_pickle_protocol",
"layer.pdparams")
with self.assertRaises(ValueError):
......@@ -143,6 +155,12 @@ class TestSaveLoadPickle(unittest.TestCase):
class TestSaveLoadAny(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_zero(self, prog, place, scope=None):
if scope is None:
scope = fluid.global_scope()
......@@ -203,7 +221,8 @@ class TestSaveLoadAny(unittest.TestCase):
t = np.array(fluid.global_scope().find_var(var.name)
.get_tensor())
base_map[var.name] = t
path = os.path.join("test_replace_static_save_load", "model")
path = os.path.join(self.temp_dir.name,
"test_replace_static_save_load", "model")
# paddle.save, legacy paddle.fluid.load
self.replace_static_save(prog, path)
self.set_zero(prog, place)
......@@ -229,7 +248,9 @@ class TestSaveLoadAny(unittest.TestCase):
for var in prog.list_vars():
if var.persistable:
tensor = var.get_value(fluid.global_scope())
paddle.save(tensor, os.path.join(path_vars, var.name))
paddle.save(
tensor,
os.path.join(self.temp_dir.name, path_vars, var.name))
with self.assertRaises(TypeError):
var.get_value('fluid.global_scope()')
with self.assertRaises(ValueError):
......@@ -248,7 +269,8 @@ class TestSaveLoadAny(unittest.TestCase):
for var in prog.list_vars():
if var.persistable:
tensor = paddle.load(
os.path.join(path_vars, var.name), return_numpy=False)
os.path.join(self.temp_dir.name, path_vars, var.name),
return_numpy=False)
var.set_value(tensor)
new_t = np.array(fluid.global_scope().find_var(var.name)
.get_tensor())
......@@ -282,7 +304,8 @@ class TestSaveLoadAny(unittest.TestCase):
y.mean().backward()
adam.step()
state_dict = adam.state_dict()
path = 'paddle_save_load_v2/model.pdparams'
path = os.path.join(self.temp_dir.name,
'paddle_save_load_v2/model.pdparams')
with self.assertRaises(TypeError):
paddle.save(state_dict, path, use_binary_format='False')
# legacy paddle.save, paddle.load
......@@ -306,7 +329,8 @@ class TestSaveLoadAny(unittest.TestCase):
# enable dygraph mode
paddle.disable_static()
layer = LinearNet()
path = 'paddle_save_load_v2/var_dygraph'
path = os.path.join(self.temp_dir.name,
'paddle_save_load_v2/var_dygraph')
tensor = layer._linear.weight
with self.assertRaises(ValueError):
paddle.save(tensor, path, pickle_protocol='3')
......@@ -348,7 +372,8 @@ class TestSaveLoadAny(unittest.TestCase):
break
scope = fluid.global_scope()
origin_tensor = np.array(tensor)
path = 'test_single_pickle_var_static/var'
path = os.path.join(self.temp_dir.name,
'test_single_pickle_var_static/var')
paddle.save(tensor, path)
self.set_zero(prog, place, scope)
# static load
......@@ -370,7 +395,8 @@ class TestSaveLoadAny(unittest.TestCase):
def test_dygraph_save_static_load(self):
inps = np.random.randn(1, IMAGE_SIZE).astype('float32')
path = 'test_dygraph_save_static_load/dy-static.pdparams'
path = os.path.join(self.temp_dir.name,
'test_dygraph_save_static_load/dy-static.pdparams')
paddle.disable_static()
with paddle.utils.unique_name.guard():
layer = LinearNet()
......@@ -414,10 +440,14 @@ class TestSaveLoadAny(unittest.TestCase):
})
obj4 = (np.random.randn(5, 6), (123, ))
path1 = "test_save_load_any_complex_object_dygraph/obj1"
path2 = "test_save_load_any_complex_object_dygraph/obj2"
path3 = "test_save_load_any_complex_object_dygraph/obj3"
path4 = "test_save_load_any_complex_object_dygraph/obj4"
path1 = os.path.join(self.temp_dir.name,
"test_save_load_any_complex_object_dygraph/obj1")
path2 = os.path.join(self.temp_dir.name,
"test_save_load_any_complex_object_dygraph/obj2")
path3 = os.path.join(self.temp_dir.name,
"test_save_load_any_complex_object_dygraph/obj3")
path4 = os.path.join(self.temp_dir.name,
"test_save_load_any_complex_object_dygraph/obj4")
paddle.save(obj1, path1)
paddle.save(obj2, path2)
paddle.save(obj3, path3)
......@@ -584,10 +614,18 @@ class TestSaveLoadAny(unittest.TestCase):
})
obj4 = (np.ndarray([3, 4], dtype="float32"), )
path1 = "test_save_load_any_complex_object_static/obj1"
path2 = "test_save_load_any_complex_object_static/obj2"
path3 = "test_save_load_any_complex_object_static/obj3"
path4 = "test_save_load_any_complex_object_static/obj4"
path1 = os.path.join(
self.temp_dir.name,
"test_save_load_any_complex_object_static/obj1")
path2 = os.path.join(
self.temp_dir.name,
"test_save_load_any_complex_object_static/obj2")
path3 = os.path.join(
self.temp_dir.name,
"test_save_load_any_complex_object_static/obj3")
path4 = os.path.join(
self.temp_dir.name,
"test_save_load_any_complex_object_static/obj4")
paddle.save(obj1, path1)
paddle.save(obj2, path2)
paddle.save(obj3, path3)
......@@ -751,7 +789,8 @@ class TestSaveLoadAny(unittest.TestCase):
def test_varbase_binary_var(self):
paddle.disable_static()
varbase = paddle.randn([3, 2], dtype='float32')
path = 'test_paddle_save_load_varbase_binary_var/varbase'
path = os.path.join(self.temp_dir.name,
'test_paddle_save_load_varbase_binary_var/varbase')
paddle.save(varbase, path, use_binary_format=True)
load_array = paddle.load(path, return_numpy=True)
load_tensor = paddle.load(path, return_numpy=False)
......@@ -836,6 +875,10 @@ class TestSaveLoad(unittest.TestCase):
# config seed
paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED)
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def build_and_train_model(self):
# create network
......@@ -863,8 +906,10 @@ class TestSaveLoad(unittest.TestCase):
layer, opt = self.build_and_train_model()
# save
layer_save_path = "test_paddle_save_load.linear.pdparams"
opt_save_path = "test_paddle_save_load.linear.pdopt"
layer_save_path = os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear.pdparams")
opt_save_path = os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear.pdopt")
layer_state_dict = layer.state_dict()
opt_state_dict = opt.state_dict()
......@@ -880,7 +925,9 @@ class TestSaveLoad(unittest.TestCase):
# test save load in static mode
paddle.enable_static()
static_save_path = "static_mode_test/test_paddle_save_load.linear.pdparams"
static_save_path = os.path.join(
self.temp_dir.name,
"static_mode_test/test_paddle_save_load.linear.pdparams")
paddle.save(layer_state_dict, static_save_path)
load_static_state_dict = paddle.load(static_save_path)
self.check_load_state_dict(layer_state_dict, load_static_state_dict)
......@@ -891,20 +938,28 @@ class TestSaveLoad(unittest.TestCase):
# 2. test save path format error
with self.assertRaises(ValueError):
paddle.save(layer_state_dict, "test_paddle_save_load.linear.model/")
paddle.save(layer_state_dict,
os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear.model/"))
# 3. test load path not exist error
with self.assertRaises(ValueError):
paddle.load("test_paddle_save_load.linear.params")
paddle.load(
os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear.params"))
# 4. test load old save path error
with self.assertRaises(ValueError):
paddle.load("test_paddle_save_load.linear")
paddle.load(
os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear"))
class TestSaveLoadProgram(unittest.TestCase):
def test_save_load_program(self):
paddle.enable_static()
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope():
layer = LinearNet()
data = paddle.static.data(
......@@ -914,8 +969,12 @@ class TestSaveLoadProgram(unittest.TestCase):
startup_program = paddle.static.default_startup_program()
origin_main = main_program.desc.serialize_to_string()
origin_startup = startup_program.desc.serialize_to_string()
path1 = "test_paddle_save_load_program/main_program.pdmodel"
path2 = "test_paddle_save_load_program/startup_program.pdmodel"
path1 = os.path.join(
temp_dir.name,
"test_paddle_save_load_program/main_program.pdmodel")
path2 = os.path.join(
temp_dir.name,
"test_paddle_save_load_program/startup_program.pdmodel")
paddle.save(main_program, path1)
paddle.save(startup_program, path2)
......@@ -924,11 +983,13 @@ class TestSaveLoadProgram(unittest.TestCase):
load_startup = paddle.load(path2).desc.serialize_to_string()
self.assertTrue(origin_main == load_main)
self.assertTrue(origin_startup == load_startup)
temp_dir.cleanup()
class TestSaveLoadLayer(unittest.TestCase):
def test_save_load_layer(self):
paddle.disable_static()
temp_dir = tempfile.TemporaryDirectory()
inps = paddle.randn([1, IMAGE_SIZE], dtype='float32')
layer1 = LinearNet()
layer2 = LinearNet()
......@@ -936,9 +997,11 @@ class TestSaveLoadLayer(unittest.TestCase):
layer2.eval()
origin_layer = (layer1, layer2)
origin = (layer1(inps), layer2(inps))
path = "test_save_load_layer_/layer.pdmodel"
path = os.path.join(temp_dir.name,
"test_save_load_layer_/layer.pdmodel")
with self.assertRaises(ValueError):
paddle.save(origin_layer, path)
temp_dir.cleanup()
if __name__ == '__main__':
......
python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py
浏览文件 @ 2eb60ddb
......@@ -21,6 +21,7 @@ import os
import sys
import six
import platform
import tempfile
import paddle
import paddle.nn as nn
......@@ -37,6 +38,10 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
def setUp(self):
# enable static graph mode
paddle.enable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_zero(self, prog, place, scope=None):
if scope is None:
......@@ -94,7 +99,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
# test for replace_save_vars/io.load_vars
path_vars1 = 'test_replace_save_load_vars_binary1/model'
path_vars1 = os.path.join(
self.temp_dir.name, 'test_replace_save_load_vars_binary1/model')
self.replace_save_vars(prog, path_vars1)
# set var to zero
self.set_zero(prog, place)
......@@ -111,7 +117,9 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self.assertTrue(np.array_equal(new_t, base_t))
# test for io.save_vars/replace_load_vars
path_vars2 = 'test_replace_save_load_vars_binary2/model/'
path_vars2 = os.path.join(
self.temp_dir.name,
'test_replace_save_load_vars_binary2/model/')
fluid.io.save_vars(
exe, path_vars2, main_program=prog, vars=var_list)
self.set_zero(prog, place)
......@@ -141,7 +149,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
prog = paddle.static.default_main_program()
exe.run(fluid.default_startup_program())
dirname = 'test_save_load_lod_tensor1/tensor_'
dirname = os.path.join(self.temp_dir.name,
'test_save_load_lod_tensor1/tensor_')
for var in prog.list_vars():
if var.persistable and list(
var.shape) == [IMAGE_SIZE, OUTPUT_NUM]:
......@@ -162,12 +171,13 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self.assertTrue(np.array_equal(origin, to_array))
with self.assertRaises(NotImplementedError):
path = 'test_save_load_error/temp'
path = os.path.join(self.temp_dir.name, 'test_save_load_error/temp')
paddle.save({}, path, use_binary_format=True)
# On the Windows platform, when parsing a string that can't be parsed as a `Program`, `desc_.ParseFromString` has a timeout risk.
if 'Windows' != platform.system():
with self.assertRaises(ValueError):
path = 'test_save_load_error/temp'
path = os.path.join(self.temp_dir.name,
'test_save_load_error/temp')
with open(path, "w") as f:
f.write('\0')
paddle.load(path)
......@@ -178,11 +188,17 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
with self.assertRaises(RuntimeError):
fluid.core.save_lod_tensor(
temp_lod, 'test_save_load_error_not_exist_file/not_exist_file')
temp_lod,
os.path.join(
self.temp_dir.name,
'test_save_load_error_not_exist_file/not_exist_file'))
with self.assertRaises(RuntimeError):
fluid.core.load_lod_tensor(
temp_lod, 'test_save_load_error_not_exist_file/not_exist_file')
temp_lod,
os.path.join(
self.temp_dir.name,
'test_save_load_error_not_exist_file/not_exist_file'))
# save to memory
byio = BytesIO()
......@@ -206,7 +222,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
rows = [0, 4, 7]
row_numel = 12
selected_rows = fluid.core.SelectedRows(rows, height)
path = 'test_paddle_save_load_selected_rows/sr.pdsr'
path = os.path.join(self.temp_dir.name,
'test_paddle_save_load_selected_rows/sr.pdsr')
with self.assertRaises(ValueError):
paddle.save(selected_rows, path, use_binary_format=True)
......@@ -227,11 +244,15 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
with self.assertRaises(RuntimeError):
fluid.core.save_selected_rows(
selected_rows,
'test_paddle_save_load_selected_rows_not_exist_file/temp')
os.path.join(
self.temp_dir.name,
'test_paddle_save_load_selected_rows_not_exist_file/temp'))
with self.assertRaises(RuntimeError):
fluid.core.load_selected_rows(
selected_rows,
'test_paddle_save_load_selected_rows_not_exist_file/temp')
os.path.join(
self.temp_dir.name,
'test_paddle_save_load_selected_rows_not_exist_file/temp'))
# save to memory
byio = BytesIO()
......
python/paddle/fluid/tests/unittests/test_static_save_load.py
浏览文件 @ 2eb60ddb
......@@ -30,6 +30,7 @@ import six
import pickle
import os
import errno
import tempfile
paddle.enable_static()
......@@ -230,6 +231,7 @@ class TestSaveLoadBase(unittest.TestCase):
init_scale = 0.1
batch_size = 4
batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope():
fluid.default_startup_program().random_seed = seed
......@@ -297,7 +299,7 @@ class TestSaveLoadBase(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, "./test_1")
fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
# set var to zero
for var in main_program.list_vars():
......@@ -310,7 +312,8 @@ class TestSaveLoadBase(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "./test_1.pdparams", exe)
fluid.load(main_program,
os.path.join(temp_dir.name, "test_1.pdparams"), exe)
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -318,6 +321,7 @@ class TestSaveLoadBase(unittest.TestCase):
.get_tensor())
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestSaveLoadPartial(unittest.TestCase):
......@@ -334,6 +338,7 @@ class TestSaveLoadPartial(unittest.TestCase):
init_scale = 0.1
batch_size = 4
batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope():
fluid.default_startup_program().random_seed = seed
......@@ -409,7 +414,7 @@ class TestSaveLoadPartial(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, "./test_1")
fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
# set var to zero
for var in main_program.list_vars():
......@@ -422,7 +427,8 @@ class TestSaveLoadPartial(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(test_program, "./test_1.pdopt", None)
fluid.load(test_program,
os.path.join(temp_dir.name, "test_1.pdopt"), None)
for var in test_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -430,7 +436,9 @@ class TestSaveLoadPartial(unittest.TestCase):
.get_tensor())
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
fluid.load(test_program, "./test_1.pdmodel", None)
fluid.load(test_program,
os.path.join(temp_dir.name, "test_1.pdmodel"), None)
temp_dir.cleanup()
class TestSaveLoadSetStateDict(unittest.TestCase):
......@@ -447,6 +455,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
init_scale = 0.1
batch_size = 4
batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope():
fluid.default_startup_program().random_seed = seed
......@@ -514,7 +523,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, "./test_1")
fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
# set var to zero
for var in main_program.list_vars():
......@@ -527,7 +536,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "./test_1", exe)
fluid.load(main_program, os.path.join(temp_dir.name, "test_1"), exe)
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -535,6 +544,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
.get_tensor())
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestProgramStatePartial(unittest.TestCase):
......@@ -551,6 +561,7 @@ class TestProgramStatePartial(unittest.TestCase):
init_scale = 0.1
batch_size = 4
batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope():
fluid.default_startup_program().random_seed = seed
......@@ -626,7 +637,7 @@ class TestProgramStatePartial(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, os.path.join('some_dir', 'test_1'))
fluid.save(main_program, os.path.join(temp_dir.name, 'test_1'))
# set var to zero
for var in main_program.list_vars():
......@@ -641,16 +652,16 @@ class TestProgramStatePartial(unittest.TestCase):
#fluid.load(test_program, "./test_1", None )
program_state = fluid.load_program_state(
os.path.join('some_dir', 'test_1'))
os.path.join(temp_dir.name, 'test_1'))
program_state_1 = fluid.load_program_state(
os.path.join('some_dir', 'test_1.pdparams'))
os.path.join(temp_dir.name, 'test_1.pdparams'))
program_state_2 = fluid.load_program_state(
os.path.join('some_dir', 'test_1.pdopt'))
os.path.join(temp_dir.name, 'test_1.pdopt'))
program_state_3 = fluid.load_program_state(
os.path.join('some_dir', 'test_1.pdmodel'))
os.path.join(temp_dir.name, 'test_1.pdmodel'))
fluid.set_program_state(test_program, program_state)
......@@ -720,6 +731,7 @@ class TestProgramStatePartial(unittest.TestCase):
.get_tensor())
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestVariableInit(unittest.TestCase):
......@@ -737,7 +749,9 @@ class TestVariableInit(unittest.TestCase):
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
fluid.save(fluid.default_main_program(), "./test_path")
temp_dir = tempfile.TemporaryDirectory()
fluid.save(fluid.default_main_program(),
os.path.join(temp_dir.name, "test_path"))
def set_var(var, ndarray):
t = var.get_tensor()
......@@ -763,7 +777,7 @@ class TestVariableInit(unittest.TestCase):
fluid.core._create_loaded_parameter(parameter_list, new_scope,
exe._default_executor)
parameter_file_name = "./test_path.pdparams"
parameter_file_name = os.path.join(temp_dir.name, "test_path.pdparams")
with open(parameter_file_name, 'rb') as f:
load_dict = pickle.load(f)
......@@ -779,7 +793,7 @@ class TestVariableInit(unittest.TestCase):
fluid.core._create_loaded_parameter(opt_list, new_scope,
exe._default_executor)
opt_file_name = "./test_path.pdopt"
opt_file_name = os.path.join(temp_dir.name, "test_path.pdopt")
with open(opt_file_name, 'rb') as f:
load_dict = pickle.load(f)
......@@ -805,6 +819,7 @@ class TestVariableInit(unittest.TestCase):
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestLoadFromOldInterface(unittest.TestCase):
......@@ -815,10 +830,15 @@ class TestLoadFromOldInterface(unittest.TestCase):
if os.path.exists("test_static_load_var_list.pdparams"):
os.remove("test_static_load_var_list.pdparams")
self.temp_dir = tempfile.TemporaryDirectory()
def set_place(self):
return fluid.CPUPlace() if not core.is_compiled_with_cuda(
) else fluid.CUDAPlace(0)
def tearDown(self):
self.temp_dir.cleanup()
def test_load_from_old_interface(self):
seed = 90
hidden_size = 10
......@@ -898,7 +918,9 @@ class TestLoadFromOldInterface(unittest.TestCase):
base_map[var.name] = t
#fluid.save(main_program, "./test_1")
fluid.io.save_persistables(exe, "test_path", main_program)
fluid.io.save_persistables(
exe,
os.path.join(self.temp_dir.name, "test_path"), main_program)
# set var to zero
for var in main_program.list_vars():
......@@ -911,7 +933,8 @@ class TestLoadFromOldInterface(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "test_path", exe)
fluid.load(main_program,
os.path.join(self.temp_dir.name, "test_path"), exe)
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -928,11 +951,13 @@ class TestLoadFromOldInterface(unittest.TestCase):
var.desc.set_shape(new_shape)
with self.assertRaises(RuntimeError):
fluid.load(main_program, "test_path", exe)
fluid.load(main_program,
os.path.join(self.temp_dir.name, "test_path"), exe)
# check unused parameter
fluid.load(test_clone_program, "test_path", exe)
fluid.load(test_clone_program,
os.path.join(self.temp_dir.name, "test_path"), exe)
def test_load_from_old_interface_var_list(self):
seed = 90
......@@ -1013,7 +1038,9 @@ class TestLoadFromOldInterface(unittest.TestCase):
base_map[var.name] = t
#fluid.save(main_program, "./test_1")
fluid.io.save_persistables(exe, "test_static_load_var_list",
fluid.io.save_persistables(
exe,
os.path.join(self.temp_dir.name, "test_static_load_var_list"),
main_program)
# set var to zero
......@@ -1030,7 +1057,10 @@ class TestLoadFromOldInterface(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "test_static_load_var_list", exe, var_list)
fluid.load(
main_program,
os.path.join(self.temp_dir.name, "test_static_load_var_list"),
exe, var_list)
var_list_names = [var.name for var in var_list]
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -1059,6 +1089,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
init_scale = 0.1
batch_size = 4
batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope():
fluid.default_startup_program().random_seed = seed
......@@ -1125,10 +1156,10 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
save_dir = os.path.join(temp_dir.name, "test_path")
#fluid.save(main_program, "./test_1")
fluid.io.save_persistables(
exe, "test_path", main_program, filename="model_single")
exe, save_dir, main_program, filename="model_single")
# set var to zero
for var in main_program.list_vars():
......@@ -1141,7 +1172,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
file_model_path = os.path.join("test_path", "model_single")
file_model_path = os.path.join(save_dir, "model_single")
fluid.load(main_program, file_model_path, exe,
fluid.io.get_program_persistable_vars(main_program))
......@@ -1190,11 +1221,16 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
all_var_list = list(main_program.list_vars())
fluid.load(main_program, file_model_path, exe,
all_var_list + [temp_var])
temp_dir.cleanup()
class TestProgramStateOldSave(unittest.TestCase):
def setUp(self):
self.test_dygraph = True
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_place(self):
return fluid.CPUPlace() if not core.is_compiled_with_cuda(
......@@ -1283,8 +1319,8 @@ class TestProgramStateOldSave(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.io.save_persistables(exe, "test_program_1", main_program)
save_dir = os.path.join(self.temp_dir.name, "test_program_1")
fluid.io.save_persistables(exe, save_dir, main_program)
# set var to zero
for var in main_program.list_vars():
......@@ -1298,7 +1334,7 @@ class TestProgramStateOldSave(unittest.TestCase):
self.assertTrue(np.sum(np.abs(new_t)) == 0)
# case 1: load basic
program_state = fluid.load_program_state("test_program_1")
program_state = fluid.load_program_state(save_dir)
fluid.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map)
......@@ -1313,24 +1349,20 @@ class TestProgramStateOldSave(unittest.TestCase):
else:
raise e
orig_filepath = './test_program_1/fc_0.w_0'
symlink_filepath = './test_program_1/link_fc_0.w_0'
# create a needless link file for coverage
symlink_force(orig_filepath, symlink_filepath)
program_state = fluid.load_program_state("test_program_1")
program_state = fluid.load_program_state(save_dir)
fluid.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map)
# case 3: load with var_list
program_state = fluid.load_program_state(
"test_program_1", main_program.all_parameters())
save_dir, main_program.all_parameters())
fluid.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map)
if self.test_dygraph:
# make sure `load_program_state` can be used in dynamic graph mode
with fluid.dygraph.guard(place):
load_state = fluid.load_program_state("test_program_1")
load_state = fluid.load_program_state(save_dir)
for k, v in load_state.items():
self.assertTrue(np.array_equal(base_map[k], v))
......@@ -1365,6 +1397,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
init_scale = 0.1
batch_size = 4
batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope():
fluid.default_startup_program().random_seed = seed
......@@ -1440,8 +1473,9 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
save_dir = os.path.join(temp_dir.name, "test_program_2")
fluid.io.save_persistables(
exe, "test_program_2", main_program, filename="model_1")
exe, save_dir, main_program, filename="model_1")
# set var to zero
for var in main_program.list_vars():
......@@ -1456,7 +1490,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
#fluid.load(test_program, "./test_1", None )
program_state = fluid.load_program_state(
os.path.join("test_program_2", "model_1"),
os.path.join(save_dir, "model_1"),
var_list=fluid.io.get_program_persistable_vars(main_program))
fluid.set_program_state(main_program, program_state)
......@@ -1468,21 +1502,20 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
self.assertTrue(np.array_equal(new_t, base_t))
with self.assertRaises(ValueError):
fluid.load_program_state(
os.path.join("test_program_2", "model_1"))
fluid.load_program_state(os.path.join(save_dir, "model_1"))
with self.assertRaises(TypeError):
fluid.load_program_state(
os.path.join("test_program_2", "model_1"),
var_list=["str"])
os.path.join(save_dir, "model_1"), var_list=["str"])
with self.assertRaises(RuntimeError):
fluid.load_program_state(
os.path.join("test_program_2", "model_1"),
os.path.join(save_dir, "model_1"),
var_list=[
main_program.global_block().create_var(
name="fake_var_name", persistable=True)
])
temp_dir.cleanup()
class TestStaticSaveLoadPickle(unittest.TestCase):
......@@ -1511,7 +1544,8 @@ class TestStaticSaveLoadPickle(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
path = os.path.join("test_static_save_load_pickle",
temp_dir = tempfile.TemporaryDirectory()
path = os.path.join(temp_dir.name, "test_static_save_load_pickle",
"pickle_protocol")
with self.assertRaises(ValueError):
......
python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py
浏览文件 @ 2eb60ddb
......@@ -23,11 +23,19 @@ from paddle.fluid.optimizer import SGDOptimizer
from paddle.fluid.tests.unittests.test_imperative_base import new_program_scope
from paddle.fluid.tests.unittests.test_static_save_load import PtbModel
import numpy as np
import tempfile
import os
@unittest.skipIf(not core.supports_bfloat16(),
"place does not support BF16 evaluation")
class TestSaveLoadBF16(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_place(self):
return fluid.CPUPlace()
......@@ -108,8 +116,8 @@ class TestSaveLoadBF16(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, "./test_1")
save_dir = os.path.join(self.temp_dir.name, "test_1")
fluid.save(main_program, save_dir)
# set var to zero
for var in main_program.list_vars():
......@@ -122,7 +130,8 @@ class TestSaveLoadBF16(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "./test_1.pdparams", exe)
fluid.load(main_program,
os.path.join(self.temp_dir.name, "test_1.pdparams"), exe)
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......
python/paddle/fluid/tests/unittests/test_static_save_load_large.py
浏览文件 @ 2eb60ddb
......@@ -23,6 +23,7 @@ from test_imperative_base import new_program_scope
import numpy as np
import pickle
import os
import tempfile
LARGE_PARAM = 2**26
......@@ -51,9 +52,10 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
path = os.path.join("test_static_save_load_large_param",
"static_save")
temp_dir = tempfile.TemporaryDirectory()
path = os.path.join(temp_dir.name,
"test_static_save_load_large_param")
path = os.path.join(path, "static_save")
protocol = 4
paddle.fluid.save(prog, path, pickle_protocol=protocol)
# set var to zero
......@@ -93,6 +95,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
.get_tensor())
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
if __name__ == '__main__':
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或