Skip to content

P
Paddle

BaiXuePrincess / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle
未验证 提交 0d0258f8 编写于 作者: Y YuanRisheng 提交者： GitHub
浏览文件
操作

change saved file dir for save_load unittest (#43416)

上级 88504892
隐藏空白更改
内联 并排
Showing with 501 addition and 164 deletion
python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py
浏览文件 @ 0d0258f8
......@@ -27,6 +27,7 @@ from paddle.fluid.layer_helper import LayerHelper
from paddle import _C_ops
import sys
import tempfile
sys.path.append("./tokenizer")
from tokenizer.bert_tokenizer import BertTokenizer
......@@ -157,11 +158,15 @@ class Predictor(object):
class TestBertTokenizerOp(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.bert_tokenizer = BertTokenizer.from_pretrained("bert-base-chinese")
self.save_path = os.path.join(DATA_HOME, "fast_tokenizer")
self.save_path = os.path.join(self.temp_dir.name, "fast_tokenizer")
self.param_path = os.path.join(self.save_path, "model.pdparams")
self.inference_path = os.path.join(self.save_path, "inference")
def tearDown(self):
self.temp_dir.cleanup()
def init_data(self):
self.faster_tokenizer = FasterTokenizer(self.bert_tokenizer.vocab)
self.text = [
......
python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py
浏览文件 @ 0d0258f8
......@@ -17,12 +17,15 @@ import paddle.fluid as fluid
import paddle.fluid.framework as framework
from paddle.fluid.dygraph.nn import *
import numpy as np
import os
import tempfile
class TestDygraphLoadStatic(unittest.TestCase):
def testLoadStaticModel(self):
# static mode
temp_dir = tempfile.TemporaryDirectory()
a = fluid.data(name="a", shape=[10, 10])
conv_in = fluid.data(name="conv_in", shape=[None, 10, 10, 10])
......@@ -144,9 +147,11 @@ class TestDygraphLoadStatic(unittest.TestCase):
) if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0))
out = exe.run(framework.default_startup_program())
fluid.save(framework.default_main_program(), "./test_1")
fluid.save(framework.default_main_program(),
os.path.join(temp_dir.name, "test_1"))
para_dict = fluid.load_program_state("./test_1")
para_dict = fluid.load_program_state(
os.path.join(temp_dir.name, "test_1"))
new_dict = {}
for k, v in para_dict.items():
......@@ -214,6 +219,7 @@ class TestDygraphLoadStatic(unittest.TestCase):
my_test.set_dict(new_dict, use_structured_name=False)
for k, v in my_test.state_dict().items():
self.assertTrue(np.array_equal(v.numpy(), new_dict[v.name]))
temp_dir.cleanup()
if __name__ == '__main__':
......
python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py
浏览文件 @ 0d0258f8
......@@ -29,6 +29,8 @@ import six
import paddle
from paddle.fluid.framework import _test_eager_guard
import tempfile
class SimpleLSTMRNN(fluid.Layer):
......@@ -219,6 +221,12 @@ class PtbModel(fluid.Layer):
class TestDygraphPtbRnn(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def func_setUp(self):
seed = 90
hidden_size = 10
......@@ -295,7 +303,8 @@ class TestDygraphPtbRnn(unittest.TestCase):
else:
self.base_opti[k] = v
paddle.save(self.opti_dict, "./test_dy_v2.pdopt")
paddle.save(self.opti_dict,
os.path.join(self.temp_dir.name, "test_dy_v2.pdopt"))
self.state_dict = ptb_model.state_dict()
......@@ -304,10 +313,10 @@ class TestDygraphPtbRnn(unittest.TestCase):
np_t = v.numpy()
self.model_base[k] = np_t
paddle.save(self.state_dict, "./test_dy_v2.pdparams")
paddle.save(self.state_dict,
os.path.join(self.temp_dir.name, "test_dy_v2.pdparams"))
def func_testLoadAndSetVarBase(self):
self.setUp()
seed = 90
hidden_size = 10
vocab_size = 1000
......@@ -383,8 +392,10 @@ class TestDygraphPtbRnn(unittest.TestCase):
self.assertTrue(np.sum(np.abs(v.numpy())) == 0)
para_state_dict = paddle.load("./test_dy_v2.pdparams")
opti_state_dict = paddle.load("./test_dy_v2.pdopt")
para_state_dict = paddle.load(
os.path.join(self.temp_dir.name, "test_dy_v2.pdparams"))
opti_state_dict = paddle.load(
os.path.join(self.temp_dir.name, "test_dy_v2.pdopt"))
adam.set_state_dict(opti_state_dict)
opti_dict = adam.state_dict()
......@@ -752,7 +763,8 @@ class TestDygraphPtbRnn(unittest.TestCase):
last_hidden = None
last_cell = None
state_dict, opti_dict = fluid.load_dygraph("./test_dy_v2")
state_dict, opti_dict = fluid.load_dygraph(
os.path.join(self.temp_dir.name, "test_dy_v2"))
adam.set_state_dict(opti_dict)
ptb_model.set_dict(state_dict)
......@@ -907,19 +919,23 @@ class TestDygraphPtbRnn(unittest.TestCase):
with fluid.dygraph.guard():
emb = fluid.dygraph.Embedding([10, 10])
state_dict = emb.state_dict()
paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams'))
paddle.save(
state_dict,
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict = paddle.load(
os.path.join('saved_dy', 'emb_dy.pdparams'))
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
def func_test_no_state_in_input_dict(self):
with fluid.dygraph.guard():
emb = fluid.dygraph.Embedding([10, 10])
state_dict = emb.state_dict()
paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams'))
paddle.save(
state_dict,
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict = paddle.load(
os.path.join('saved_dy', 'emb_dy.pdparams'))
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict.pop('weight')
emb.set_state_dict(para_state_dict)
......@@ -928,9 +944,12 @@ class TestDygraphPtbRnn(unittest.TestCase):
with fluid.dygraph.guard():
emb = fluid.dygraph.Embedding([10, 10])
state_dict = emb.state_dict()
paddle.save(state_dict, os.path.join('saved_dy', 'emb_dy.pdparams'))
paddle.save(
state_dict,
os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'))
para_state_dict = paddle.load(os.path.join('saved_dy',
para_state_dict = paddle.load(os.path.join(self.temp_dir.name,
'saved_dy',
'emb_dy.pdparams'),
return_numpy=True)
para_state_dict['weight'] = np.expand_dims(
......
python/paddle/fluid/tests/unittests/test_io_save_load.py
浏览文件 @ 0d0258f8
......@@ -19,10 +19,19 @@ import paddle
import paddle.fluid as fluid
from paddle.fluid import core
from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph
import tempfile
import os
class TestSaveLoadAPIError(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.save_dir = os.path.join(self.temp_dir.name, "fake_dir")
def tearDown(self):
self.temp_dir.cleanup()
def func_test_get_valid_program_error(self):
# case 1: CompiledProgram no program
graph = core.Graph(core.ProgramDesc())
......@@ -45,13 +54,13 @@ class TestSaveLoadAPIError(unittest.TestCase):
# case 1: main_program type error when vars None
with self.assertRaises(TypeError):
fluid.io.load_vars(executor=exe,
dirname="./fake_dir",
dirname=self.save_dir,
main_program="program")
# case 2: main_program type error when vars not None
with self.assertRaises(TypeError):
fluid.io.load_vars(executor=exe,
dirname="./fake_dir",
dirname=self.save_dir,
main_program="program",
vars="vars")
......@@ -63,6 +72,12 @@ class TestSaveLoadAPIError(unittest.TestCase):
class TestSaveInferenceModelAPIError(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def func_test_useless_feeded_var_names(self):
start_prog = fluid.Program()
main_prog = fluid.Program()
......@@ -75,7 +90,8 @@ class TestSaveInferenceModelAPIError(unittest.TestCase):
exe.run(start_prog)
with self.assertRaisesRegexp(
ValueError, "not involved in the target_vars calculation"):
fluid.io.save_inference_model(dirname='./model',
fluid.io.save_inference_model(dirname=os.path.join(
self.temp_dir.name, 'model'),
feeded_var_names=['x', 'y'],
target_vars=[z],
executor=exe,
......@@ -89,13 +105,20 @@ class TestSaveInferenceModelAPIError(unittest.TestCase):
class TestWhenTrainWithNoGrad(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def func_test_when_train_with_no_grad(self):
paddle.disable_static()
net = paddle.nn.Linear(1024, 1)
net = paddle.jit.to_static(net)
x = paddle.rand([1024], 'float32')
net(x)
save_path = './train_with_no_grad'
save_path = os.path.join(self.temp_dir.name, 'train_with_no_grad')
paddle.jit.save(net, save_path)
net = paddle.jit.load(save_path)
net.train()
......
python/paddle/fluid/tests/unittests/test_jit_save_load.py
浏览文件 @ 0d0258f8
......@@ -18,6 +18,7 @@ import os
import pickle
import shutil
import unittest
import tempfile
import numpy as np
import paddle
from paddle.static import InputSpec
......@@ -347,13 +348,18 @@ def train_with_label(layer, input_size=784, label_size=1):
class TestJitSaveLoad(unittest.TestCase):
def setUp(self):
self.model_path = "test_jit_save_load/model"
self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"test_jit_save_load/model")
# enable dygraph mode
fluid.enable_dygraph()
# config seed
paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED)
def tearDown(self):
self.temp_dir.cleanup()
def train_and_save_model(self, model_path=None):
layer = LinearNet(784, 1)
example_inputs, layer, _ = train(layer)
......@@ -412,12 +418,14 @@ class TestJitSaveLoad(unittest.TestCase):
new_layer(x).numpy()))
def test_load_dygraph_no_path(self):
model_path = "test_jit_save_load.no_path/model_path"
model_path = os.path.join(self.temp_dir.name,
"test_jit_save_load.no_path/model_path")
with self.assertRaises(ValueError):
model_dict, _ = fluid.dygraph.load_dygraph(model_path)
def test_jit_load_no_path(self):
path = "test_jit_save_load.no_path/model_path"
path = os.path.join(self.temp_dir.name,
"test_jit_save_load.no_path/model_path")
with self.assertRaises(ValueError):
loaded_layer = paddle.jit.load(path)
......@@ -427,6 +435,10 @@ class TestSaveLoadWithNestOut(unittest.TestCase):
def setUp(self):
# enable dygraph mode
fluid.enable_dygraph()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_nest_output(self):
x = fluid.dygraph.to_variable(
......@@ -436,7 +448,7 @@ class TestSaveLoadWithNestOut(unittest.TestCase):
dy_outs = flatten(net(x))
net = declarative(net, input_spec=[InputSpec([None, 8], name='x')])
model_path = "net_with_nest_out/model"
model_path = os.path.join(self.temp_dir.name, "net_with_nest_out/model")
paddle.jit.save(net, model_path)
load_net = paddle.jit.load(model_path)
......@@ -459,8 +471,9 @@ class TestSaveLoadWithDictInput(unittest.TestCase):
# {'img': var img : fluid.VarType.LOD_TENSOR.shape(-1, 8).astype(VarType.FP32)},
# {'label': var label : fluid.VarType.LOD_TENSOR.shape(-1, 1).astype(VarType.INT64)})
self.assertEqual(len(net.forward.concrete_program.inputs), 3)
path = "test_jit_save_load_with_dict_input/model"
temp_dir = tempfile.TemporaryDirectory()
path = os.path.join(temp_dir.name,
"test_jit_save_load_with_dict_input/model")
# prune inputs
paddle.jit.save(layer=net,
path=path,
......@@ -478,14 +491,16 @@ class TestSaveLoadWithDictInput(unittest.TestCase):
# loaded_net._input_spec():
# [InputSpec(shape=(-1, 8), dtype=VarType.FP32, name=img)]
self.assertEqual(len(loaded_net._input_spec()), 1)
temp_dir.cleanup()
class TestSaveLoadWithDictInputNoPrune(unittest.TestCase):
def test_dict_input(self):
net = LinearNetWithDictInputNoPrune(8, 8)
path = "test_jit_save_load_with_dict_input_no_prune/model"
temp_dir = tempfile.TemporaryDirectory()
path = os.path.join(
temp_dir.name, "test_jit_save_load_with_dict_input_no_prune/model")
# prune inputs
paddle.jit.save(layer=net,
path=path,
......@@ -506,6 +521,7 @@ class TestSaveLoadWithDictInputNoPrune(unittest.TestCase):
loaded_out = loaded_net(img, img2)
self.assertEqual(len(loaded_net._input_spec()), 2)
temp_dir.cleanup()
class TestSaveLoadWithInputSpec(unittest.TestCase):
......@@ -513,6 +529,10 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
def setUp(self):
# enable dygraph mode
fluid.enable_dygraph()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_with_input_spec(self):
net = LinearNetReturnLoss(8, 8)
......@@ -520,7 +540,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
net.forward = declarative(net.forward,
input_spec=[InputSpec([None, 8], name='x')])
model_path = "input_spec.output_spec/model"
model_path = os.path.join(self.temp_dir.name,
"input_spec.output_spec/model")
# check inputs and outputs
self.assertTrue(len(net.forward.inputs) == 1)
input_x = net.forward.inputs[0]
......@@ -540,7 +561,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
def test_multi_in_out(self):
net = LinearNetMultiInput(8, 8)
model_path = "multi_inout.output_spec1/model"
model_path = os.path.join(self.temp_dir.name,
"multi_inout.output_spec1/model")
# 1. check inputs and outputs
self.assertTrue(len(net.forward.inputs) == 2)
input_x = net.forward.inputs[0]
......@@ -562,7 +584,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
pred_x, pred_y = infer_layer(x, y)
# 1. prune y and loss
model_path = "multi_inout.output_spec2/model"
model_path = os.path.join(self.temp_dir.name,
"multi_inout.output_spec2/model")
output_spec = net.forward.outputs[:1]
paddle.jit.save(net, model_path, [input_x], output_spec=output_spec)
# 2. load again
......@@ -576,7 +599,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
def test_multi_in_out1(self):
net = LinearNetMultiInput1(8, 8)
model_path = "multi_inout1.output_spec1/model"
model_path = os.path.join(self.temp_dir.name,
"multi_inout1.output_spec1/model")
# 1. check inputs and outputs
self.assertTrue(len(net.forward.inputs) == 2)
input_x = net.forward.inputs[0]
......@@ -598,7 +622,8 @@ class TestSaveLoadWithInputSpec(unittest.TestCase):
pred_x, pred_y = infer_layer(x, y)
# 1. prune y and loss
model_path = "multi_inout1.output_spec2/model"
model_path = os.path.join(self.temp_dir.name,
"multi_inout1.output_spec2/model")
output_spec = net.forward.outputs[:1]
paddle.jit.save(net, model_path, (input_x, ), output_spec=output_spec)
# 2. load again
......@@ -618,6 +643,10 @@ class TestJitSaveLoadConfig(unittest.TestCase):
# config seed
paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED)
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_output_spec(self):
train_layer = LinearNetReturnLoss(8, 8)
......@@ -631,7 +660,8 @@ class TestJitSaveLoadConfig(unittest.TestCase):
adam.minimize(loss)
train_layer.clear_gradients()
model_path = "save_load_config.output_spec"
model_path = os.path.join(self.temp_dir.name,
"save_load_config.output_spec")
output_spec = [out]
paddle.jit.save(layer=train_layer,
path=model_path,
......@@ -648,22 +678,22 @@ class TestJitSaveLoadConfig(unittest.TestCase):
def test_save_no_support_config_error(self):
layer = LinearNet(784, 1)
path = "no_support_config_test"
path = os.path.join(self.temp_dir.name, "no_support_config_test")
with self.assertRaises(ValueError):
paddle.jit.save(layer=layer, path=path, model_filename="")
def test_load_empty_model_filename_error(self):
path = "error_model_filename_test"
path = os.path.join(self.temp_dir.name, "error_model_filename_test")
with self.assertRaises(ValueError):
paddle.jit.load(path, model_filename="")
def test_load_empty_params_filename_error(self):
path = "error_params_filename_test"
path = os.path.join(self.temp_dir.name, "error_params_filename_test")
with self.assertRaises(ValueError):
paddle.jit.load(path, params_filename="")
def test_load_with_no_support_config(self):
path = "no_support_config_test"
path = os.path.join(self.temp_dir.name, "no_support_config_test")
with self.assertRaises(ValueError):
paddle.jit.load(path, separate_params=True)
......@@ -672,7 +702,9 @@ class TestJitMultipleLoading(unittest.TestCase):
def setUp(self):
self.linear_size = 4
self.model_path = "jit_multi_load/model"
self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"jit_multi_load/model")
# enable dygraph mode
fluid.enable_dygraph()
# config seed
......@@ -681,6 +713,9 @@ class TestJitMultipleLoading(unittest.TestCase):
# train and save base model
self.train_and_save_orig_model()
def tearDown(self):
self.temp_dir.cleanup()
def train_and_save_orig_model(self):
layer = LinearNet(self.linear_size, self.linear_size)
example_inputs, layer, _ = train(layer, self.linear_size, 1)
......@@ -702,13 +737,18 @@ class TestJitPruneModelAndLoad(unittest.TestCase):
def setUp(self):
self.linear_size = 4
self.model_path = "jit_prune_model_and_load/model"
self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"jit_prune_model_and_load/model")
# enable dygraph mode
fluid.enable_dygraph()
# config seed
paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED)
def tearDown(self):
self.temp_dir.cleanup()
def train_and_save(self):
train_layer = LinearNetReturnHidden(8, 8)
adam = fluid.optimizer.AdamOptimizer(
......@@ -764,6 +804,10 @@ class TestJitSaveMultiCases(unittest.TestCase):
# config seed
paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED)
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def verify_inference_correctness(self,
layer,
......@@ -796,7 +840,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train(layer)
model_path = "test_no_prune_to_static_after_train/model"
model_path = os.path.join(self.temp_dir.name,
"test_no_prune_to_static_after_train/model")
paddle.jit.save(layer, model_path)
self.verify_inference_correctness(layer, model_path)
......@@ -804,7 +849,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_no_prune_to_static_no_train(self):
layer = LinearNetWithInputSpec(784, 1)
model_path = "test_no_prune_to_static_no_train/model"
model_path = os.path.join(self.temp_dir.name,
"test_no_prune_to_static_no_train/model")
paddle.jit.save(layer, model_path)
self.verify_inference_correctness(layer, model_path)
......@@ -814,7 +860,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train(layer)
model_path = "test_no_prune_no_to_static_after_train/model"
model_path = os.path.join(
self.temp_dir.name, "test_no_prune_no_to_static_after_train/model")
paddle.jit.save(
layer,
model_path,
......@@ -827,7 +874,9 @@ class TestJitSaveMultiCases(unittest.TestCase):
example_inputs, _, _ = train(layer)
model_path = "test_no_prune_no_to_static_after_train_with_examples/model"
model_path = os.path.join(
self.temp_dir.name,
"test_no_prune_no_to_static_after_train_with_examples/model")
paddle.jit.save(layer=layer, path=model_path, input_spec=example_inputs)
self.verify_inference_correctness(layer, model_path)
......@@ -835,7 +884,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_no_prune_no_to_static_no_train(self):
layer = LinearNetNotDeclarative(784, 1)
model_path = "test_no_prune_no_to_static_no_train/model"
model_path = os.path.join(self.temp_dir.name,
"test_no_prune_no_to_static_no_train/model")
paddle.jit.save(
layer,
model_path,
......@@ -848,7 +898,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
out = train_with_label(layer)
model_path = "test_prune_to_static_after_train/model"
model_path = os.path.join(self.temp_dir.name,
"test_prune_to_static_after_train/model")
paddle.jit.save(layer,
model_path,
input_spec=[
......@@ -865,7 +916,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_prune_to_static_no_train(self):
layer = LinerNetWithLabel(784, 1)
model_path = "test_prune_to_static_no_train/model"
model_path = os.path.join(self.temp_dir.name,
"test_prune_to_static_no_train/model")
# TODO: no train, cannot get output_spec var here
# now only can use index
output_spec = layer.forward.outputs[:1]
......@@ -885,7 +937,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_prune_input_to_static_no_train(self):
layer = LinerNetWithPruneInput(784, 1)
model_path = "test_prune_input_to_static_no_train/model"
model_path = os.path.join(self.temp_dir.name,
"test_prune_input_to_static_no_train/model")
paddle.jit.save(layer,
model_path,
input_spec=[
......@@ -899,7 +952,9 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_prune_useless_input_to_static_no_train(self):
layer = LinerNetWithUselessInput(784, 1)
model_path = "test_prune_useless_input_to_static_no_train/model"
model_path = os.path.join(
self.temp_dir.name,
"test_prune_useless_input_to_static_no_train/model")
paddle.jit.save(layer,
model_path,
input_spec=[
......@@ -915,7 +970,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train(layer)
model_path = "test_no_prune_input_spec_name_warning/model"
model_path = os.path.join(
self.temp_dir.name, "test_no_prune_input_spec_name_warning/model")
paddle.jit.save(
layer,
model_path,
......@@ -935,7 +991,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train(layer)
model_path = "test_not_prune_output_spec_name_warning/model"
model_path = os.path.join(
self.temp_dir.name, "test_not_prune_output_spec_name_warning/model")
out = paddle.to_tensor(np.random.random((1, 1)).astype('float'))
paddle.jit.save(layer, model_path, output_spec=[out])
......@@ -944,7 +1001,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
def test_prune_input_spec_name_error(self):
layer = LinerNetWithLabel(784, 1)
model_path = "test_prune_input_spec_name_error/model"
model_path = os.path.join(self.temp_dir.name,
"test_prune_input_spec_name_error/model")
with self.assertRaises(ValueError):
paddle.jit.save(
layer,
......@@ -964,7 +1022,8 @@ class TestJitSaveMultiCases(unittest.TestCase):
train_with_label(layer)
model_path = "test_prune_to_static_after_train/model"
model_path = os.path.join(self.temp_dir.name,
"test_prune_to_static_after_train/model")
out = paddle.to_tensor(np.random.random((1, 1)).astype('float'))
with self.assertRaises(ValueError):
paddle.jit.save(layer,
......@@ -980,10 +1039,15 @@ class TestJitSaveMultiCases(unittest.TestCase):
class TestJitSaveLoadEmptyLayer(unittest.TestCase):
def setUp(self):
self.model_path = "jit_save_load_empty_layer/model"
self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"jit_save_load_empty_layer/model")
# enable dygraph mode
paddle.disable_static()
def tearDown(self):
self.temp_dir.cleanup()
def test_save_load_empty_layer(self):
layer = EmptyLayer()
x = paddle.to_tensor(np.random.random((10)).astype('float32'))
......@@ -997,10 +1061,15 @@ class TestJitSaveLoadEmptyLayer(unittest.TestCase):
class TestJitSaveLoadNoParamLayer(unittest.TestCase):
def setUp(self):
self.model_path = "jit_save_load_no_param_layer/model"
self.temp_dir = tempfile.TemporaryDirectory()
self.model_path = os.path.join(self.temp_dir.name,
"jit_save_load_no_param_layer/model")
# enable dygraph mode
paddle.disable_static()
def tearDown(self):
self.temp_dir.cleanup()
def test_save_load_no_param_layer(self):
layer = NoParamLayer()
x = paddle.to_tensor(np.random.random((5)).astype('float32'))
......@@ -1017,9 +1086,14 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase):
def setUp(self):
# enable dygraph mode
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_inference(self):
model_path_inference = "jit_save_load_multi_methods/model"
model_path_inference = os.path.join(
self.temp_dir.name, "jit_save_load_multi_methods/model")
IMAGE_SIZE = 224
layer = LinearNetWithMultiStaticFunc(IMAGE_SIZE, 10)
inps = paddle.randn([1, IMAGE_SIZE])
......@@ -1035,7 +1109,8 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase):
getattr(load_net, func, None)(inps)).abs().max()) < 1e-5)
def test_jit_save_load_multi_methods_inputspec(self):
model_path = 'jit_save_load_multi_methods/model'
model_path = os.path.join(self.temp_dir.name,
'jit_save_load_multi_methods/model')
layer = LinearNetWithMultiStaticFunc(784, 1)
with self.assertRaises(ValueError):
paddle.jit.save(layer,
......@@ -1043,7 +1118,8 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase):
input_spec=[InputSpec(shape=[None, 784])])
def test_parse_name(self):
model_path_inference = "jit_save_load_parse_name/model"
model_path_inference = os.path.join(self.temp_dir.name,
"jit_save_load_parse_name/model")
IMAGE_SIZE = 224
layer = LinearNet(IMAGE_SIZE, 1)
inps = paddle.randn([1, IMAGE_SIZE])
......@@ -1116,9 +1192,14 @@ class TestJitSaveLoadSaveWithoutRunning(unittest.TestCase):
def setUp(self):
# enable dygraph mode
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_save_load_finetune_load(self):
model_path = "test_jit_save_load_save_without_running/model"
model_path = os.path.join(
self.temp_dir.name, "test_jit_save_load_save_without_running/model")
IMAGE_SIZE = 224
inps0 = paddle.randn([1, IMAGE_SIZE])
inps1 = paddle.randn([2, IMAGE_SIZE])
......@@ -1157,9 +1238,14 @@ class TestJitSaveLoadFinetuneLoad(unittest.TestCase):
def setUp(self):
# enable dygraph mode
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_save_load_finetune_load(self):
model_path = "test_jit_save_load_finetune_load/model"
model_path = os.path.join(self.temp_dir.name,
"test_jit_save_load_finetune_load/model")
IMAGE_SIZE = 224
inps0 = paddle.randn([1, IMAGE_SIZE])
inps1 = paddle.randn([2, IMAGE_SIZE])
......@@ -1196,6 +1282,10 @@ class TestJitSaveLoadFunctionCase1(unittest.TestCase):
def setUp(self):
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_static_function(self):
......@@ -1203,7 +1293,8 @@ class TestJitSaveLoadFunctionCase1(unittest.TestCase):
def fun(inputs):
return paddle.tanh(inputs)
path = 'test_jit_save_load_function_1/func'
path = os.path.join(self.temp_dir.name,
'test_jit_save_load_function_1/func')
inps = paddle.rand([3, 6])
origin = fun(inps)
......@@ -1218,6 +1309,10 @@ class TestJitSaveLoadFunctionCase2(unittest.TestCase):
def setUp(self):
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function_input_spec(self):
......@@ -1227,7 +1322,8 @@ class TestJitSaveLoadFunctionCase2(unittest.TestCase):
def fun(inputs):
return paddle.nn.functional.relu(inputs)
path = 'test_jit_save_load_function_2/func'
path = os.path.join(self.temp_dir.name,
'test_jit_save_load_function_2/func')
inps = paddle.rand([3, 6])
origin = fun(inps)
......@@ -1241,13 +1337,18 @@ class TestJitSaveLoadFunctionCase3(unittest.TestCase):
def setUp(self):
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function_function(self):
def fun(inputs):
return paddle.tanh(inputs)
path = 'test_jit_save_load_function_3/func'
path = os.path.join(self.temp_dir.name,
'test_jit_save_load_function_3/func')
inps = paddle.rand([3, 6])
origin = fun(inps)
......@@ -1268,6 +1369,10 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase):
def setUp(self):
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function(self):
......@@ -1290,7 +1395,9 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase):
func = paddle.jit.to_static(layer.anothor_forward,
[paddle.static.InputSpec(shape=[-1, 5])])
path = 'test_jit_save_load_function_with_params_case1/func'
path = os.path.join(
self.temp_dir.name,
'test_jit_save_load_function_with_params_case1/func')
paddle.jit.save(func, path)
load_func = paddle.jit.load(path)
......@@ -1302,6 +1409,10 @@ class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase):
def setUp(self):
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function(self):
......@@ -1322,7 +1433,9 @@ class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase):
inps = paddle.rand([3, 5])
path = 'test_jit_save_load_function_with_params_case2/func'
path = os.path.join(
self.temp_dir.name,
'test_jit_save_load_function_with_params_case2/func')
paddle.jit.save(layer.anothor_forward, path)
origin_result = layer.anothor_forward(inps)
load_func = paddle.jit.load(path)
......@@ -1337,6 +1450,10 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase):
def setUp(self):
paddle.disable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_jit_save_load_function(self):
......@@ -1358,7 +1475,9 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase):
inps = paddle.rand([3, 5])
origin = layer.anothor_forward(inps)
path = 'test_jit_save_load_function_with_params_case3/func'
path = os.path.join(
self.temp_dir.name,
'test_jit_save_load_function_with_params_case3/func')
paddle.jit.save(layer.anothor_forward, path)
load_func = paddle.jit.load(path)
......@@ -1368,6 +1487,12 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase):
class TestJitSaveLoadDataParallel(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def verify_inference_correctness(self, layer, path):
layer.eval()
loaded_layer = paddle.jit.load(path)
......@@ -1384,8 +1509,8 @@ class TestJitSaveLoadDataParallel(unittest.TestCase):
def test_jit_save_data_parallel_with_inputspec(self):
layer = LinearNetNotDeclarative(784, 1)
layer = paddle.DataParallel(layer)
path = "jit_save_data_parallel_with_inputspec/model"
path = os.path.join(self.temp_dir.name,
"jit_save_data_parallel_with_inputspec/model")
paddle.jit.save(layer=layer,
path=path,
input_spec=[InputSpec(shape=[None, 784])])
......@@ -1396,7 +1521,8 @@ class TestJitSaveLoadDataParallel(unittest.TestCase):
layer = LinearNetWithInputSpec(784, 1)
layer = paddle.DataParallel(layer)
path = "jit_save_data_parallel_with_to_static/model"
path = os.path.join(self.temp_dir.name,
"jit_save_data_parallel_with_to_static/model")
paddle.jit.save(layer, path)
self.verify_inference_correctness(layer, path)
......@@ -1417,6 +1543,12 @@ class InputSepcLayer(paddle.nn.Layer):
class TestInputSpecCompatibility(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def _assert_input_spec_layer_return(self, expect_layer, test_layer):
input_x = paddle.uniform([8, 8], dtype='float32')
input_y = paddle.uniform([8, 1], dtype='float64')
......@@ -1429,7 +1561,8 @@ class TestInputSpecCompatibility(unittest.TestCase):
def test_jit_save_compatible_input_sepc(self):
layer = InputSepcLayer()
save_dir = "jit_save_compatible_input_spec"
save_dir = os.path.join(self.temp_dir.name,
"jit_save_compatible_input_spec")
path = save_dir + "/model"
paddle.jit.save(layer=layer, path=path)
......@@ -1463,7 +1596,8 @@ class TestInputSpecCompatibility(unittest.TestCase):
def test_jit_save_incompatible_input_sepc(self):
layer = InputSepcLayer()
save_dir = "jit_save_compatible_input_spec"
save_dir = os.path.join(self.temp_dir.name,
"jit_save_compatible_input_spec")
path = save_dir + "/model"
with self.assertRaises(ValueError):
......
python/paddle/fluid/tests/unittests/test_load_op.py
浏览文件 @ 0d0258f8
......@@ -19,6 +19,8 @@ import numpy as np
from op_test import OpTest, randomize_probability
import paddle.fluid as fluid
import paddle.fluid.layers as layers
import os
import tempfile
class TestLoadOp(unittest.TestCase):
......@@ -26,6 +28,7 @@ class TestLoadOp(unittest.TestCase):
"""
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.ones = np.ones((4, 4)).astype('float32')
main_prog = fluid.Program()
start_prog = fluid.Program()
......@@ -41,15 +44,20 @@ class TestLoadOp(unittest.TestCase):
exe = fluid.Executor(fluid.CPUPlace())
exe.run(start_prog)
fluid.io.save_persistables(exe,
dirname="./model",
dirname=os.path.join(self.temp_dir.name,
"./model"),
main_program=main_prog)
def tearDown(self):
self.temp_dir.cleanup()
def test_load(self):
main_prog = fluid.Program()
start_prog = fluid.Program()
with fluid.program_guard(main_prog, start_prog):
var = layers.create_tensor(dtype='float32')
layers.load(var, file_path='./model/w')
layers.load(var,
file_path=os.path.join(self.temp_dir.name, './model/w'))
exe = fluid.Executor(fluid.CPUPlace())
exe.run(start_prog)
......
python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py
浏览文件 @ 0d0258f8
......@@ -23,6 +23,7 @@ import paddle
import paddle.fluid as fluid
from paddle.fluid import core
from test_imperative_base import new_program_scope
import tempfile
def convolutional_neural_network(img):
......@@ -58,6 +59,7 @@ def static_train_net(img, label):
class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.seed = 90
self.epoch_num = 1
self.batch_size = 128
......@@ -65,6 +67,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
# enable static mode
paddle.enable_static()
def tearDown(self):
self.temp_dir.cleanup()
def train_and_save_model(self, only_params=False):
with new_program_scope():
startup_program = fluid.default_startup_program()
......@@ -121,7 +126,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.assertTrue(np.array_equal(value, load_dict[var_name]))
def test_load_default(self):
self.save_dirname = "static_mnist.load_state_dict.default"
self.save_dirname = os.path.join(
self.temp_dir.name, "static_mnist.load_state_dict.default")
self.model_filename = None
self.params_filename = None
orig_param_dict = self.train_and_save_model()
......@@ -133,7 +139,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_with_model_filename(self):
self.save_dirname = "static_mnist.load_state_dict.model_filename"
self.save_dirname = os.path.join(
self.temp_dir.name, "static_mnist.load_state_dict.model_filename")
self.model_filename = "static_mnist.model"
self.params_filename = None
orig_param_dict = self.train_and_save_model()
......@@ -147,7 +154,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_with_param_filename(self):
self.save_dirname = "static_mnist.load_state_dict.param_filename"
self.save_dirname = os.path.join(
self.temp_dir.name, "static_mnist.load_state_dict.param_filename")
self.model_filename = None
self.params_filename = "static_mnist.params"
orig_param_dict = self.train_and_save_model()
......@@ -161,7 +169,9 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_with_model_and_param_filename(self):
self.save_dirname = "static_mnist.load_state_dict.model_and_param_filename"
self.save_dirname = os.path.join(
self.temp_dir.name,
"static_mnist.load_state_dict.model_and_param_filename")
self.model_filename = "static_mnist.model"
self.params_filename = "static_mnist.params"
orig_param_dict = self.train_and_save_model()
......@@ -178,7 +188,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase):
self.check_load_state_dict(orig_param_dict, new_load_param_dict)
def test_load_state_dict_from_save_params(self):
self.save_dirname = "static_mnist.load_state_dict.save_params"
self.save_dirname = os.path.join(
self.temp_dir.name, "static_mnist.load_state_dict.save_params")
self.params_filename = None
orig_param_dict = self.train_and_save_model(True)
......
python/paddle/fluid/tests/unittests/test_paddle_save_load.py
浏览文件 @ 0d0258f8
......@@ -19,6 +19,7 @@ import numpy as np
import os
import sys
from io import BytesIO
import tempfile
import paddle
import paddle.nn as nn
......@@ -95,7 +96,10 @@ def train(layer, loader, loss_fn, opt):
class TestSaveLoadLargeParameters(unittest.TestCase):
def setUp(self):
pass
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_large_parameters_paddle_save(self):
# enable dygraph mode
......@@ -105,7 +109,8 @@ class TestSaveLoadLargeParameters(unittest.TestCase):
layer = LayerWithLargeParameters()
save_dict = layer.state_dict()
path = os.path.join("test_paddle_save_load_large_param_save",
path = os.path.join(self.temp_dir.name,
"test_paddle_save_load_large_param_save",
"layer.pdparams")
protocol = 4
paddle.save(save_dict, path, protocol=protocol)
......@@ -117,6 +122,12 @@ class TestSaveLoadLargeParameters(unittest.TestCase):
class TestSaveLoadPickle(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def test_pickle_protocol(self):
# enable dygraph mode
paddle.disable_static()
......@@ -124,7 +135,8 @@ class TestSaveLoadPickle(unittest.TestCase):
layer = LinearNet()
save_dict = layer.state_dict()
path = os.path.join("test_paddle_save_load_pickle_protocol",
path = os.path.join(self.temp_dir.name,
"test_paddle_save_load_pickle_protocol",
"layer.pdparams")
with self.assertRaises(ValueError):
......@@ -152,6 +164,12 @@ class TestSaveLoadPickle(unittest.TestCase):
class TestSaveLoadAny(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_zero(self, prog, place, scope=None):
if scope is None:
scope = fluid.global_scope()
......@@ -213,7 +231,8 @@ class TestSaveLoadAny(unittest.TestCase):
t = np.array(fluid.global_scope().find_var(
var.name).get_tensor())
base_map[var.name] = t
path = os.path.join("test_replace_static_save_load", "model")
path = os.path.join(self.temp_dir.name,
"test_replace_static_save_load", "model")
# paddle.save, legacy paddle.fluid.load
self.replace_static_save(prog, path)
self.set_zero(prog, place)
......@@ -239,7 +258,9 @@ class TestSaveLoadAny(unittest.TestCase):
for var in prog.list_vars():
if var.persistable:
tensor = var.get_value(fluid.global_scope())
paddle.save(tensor, os.path.join(path_vars, var.name))
paddle.save(
tensor,
os.path.join(self.temp_dir.name, path_vars, var.name))
with self.assertRaises(TypeError):
var.get_value('fluid.global_scope()')
with self.assertRaises(ValueError):
......@@ -257,7 +278,8 @@ class TestSaveLoadAny(unittest.TestCase):
self.set_zero(prog, place)
for var in prog.list_vars():
if var.persistable:
tensor = paddle.load(os.path.join(path_vars, var.name),
tensor = paddle.load(os.path.join(self.temp_dir.name,
path_vars, var.name),
return_numpy=False)
var.set_value(tensor)
new_t = np.array(fluid.global_scope().find_var(
......@@ -293,7 +315,8 @@ class TestSaveLoadAny(unittest.TestCase):
y.mean().backward()
adam.step()
state_dict = adam.state_dict()
path = 'paddle_save_load_v2/model.pdparams'
path = os.path.join(self.temp_dir.name,
'paddle_save_load_v2/model.pdparams')
with self.assertRaises(TypeError):
paddle.save(state_dict, path, use_binary_format='False')
# legacy paddle.save, paddle.load
......@@ -317,7 +340,8 @@ class TestSaveLoadAny(unittest.TestCase):
# enable dygraph mode
paddle.disable_static()
layer = LinearNet()
path = 'paddle_save_load_v2/var_dygraph'
path = os.path.join(self.temp_dir.name,
'paddle_save_load_v2/var_dygraph')
tensor = layer._linear.weight
with self.assertRaises(ValueError):
paddle.save(tensor, path, pickle_protocol='3')
......@@ -361,7 +385,8 @@ class TestSaveLoadAny(unittest.TestCase):
break
scope = fluid.global_scope()
origin_tensor = np.array(tensor)
path = 'test_single_pickle_var_static/var'
path = os.path.join(self.temp_dir.name,
'test_single_pickle_var_static/var')
paddle.save(tensor, path)
self.set_zero(prog, place, scope)
# static load
......@@ -383,7 +408,8 @@ class TestSaveLoadAny(unittest.TestCase):
def test_dygraph_save_static_load(self):
inps = np.random.randn(1, IMAGE_SIZE).astype('float32')
path = 'test_dygraph_save_static_load/dy-static.pdparams'
path = os.path.join(self.temp_dir.name,
'test_dygraph_save_static_load/dy-static.pdparams')
paddle.disable_static()
with paddle.utils.unique_name.guard():
layer = LinearNet()
......@@ -427,10 +453,14 @@ class TestSaveLoadAny(unittest.TestCase):
})
obj4 = (np.random.randn(5, 6), (123, ))
path1 = "test_save_load_any_complex_object_dygraph/obj1"
path2 = "test_save_load_any_complex_object_dygraph/obj2"
path3 = "test_save_load_any_complex_object_dygraph/obj3"
path4 = "test_save_load_any_complex_object_dygraph/obj4"
path1 = os.path.join(self.temp_dir.name,
"test_save_load_any_complex_object_dygraph/obj1")
path2 = os.path.join(self.temp_dir.name,
"test_save_load_any_complex_object_dygraph/obj2")
path3 = os.path.join(self.temp_dir.name,
"test_save_load_any_complex_object_dygraph/obj3")
path4 = os.path.join(self.temp_dir.name,
"test_save_load_any_complex_object_dygraph/obj4")
paddle.save(obj1, path1)
paddle.save(obj2, path2)
paddle.save(obj3, path3)
......@@ -597,10 +627,18 @@ class TestSaveLoadAny(unittest.TestCase):
})
obj4 = (np.ndarray([3, 4], dtype="float32"), )
path1 = "test_save_load_any_complex_object_static/obj1"
path2 = "test_save_load_any_complex_object_static/obj2"
path3 = "test_save_load_any_complex_object_static/obj3"
path4 = "test_save_load_any_complex_object_static/obj4"
path1 = os.path.join(
self.temp_dir.name,
"test_save_load_any_complex_object_static/obj1")
path2 = os.path.join(
self.temp_dir.name,
"test_save_load_any_complex_object_static/obj2")
path3 = os.path.join(
self.temp_dir.name,
"test_save_load_any_complex_object_static/obj3")
path4 = os.path.join(
self.temp_dir.name,
"test_save_load_any_complex_object_static/obj4")
paddle.save(obj1, path1)
paddle.save(obj2, path2)
paddle.save(obj3, path3)
......@@ -763,7 +801,8 @@ class TestSaveLoadAny(unittest.TestCase):
def test_varbase_binary_var(self):
paddle.disable_static()
varbase = paddle.randn([3, 2], dtype='float32')
path = 'test_paddle_save_load_varbase_binary_var/varbase'
path = os.path.join(self.temp_dir.name,
'test_paddle_save_load_varbase_binary_var/varbase')
paddle.save(varbase, path, use_binary_format=True)
load_array = paddle.load(path, return_numpy=True)
load_tensor = paddle.load(path, return_numpy=False)
......@@ -851,6 +890,10 @@ class TestSaveLoad(unittest.TestCase):
# config seed
paddle.seed(SEED)
paddle.framework.random._manual_program_seed(SEED)
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def build_and_train_model(self):
# create network
......@@ -878,8 +921,10 @@ class TestSaveLoad(unittest.TestCase):
layer, opt = self.build_and_train_model()
# save
layer_save_path = "test_paddle_save_load.linear.pdparams"
opt_save_path = "test_paddle_save_load.linear.pdopt"
layer_save_path = os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear.pdparams")
opt_save_path = os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear.pdopt")
layer_state_dict = layer.state_dict()
opt_state_dict = opt.state_dict()
......@@ -895,7 +940,9 @@ class TestSaveLoad(unittest.TestCase):
# test save load in static mode
paddle.enable_static()
static_save_path = "static_mode_test/test_paddle_save_load.linear.pdparams"
static_save_path = os.path.join(
self.temp_dir.name,
"static_mode_test/test_paddle_save_load.linear.pdparams")
paddle.save(layer_state_dict, static_save_path)
load_static_state_dict = paddle.load(static_save_path)
self.check_load_state_dict(layer_state_dict, load_static_state_dict)
......@@ -906,21 +953,30 @@ class TestSaveLoad(unittest.TestCase):
# 2. test save path format error
with self.assertRaises(ValueError):
paddle.save(layer_state_dict, "test_paddle_save_load.linear.model/")
paddle.save(
layer_state_dict,
os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear.model/"))
# 3. test load path not exist error
with self.assertRaises(ValueError):
paddle.load("test_paddle_save_load.linear.params")
paddle.load(
os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear.params"))
# 4. test load old save path error
with self.assertRaises(ValueError):
paddle.load("test_paddle_save_load.linear")
paddle.load(
os.path.join(self.temp_dir.name,
"test_paddle_save_load.linear"))
class TestSaveLoadProgram(unittest.TestCase):
def test_save_load_program(self):
paddle.enable_static()
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope():
layer = LinearNet()
data = paddle.static.data(name='x_static_save',
......@@ -931,8 +987,12 @@ class TestSaveLoadProgram(unittest.TestCase):
startup_program = paddle.static.default_startup_program()
origin_main = main_program.desc.serialize_to_string()
origin_startup = startup_program.desc.serialize_to_string()
path1 = "test_paddle_save_load_program/main_program.pdmodel"
path2 = "test_paddle_save_load_program/startup_program.pdmodel"
path1 = os.path.join(
temp_dir.name,
"test_paddle_save_load_program/main_program.pdmodel")
path2 = os.path.join(
temp_dir.name,
"test_paddle_save_load_program/startup_program.pdmodel")
paddle.save(main_program, path1)
paddle.save(startup_program, path2)
......@@ -941,12 +1001,14 @@ class TestSaveLoadProgram(unittest.TestCase):
load_startup = paddle.load(path2).desc.serialize_to_string()
self.assertTrue(origin_main == load_main)
self.assertTrue(origin_startup == load_startup)
temp_dir.cleanup()
class TestSaveLoadLayer(unittest.TestCase):
def test_save_load_layer(self):
paddle.disable_static()
temp_dir = tempfile.TemporaryDirectory()
inps = paddle.randn([1, IMAGE_SIZE], dtype='float32')
layer1 = LinearNet()
layer2 = LinearNet()
......@@ -954,9 +1016,11 @@ class TestSaveLoadLayer(unittest.TestCase):
layer2.eval()
origin_layer = (layer1, layer2)
origin = (layer1(inps), layer2(inps))
path = "test_save_load_layer_/layer.pdmodel"
path = os.path.join(temp_dir.name,
"test_save_load_layer_/layer.pdmodel")
with self.assertRaises(ValueError):
paddle.save(origin_layer, path)
temp_dir.cleanup()
if __name__ == '__main__':
......
python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py
浏览文件 @ 0d0258f8
......@@ -21,6 +21,7 @@ import os
import sys
import six
import platform
import tempfile
import paddle
import paddle.nn as nn
......@@ -38,6 +39,10 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
def setUp(self):
# enable static graph mode
paddle.enable_static()
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_zero(self, prog, place, scope=None):
if scope is None:
......@@ -97,7 +102,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
# test for replace_save_vars/io.load_vars
path_vars1 = 'test_replace_save_load_vars_binary1/model'
path_vars1 = os.path.join(
self.temp_dir.name, 'test_replace_save_load_vars_binary1/model')
self.replace_save_vars(prog, path_vars1)
# set var to zero
self.set_zero(prog, place)
......@@ -116,7 +122,9 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self.assertTrue(np.array_equal(new_t, base_t))
# test for io.save_vars/replace_load_vars
path_vars2 = 'test_replace_save_load_vars_binary2/model/'
path_vars2 = os.path.join(
self.temp_dir.name,
'test_replace_save_load_vars_binary2/model/')
fluid.io.save_vars(exe,
path_vars2,
main_program=prog,
......@@ -149,7 +157,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
prog = paddle.static.default_main_program()
exe.run(fluid.default_startup_program())
dirname = 'test_save_load_lod_tensor1/tensor_'
dirname = os.path.join(self.temp_dir.name,
'test_save_load_lod_tensor1/tensor_')
for var in prog.list_vars():
if var.persistable and list(
var.shape) == [IMAGE_SIZE, OUTPUT_NUM]:
......@@ -171,12 +180,13 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
self.assertTrue(np.array_equal(origin, to_array))
with self.assertRaises(NotImplementedError):
path = 'test_save_load_error/temp'
path = os.path.join(self.temp_dir.name, 'test_save_load_error/temp')
paddle.save({}, path, use_binary_format=True)
# On the Windows platform, when parsing a string that can't be parsed as a `Program`, `desc_.ParseFromString` has a timeout risk.
if 'Windows' != platform.system():
with self.assertRaises(ValueError):
path = 'test_save_load_error/temp'
path = os.path.join(self.temp_dir.name,
'test_save_load_error/temp')
with open(path, "w") as f:
f.write('\0')
paddle.load(path)
......@@ -187,11 +197,17 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
with self.assertRaises(RuntimeError):
fluid.core.save_lod_tensor(
temp_lod, 'test_save_load_error_not_exist_file/not_exist_file')
temp_lod,
os.path.join(
self.temp_dir.name,
'test_save_load_error_not_exist_file/not_exist_file'))
with self.assertRaises(RuntimeError):
fluid.core.load_lod_tensor(
temp_lod, 'test_save_load_error_not_exist_file/not_exist_file')
temp_lod,
os.path.join(
self.temp_dir.name,
'test_save_load_error_not_exist_file/not_exist_file'))
# save to memory
byio = BytesIO()
......@@ -215,7 +231,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
rows = [0, 4, 7]
row_numel = 12
selected_rows = fluid.core.SelectedRows(rows, height)
path = 'test_paddle_save_load_selected_rows/sr.pdsr'
path = os.path.join(self.temp_dir.name,
'test_paddle_save_load_selected_rows/sr.pdsr')
with self.assertRaises(ValueError):
paddle.save(selected_rows, path, use_binary_format=True)
......@@ -236,11 +253,15 @@ class TestSaveLoadBinaryFormat(unittest.TestCase):
with self.assertRaises(RuntimeError):
fluid.core.save_selected_rows(
selected_rows,
'test_paddle_save_load_selected_rows_not_exist_file/temp')
os.path.join(
self.temp_dir.name,
'test_paddle_save_load_selected_rows_not_exist_file/temp'))
with self.assertRaises(RuntimeError):
fluid.core.load_selected_rows(
selected_rows,
'test_paddle_save_load_selected_rows_not_exist_file/temp')
os.path.join(
self.temp_dir.name,
'test_paddle_save_load_selected_rows_not_exist_file/temp'))
# save to memory
byio = BytesIO()
......
python/paddle/fluid/tests/unittests/test_static_save_load.py
浏览文件 @ 0d0258f8
......@@ -30,6 +30,7 @@ import six
import pickle
import os
import errno
import tempfile
paddle.enable_static()
......@@ -240,6 +241,7 @@ class TestSaveLoadBase(unittest.TestCase):
init_scale = 0.1
batch_size = 4
batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope():
fluid.default_startup_program().random_seed = seed
......@@ -309,7 +311,7 @@ class TestSaveLoadBase(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, "./test_1")
fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
# set var to zero
for var in main_program.list_vars():
......@@ -322,7 +324,8 @@ class TestSaveLoadBase(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "./test_1.pdparams", exe)
fluid.load(main_program,
os.path.join(temp_dir.name, "test_1.pdparams"), exe)
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -330,6 +333,7 @@ class TestSaveLoadBase(unittest.TestCase):
var.name).get_tensor())
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestSaveLoadPartial(unittest.TestCase):
......@@ -347,6 +351,7 @@ class TestSaveLoadPartial(unittest.TestCase):
init_scale = 0.1
batch_size = 4
batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope():
fluid.default_startup_program().random_seed = seed
......@@ -424,7 +429,7 @@ class TestSaveLoadPartial(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, "./test_1")
fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
# set var to zero
for var in main_program.list_vars():
......@@ -437,7 +442,8 @@ class TestSaveLoadPartial(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(test_program, "./test_1.pdopt", None)
fluid.load(test_program, os.path.join(temp_dir.name,
"test_1.pdopt"), None)
for var in test_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -445,7 +451,9 @@ class TestSaveLoadPartial(unittest.TestCase):
var.name).get_tensor())
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
fluid.load(test_program, "./test_1.pdmodel", None)
fluid.load(test_program,
os.path.join(temp_dir.name, "test_1.pdmodel"), None)
temp_dir.cleanup()
class TestSaveLoadSetStateDict(unittest.TestCase):
......@@ -463,6 +471,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
init_scale = 0.1
batch_size = 4
batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope():
fluid.default_startup_program().random_seed = seed
......@@ -532,7 +541,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, "./test_1")
fluid.save(main_program, os.path.join(temp_dir.name, "test_1"))
# set var to zero
for var in main_program.list_vars():
......@@ -545,7 +554,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "./test_1", exe)
fluid.load(main_program, os.path.join(temp_dir.name, "test_1"), exe)
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -553,6 +562,7 @@ class TestSaveLoadSetStateDict(unittest.TestCase):
var.name).get_tensor())
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestProgramStatePartial(unittest.TestCase):
......@@ -570,6 +580,7 @@ class TestProgramStatePartial(unittest.TestCase):
init_scale = 0.1
batch_size = 4
batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope():
fluid.default_startup_program().random_seed = seed
......@@ -647,7 +658,7 @@ class TestProgramStatePartial(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, os.path.join('some_dir', 'test_1'))
fluid.save(main_program, os.path.join(temp_dir.name, 'test_1'))
# set var to zero
for var in main_program.list_vars():
......@@ -662,16 +673,16 @@ class TestProgramStatePartial(unittest.TestCase):
#fluid.load(test_program, "./test_1", None )
program_state = fluid.load_program_state(
os.path.join('some_dir', 'test_1'))
os.path.join(temp_dir.name, 'test_1'))
program_state_1 = fluid.load_program_state(
os.path.join('some_dir', 'test_1.pdparams'))
os.path.join(temp_dir.name, 'test_1.pdparams'))
program_state_2 = fluid.load_program_state(
os.path.join('some_dir', 'test_1.pdopt'))
os.path.join(temp_dir.name, 'test_1.pdopt'))
program_state_3 = fluid.load_program_state(
os.path.join('some_dir', 'test_1.pdmodel'))
os.path.join(temp_dir.name, 'test_1.pdmodel'))
fluid.set_program_state(test_program, program_state)
......@@ -741,6 +752,7 @@ class TestProgramStatePartial(unittest.TestCase):
var.name).get_tensor())
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestVariableInit(unittest.TestCase):
......@@ -759,7 +771,9 @@ class TestVariableInit(unittest.TestCase):
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
fluid.save(fluid.default_main_program(), "./test_path")
temp_dir = tempfile.TemporaryDirectory()
fluid.save(fluid.default_main_program(),
os.path.join(temp_dir.name, "test_path"))
def set_var(var, ndarray):
t = var.get_tensor()
......@@ -785,7 +799,7 @@ class TestVariableInit(unittest.TestCase):
fluid.core._create_loaded_parameter(parameter_list, new_scope,
exe._default_executor)
parameter_file_name = "./test_path.pdparams"
parameter_file_name = os.path.join(temp_dir.name, "test_path.pdparams")
with open(parameter_file_name, 'rb') as f:
load_dict = pickle.load(f)
......@@ -801,7 +815,7 @@ class TestVariableInit(unittest.TestCase):
fluid.core._create_loaded_parameter(opt_list, new_scope,
exe._default_executor)
opt_file_name = "./test_path.pdopt"
opt_file_name = os.path.join(temp_dir.name, "test_path.pdopt")
with open(opt_file_name, 'rb') as f:
load_dict = pickle.load(f)
......@@ -827,6 +841,7 @@ class TestVariableInit(unittest.TestCase):
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
class TestLoadFromOldInterface(unittest.TestCase):
......@@ -838,10 +853,15 @@ class TestLoadFromOldInterface(unittest.TestCase):
if os.path.exists("test_static_load_var_list.pdparams"):
os.remove("test_static_load_var_list.pdparams")
self.temp_dir = tempfile.TemporaryDirectory()
def set_place(self):
return fluid.CPUPlace(
) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)
def tearDown(self):
self.temp_dir.cleanup()
def test_load_from_old_interface(self):
seed = 90
hidden_size = 10
......@@ -923,7 +943,9 @@ class TestLoadFromOldInterface(unittest.TestCase):
base_map[var.name] = t
#fluid.save(main_program, "./test_1")
fluid.io.save_persistables(exe, "test_path", main_program)
fluid.io.save_persistables(
exe, os.path.join(self.temp_dir.name, "test_path"),
main_program)
# set var to zero
for var in main_program.list_vars():
......@@ -936,7 +958,8 @@ class TestLoadFromOldInterface(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "test_path", exe)
fluid.load(main_program,
os.path.join(self.temp_dir.name, "test_path"), exe)
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -953,11 +976,13 @@ class TestLoadFromOldInterface(unittest.TestCase):
var.desc.set_shape(new_shape)
with self.assertRaises(RuntimeError):
fluid.load(main_program, "test_path", exe)
fluid.load(main_program,
os.path.join(self.temp_dir.name, "test_path"), exe)
# check unused parameter
fluid.load(test_clone_program, "test_path", exe)
fluid.load(test_clone_program,
os.path.join(self.temp_dir.name, "test_path"), exe)
def test_load_from_old_interface_var_list(self):
seed = 90
......@@ -1040,8 +1065,10 @@ class TestLoadFromOldInterface(unittest.TestCase):
base_map[var.name] = t
#fluid.save(main_program, "./test_1")
fluid.io.save_persistables(exe, "test_static_load_var_list",
main_program)
fluid.io.save_persistables(
exe,
os.path.join(self.temp_dir.name, "test_static_load_var_list"),
main_program)
# set var to zero
var_list = []
......@@ -1057,7 +1084,10 @@ class TestLoadFromOldInterface(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "test_static_load_var_list", exe, var_list)
fluid.load(
main_program,
os.path.join(self.temp_dir.name, "test_static_load_var_list"),
exe, var_list)
var_list_names = [var.name for var in var_list]
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......@@ -1087,6 +1117,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
init_scale = 0.1
batch_size = 4
batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope():
fluid.default_startup_program().random_seed = seed
......@@ -1155,10 +1186,10 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
save_dir = os.path.join(temp_dir.name, "test_path")
#fluid.save(main_program, "./test_1")
fluid.io.save_persistables(exe,
"test_path",
save_dir,
main_program,
filename="model_single")
......@@ -1173,7 +1204,7 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
file_model_path = os.path.join("test_path", "model_single")
file_model_path = os.path.join(save_dir, "model_single")
fluid.load(main_program, file_model_path, exe,
fluid.io.get_program_persistable_vars(main_program))
......@@ -1223,12 +1254,17 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase):
all_var_list = list(main_program.list_vars())
fluid.load(main_program, file_model_path, exe,
all_var_list + [temp_var])
temp_dir.cleanup()
class TestProgramStateOldSave(unittest.TestCase):
def setUp(self):
self.test_dygraph = True
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_place(self):
return fluid.CPUPlace(
......@@ -1319,8 +1355,8 @@ class TestProgramStateOldSave(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.io.save_persistables(exe, "test_program_1", main_program)
save_dir = os.path.join(self.temp_dir.name, "test_program_1")
fluid.io.save_persistables(exe, save_dir, main_program)
# set var to zero
for var in main_program.list_vars():
......@@ -1334,7 +1370,7 @@ class TestProgramStateOldSave(unittest.TestCase):
self.assertTrue(np.sum(np.abs(new_t)) == 0)
# case 1: load basic
program_state = fluid.load_program_state("test_program_1")
program_state = fluid.load_program_state(save_dir)
fluid.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map)
......@@ -1349,24 +1385,20 @@ class TestProgramStateOldSave(unittest.TestCase):
else:
raise e
orig_filepath = './test_program_1/fc_0.w_0'
symlink_filepath = './test_program_1/link_fc_0.w_0'
# create a needless link file for coverage
symlink_force(orig_filepath, symlink_filepath)
program_state = fluid.load_program_state("test_program_1")
program_state = fluid.load_program_state(save_dir)
fluid.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map)
# case 3: load with var_list
program_state = fluid.load_program_state(
"test_program_1", main_program.all_parameters())
save_dir, main_program.all_parameters())
fluid.set_program_state(main_program, program_state)
self.check_in_static(main_program, base_map)
if self.test_dygraph:
# make sure `load_program_state` can be used in dynamic graph mode
with fluid.dygraph.guard(place):
load_state = fluid.load_program_state("test_program_1")
load_state = fluid.load_program_state(save_dir)
for k, v in load_state.items():
self.assertTrue(np.array_equal(base_map[k], v))
......@@ -1402,6 +1434,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
init_scale = 0.1
batch_size = 4
batch_num = 200
temp_dir = tempfile.TemporaryDirectory()
with new_program_scope():
fluid.default_startup_program().random_seed = seed
......@@ -1479,8 +1512,9 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
save_dir = os.path.join(temp_dir.name, "test_program_2")
fluid.io.save_persistables(exe,
"test_program_2",
save_dir,
main_program,
filename="model_1")
......@@ -1497,7 +1531,7 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
#fluid.load(test_program, "./test_1", None )
program_state = fluid.load_program_state(
os.path.join("test_program_2", "model_1"),
os.path.join(save_dir, "model_1"),
var_list=fluid.io.get_program_persistable_vars(main_program))
fluid.set_program_state(main_program, program_state)
......@@ -1509,21 +1543,20 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase):
self.assertTrue(np.array_equal(new_t, base_t))
with self.assertRaises(ValueError):
fluid.load_program_state(
os.path.join("test_program_2", "model_1"))
fluid.load_program_state(os.path.join(save_dir, "model_1"))
with self.assertRaises(TypeError):
fluid.load_program_state(os.path.join("test_program_2",
"model_1"),
fluid.load_program_state(os.path.join(save_dir, "model_1"),
var_list=["str"])
with self.assertRaises(RuntimeError):
fluid.load_program_state(
os.path.join("test_program_2", "model_1"),
os.path.join(save_dir, "model_1"),
var_list=[
main_program.global_block().create_var(
name="fake_var_name", persistable=True)
])
temp_dir.cleanup()
class TestStaticSaveLoadPickle(unittest.TestCase):
......@@ -1552,7 +1585,8 @@ class TestStaticSaveLoadPickle(unittest.TestCase):
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
path = os.path.join("test_static_save_load_pickle",
temp_dir = tempfile.TemporaryDirectory()
path = os.path.join(temp_dir.name, "test_static_save_load_pickle",
"pickle_protocol")
with self.assertRaises(ValueError):
......
python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py
浏览文件 @ 0d0258f8
......@@ -23,12 +23,20 @@ from paddle.fluid.optimizer import SGDOptimizer
from paddle.fluid.tests.unittests.test_imperative_base import new_program_scope
from paddle.fluid.tests.unittests.test_static_save_load import PtbModel
import numpy as np
import tempfile
import os
@unittest.skipIf(not core.supports_bfloat16(),
"place does not support BF16 evaluation")
class TestSaveLoadBF16(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
def tearDown(self):
self.temp_dir.cleanup()
def set_place(self):
return fluid.CPUPlace()
......@@ -111,8 +119,8 @@ class TestSaveLoadBF16(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
fluid.save(main_program, "./test_1")
save_dir = os.path.join(self.temp_dir.name, "test_1")
fluid.save(main_program, save_dir)
# set var to zero
for var in main_program.list_vars():
......@@ -125,7 +133,8 @@ class TestSaveLoadBF16(unittest.TestCase):
# make sure all the paramerter or optimizer var have been set to zero
self.assertTrue(np.sum(np.abs(new_t)) == 0)
fluid.load(main_program, "./test_1.pdparams", exe)
fluid.load(main_program,
os.path.join(self.temp_dir.name, "test_1.pdparams"), exe)
for var in main_program.list_vars():
if isinstance(var, framework.Parameter) or var.persistable:
......
python/paddle/fluid/tests/unittests/test_static_save_load_large.py
浏览文件 @ 0d0258f8
......@@ -23,6 +23,7 @@ from test_imperative_base import new_program_scope
import numpy as np
import pickle
import os
import tempfile
LARGE_PARAM = 2**26
......@@ -51,9 +52,10 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
# make sure all the paramerter or optimizer var have been update
self.assertTrue(np.sum(np.abs(t)) != 0)
base_map[var.name] = t
path = os.path.join("test_static_save_load_large_param",
"static_save")
temp_dir = tempfile.TemporaryDirectory()
path = os.path.join(temp_dir.name,
"test_static_save_load_large_param")
path = os.path.join(path, "static_save")
protocol = 4
paddle.fluid.save(prog, path, pickle_protocol=protocol)
# set var to zero
......@@ -93,6 +95,7 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase):
var.name).get_tensor())
base_t = base_map[var.name]
self.assertTrue(np.array_equal(new_t, base_t))
temp_dir.cleanup()
if __name__ == '__main__':
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或